diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d92a5b2..e74009a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,8 +5,8 @@ on: push: pull_request: schedule: - # Weekly live validation against Census-published values (Mondays 06:00 UTC). - - cron: "0 6 * * 1" + # Weekly live validation against Census-published values (Mondays 06:17 UTC; off the hour, since GitHub may delay or drop top-of-hour schedules). + - cron: "17 6 * * 1" workflow_dispatch: permissions: diff --git a/.gitignore b/.gitignore index 365c738..11122b2 100644 --- a/.gitignore +++ b/.gitignore @@ -44,3 +44,6 @@ docs/audits/*.csv # Agent git worktrees (local only) .claude/worktrees/ + +# Literature PDFs downloaded by hand (copyrighted; never committed) +literature_pdfs/ diff --git a/.zenodo.json b/.zenodo.json index ace836d..7f53b25 100644 --- a/.zenodo.json +++ b/.zenodo.json @@ -1,5 +1,5 @@ { - "title": "US Census ACS 5-Year Data Batch Downloader", + "title": "Census Research Engine", "description": "A Python toolkit and MCP server for extracting American Community Survey data from the US Census Bureau API into research-ready files, with vintage-safe variable concordance, margins of error for derived estimates, replicate-based variance, inference with sampling error, journal maps, and validation against Census-published values.", "upload_type": "software", "creators": [ @@ -8,6 +8,6 @@ "license": "AGPL-3.0-or-later", "keywords": ["American Community Survey", "US Census Bureau", "margins of error", "survey statistics", "research data"], "related_identifiers": [ - {"identifier": "https://github.com/cafeTechne/US-Census-5-Year-ACS-Data-Batch-Downloader", "relation": "isSupplementTo", "scheme": "url"} + {"identifier": "https://github.com/cafeTechne/Census-Research-Engine", "relation": "isSupplementTo", "scheme": "url"} ] } diff --git a/CHANGELOG.md b/CHANGELOG.md index fae6ca7..87e0259 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,45 @@ each tagged release (`v`) is archived on Zenodo with its own DOI (see [Releasing](#releasing)). Plans and acceptance criteria for each round are in `PLAN_ROUND_.md`. +## 0.15.0 (Round 16): spatial methods validated under ACS sampling error + +- Project renamed **Census Research Engine** (github.com/cafeTechne/Census-Research-Engine); + the distribution name `census-acs-toolkit` is unchanged. +- **Lag-model effects:** `spatial_regression(model="lag")` reports direct, indirect and + total effects first (delta-method and replicate SEs; verified against the S(W) algebra + and spreg's multipliers), with notes that beta is not the marginal effect and that a + rho = 0 test is not a specification test. Coverage and a heavy-tail caveat at high rho + are measured (`docs/audits/spatial_regression_validation.md`). +- **New option `spatial_regression(model="error", correct_outcome_error=True, se_col=...)`:** + corrects lambda for outcome sampling error using spatial Fay-Herriot (REML). Lambda + coverage is 0.86-0.93 against 0.003-0.85 uncorrected, measured on the shipped estimator. + - Assumes errors independent across areas. Warns when errors are correlated, and when + the outcome is a controlled total, where the sign of the bias depends on the release. + - Limited to 2,000 areas. +- **ACS sampling-error covariance measured from VRE replicates** (Michigan and New York, + 2015-2019 and 2020-2024; `docs/audits/acs_sampling_covariance.md`): + - neighbour error correlation is about 0 for rates, shares, medians and most counts; + - controlled totals are correlated within county, with a sign that can differ by + release. +- **LISA validation:** under ACS error (`docs/audits/lisa_validation.md`), with no default + change. + - The Round 15 "38%" caveat is corrected: it counts maps with at least one significant + area, not contiguous false clusters. + - Replicate stability and Fay-Herriot reruns are shown not to be false-positive checks. +- **Common-factor and specification studies:** LR size measured in Juhl's design; Koley & + Bera score tests shown not to identify SEM vs SDM. All new studies are info-only and + run on demand. +- **Polite census.gov downloads:** one shared helper (`census/http.py`) with + - backoff and Retry-After on 429/5xx; + - per-host pacing; + - a clear error for the firewall "Request Rejected" page, with the support ID and the + cache path for a manual download; + - validation of manually placed files. +- **Literature:** full-text reading notes with verification levels + (`docs/literature/full_text_reading_notes.md`). Remote-access planner decisions are + recorded in `docs/remote_access_decisions.md`. +- **CI:** the weekly live-validation cron moved off the hour (Mondays 06:17 UTC). + ## 0.14.0 (Round 15): studies across releases, spatial coverage, 2020 Census - Confirmatory studies across releases: CPI conversion to a declared `dollar_year`, then diff --git a/CITATION.cff b/CITATION.cff index f863694..1088182 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -1,6 +1,6 @@ cff-version: 1.2.0 message: "If you use this software in research, please cite it as below." -title: "US Census ACS 5-Year Data Batch Downloader" +title: "Census Research Engine" abstract: >- A Python toolkit and MCP server for extracting American Community Survey 5-year data from the US Census Bureau API into research-ready files, with @@ -11,11 +11,11 @@ type: software authors: - alias: cafeTechne license: AGPL-3.0-or-later -repository-code: "https://github.com/cafeTechne/US-Census-5-Year-ACS-Data-Batch-Downloader" -version: "0.14.0" +repository-code: "https://github.com/cafeTechne/Census-Research-Engine" +version: "0.15.0" doi: "10.5281/zenodo.23925301" # doi: Zenodo concept DOI, added after the first archived release (see CHANGELOG.md "Releasing"). -date-released: "2026-11-08" +date-released: "2026-11-09" keywords: - American Community Survey - US Census Bureau diff --git a/PLAN_ROUND_16.md b/PLAN_ROUND_16.md new file mode 100644 index 0000000..3cbfa3c --- /dev/null +++ b/PLAN_ROUND_16.md @@ -0,0 +1,173 @@ +# Round 16: Validating Spatial Methods Under ACS Sampling Error + +**Date:** 2026-11-08 +**Status:** Implemented (section 7); released as v0.15.0 +**Baseline:** v0.14.0 (spreg 1.9.1, esda 2.10.0, libpysal 4.15.1) +**Companions:** [PLAN_ROUND_15.md](PLAN_ROUND_15.md) (spatial coverage studies), [docs/remote_access_decisions.md](docs/remote_access_decisions.md) (remote access follows this round) + +## 0. Principle and findings carried in + +**Validation first:** understand each failure, test competing explanations, and only then change an estimator. New estimators or substantial methods come in only when the experiments **and** accessible methodological sources support them. Otherwise findings are documented and an implementation round is proposed. + +| ID | Finding (Round 15) | Correct reading | +| :--- | :--- | :--- | +| F1 | Error-model λ coverage 0.16-0.58 under outcome sampling error | λ is spatial autocorrelation of the **disturbances**, not a substantive covariate spillover. The failure is **attenuation** of λ by measurement error in y (no-sampling-error reference covers 0.88-0.90). Earlier summaries calling λ a "spillover term" were imprecise | +| F2 | Lag model reports β and ρ only | β is not the marginal effect in a lag model. There is no direct/indirect/total effects output, so users can misread β | +| F3 | LISA false clusters in 38% of maps ("patterned" scenario) | That scenario has **spatially heterogeneous but independent** sampling errors (CVs sorted along a gradient). It is **not** evidence about spatially **correlated** sampling errors, which are untested | + +**Two different false-cluster situations**, kept distinct throughout: + +| Scenario | Definition | +| :--- | :--- | +| Heterogeneous variances | Sampling variances vary across space, errors independent | +| Correlated errors | Sampling errors covary between areas (survey design, weighting controls) | + +A method that handles one may fail the other. FDR control does not repair p-values computed under a wrong null. + +## 1. Workstream A: spatial regression (P0) + +**A1. λ attenuation (error model).** +- Establish, from the original source, the assumptions under which a measurement-error correction for λ is valid (candidate: Lorincz-Comi et al. 2021, *Spatial Statistics* 45:100523; others if accessible). In particular, does it need only the error variances, or their covariance? +- Compare uncorrected and corrected estimators under known DGPs: bias, RMSE, interval coverage, numerical stability (non-convergence, boundary estimates). +- Sensitivity to the weights matrix (rook, queen, kNN) and to measurement-error magnitude (CV profiles). +- Run under independent and under spatially correlated errors separately. A correction validated under independence is not assumed valid under correlation. +- Implement the correction in the tool only if it validates and its assumptions are verified from an accessible source; otherwise document and propose. + +**A2. Effects for the lag model (approved).** +- First inspect spreg 1.9.1 for established impacts functionality (spatial multiplier, direct/indirect/total effects, inference) and use it if available. +- Verify the reported effects against independently computed impacts from S(W) = (I - ρW)^-1 β: the direct effect is the mean of the diagonal, the total effect is the mean row sum, and indirect = total - direct. +- Report the effects, with uncertainty, in the tool output and results. Make it impossible to mistake β for the marginal effect: the label and notes say β is not the marginal effect, and the effects table comes first. +- Measure effects coverage in the simulation. + +**A3. DGP set (spatial Durbin model approved as a DGP only).** +- Simulate: + - SEM (no substantive spillovers); + - SAR lag with known effects; + - SDM, y = ρWy + Xβ + WXθ + ε, with known effects; + - an omitted spatially correlated covariate. +- Fit the tool's models to each. Report parameter bias and coverage, and recovery of direct/indirect/total effects where they are defined. +- Separate the recovery of disturbance parameters from the recovery of substantive effects. +- **No production SDM estimator in this round.** + +**A4. Common-factor test (SEM as SDM with θ = -ρβ).** +- Evaluate a likelihood-ratio test only where both models can be fitted reliably. Confirm: + - nesting and the restriction; + - comparable likelihoods (same data, same Jacobian treatment); + - finite-sample size and power. +- Asymptotic Wald tests are known to be parameterization-dependent (Political Analysis 2020), so use LR, with simulation or bootstrap critical values if the LR size is off. +- Add the test to the tool only if it earns its place; no checklist additions. + +## 2. Workstream B: LISA under sampling error (P0) + +**B1. Five scenarios** (approved): no true clustering with +1. independent homogeneous errors; +2. heterogeneous independent errors; +3. spatially correlated errors; + +plus true clusters with +4. independent errors; +5. correlated errors. + +Correlation structures in scenarios 3 and 5 are parameterized, with ranges informed by C1 and stated as assumptions. + +**B2. Four candidate methods**, all on the same simulated maps: +1. conventional LISA (current default); +2. `replicate_quadrant_share` (a stability diagnostic, not a calibrated probability); +3. LISA on spatial Fay-Herriot EBLUPs. Explicitly test whether the smoothing **induces or alters** spatial patterns (false clusters created by smoothing; true clusters attenuated); +4. a Wong (2021) threshold-first, uncertainty-aware approach, implemented only as far as the open-access paper specifies it. + +**B3. Metrics, defined in the plan and the report:** +- **false-positive rate:** per-area share of null areas flagged; +- **FDR:** the expected share of flagged areas that are null; +- **false-cluster rate:** the share of maps with at least one flagged contiguous cluster, of size ≥ k, made only of null areas; k is stated; +- **power:** the share of true-cluster areas flagged; +- **cluster-location accuracy:** for example the Jaccard overlap of the flagged cluster with the true cluster; +- **sensitivity to W.** + +Every metric carries a Monte Carlo SE. + +Decompose errors attributable to sampling uncertainty from errors attributable to misspecified spatial dependence (for example by comparing the same maps with and without sampling error, and under the correct vs a misspecified W). + +A preferred method is never chosen by minimizing false positives alone: power and smoothing bias are weighed. + +**B4. Three questions kept separate:** +- stability of a classification across replicate maps; +- statistical significance; +- evidence of a genuine underlying pattern. + +## 3. Workstream C: ACS sampling-error covariance from replicates (P0, central question) + +**C1. Measure it, and measure how reliably it can be measured.** +- Use the ACS VRE replicates for Michigan and New York tracts (and counties), for several variables of different types (a count, a proportion, a median if replicated). +- Estimate the sampling covariance between estimates with the ACS replicate formula, Cov = (4/80) Σ_r (X_r - X)(Y_r - Y), consistent with the variance scaling. +- Analyse: + - neighbouring-pair correlations against non-neighbour pairs, by distance or contiguity order, and inside vs across county lines (weighting controls); + - structured summaries instead of unrestricted matrices, since 80 replicates give a rank-limited covariance across hundreds of tracts: mean or median neighbour correlation, a fitted structured model (for example one correlation parameter per contiguity order), shrinkage or regularization where justified; + - the stability of those summaries: bootstrap or jackknife over replicates, and split-half agreement; + - the noise floor: what pairwise correlation magnitude 80 replicates can distinguish from zero. +- Pairwise correlations are never treated as established individually. + +**C2. Report** whether the covariance structure is estimable well enough to use in a correction (A1) or a null model (B). If so, that is a methodological contribution to the engine; if not, the limit is documented with evidence. Results feed the B1 correlation parameters as stated assumptions. + +## 4. Workstream D: literature notes (P1) + +`docs/literature/spatial_methods.md` lists the references the owner's planner provided. Each entry records: +- full citation and DOI/URL; +- **verification level:** + - (1) verified from full text; + - (2) supported by abstract or accessible excerpt only; + - (3) a hypothesis or implementation suggestion still to verify; +- what it is used for, and which experiment tests it. + +The planner's summaries are not substitutes for the sources. Where access is restricted, document it, and don't implement methods whose essential assumptions cannot be verified. + +References: +- Lacombe & LeSage 2015 +- Koley & Bera 2024 +- Political Analysis 2020 (common-factor Wald test) +- IJGIS 2018 (ACS spatial autocorrelation critique) +- Wong 2021 +- SBC 2019 +- Bayesian areal ACS 2021 +- Sun et al. 2015 +- Arthur 2024 +- LeSage & Pace 2014 +- Lorincz-Comi et al. 2021 +- Rogerson 2024 + +## 5. Scope + +- No unrelated architectural changes; preserve the existing safeguards, reproducibility contracts and working functionality. +- Studies are `info`. Gating is an owner decision after the evidence is in. +- Out of scope this round: + - hierarchical Bayesian measurement-error models (a later research extension); + - Arthur-style constrained resampling and spatial FDR procedures (Sun et al.), unless B shows the basic null-model problem is solved first; + - a production SDM estimator. +- Remote laptop access follows this round. + +## 6. Release + +v0.15.0 with a Zenodo DOI, after code review, fixes, the full gate, and a test run on both Python 3.10 and 3.14 before the push. + +--- + +## 7. Implementation status (2026-11-09) + +| Workstream | Result | +| :--- | :--- | +| A spatial regression | **λ attenuation is real and correctable:** spatial Fay-Herriot recovers λ (coverage 0.86-0.93, uncorrected 0.003-0.85) under independent errors, and fails under positively correlated errors. **Lag effects** were added to the tool. **SDM data** break default GMM lag (documented). **LR common-factor test:** size fine for n ≥ 100, power weak, so not added. **Koley & Bera tests** don't identify SEM vs SDM | +| B LISA | No default change. "38%" corrected. Fay-Herriot smoothing induces clusters only under positively correlated errors. Stability ≠ evidence. Wong 2021 targets a different estimand | +| C covariance | Rates/shares/medians: neighbour error r ≈ 0 (±0.003 tracts). Controlled totals are correlated within county, with a release-dependent sign (2020-2024 negative; 2015-2019 households positive). The single 2019 Michigan population file is rejected by the census.gov firewall (URL-specific, not rate limiting) | +| D literature | 12 sources read in full (owner-supplied PDFs for the blocked ones); verification levels recorded | +| Owner decisions | Include `correct_outcome_error` in v0.15.0 (approved 2026-11-09) | +| Review | `/code-review high`: 10 findings fixed. The shipped λ estimator is now the one the studies run, with coverage unchanged within 0.007 | + +**Proposals for a later round:** +- the spatial Durbin model via spreg `slx_lags`, plus the common-factor test (LR/bootstrap); +- simulated percentile intervals for lag effects at high ρ; +- adaptive (Storey) BH and cluster-level FDR for LISA, scored on the existing runs; +- a corrected global Moran's I (Lorincz-Comi Eq. 4) using the replicate covariance, after its inference is verified; +- posterior exceedance probabilities (Sun et al. 2015); +- the LISA weights guidance (P-B5). + +**Next:** minimum SSH remote access (docs/remote_access_decisions.md). diff --git a/README.md b/README.md index fe97949..9111b2b 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# US Census ACS 5-Year Data Batch Downloader +# Census Research Engine [![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.23925301.svg)](https://doi.org/10.5281/zenodo.23925301) @@ -22,8 +22,8 @@ All four interfaces share one pipeline and the same defaults. Tested on Python 3.10–3.12 (Linux and Windows CI). ```bash -git clone https://github.com/cafeTechne/US-Census-5-Year-ACS-Data-Batch-Downloader -cd US-Census-5-Year-ACS-Data-Batch-Downloader +git clone https://github.com/cafeTechne/Census-Research-Engine +cd Census-Research-Engine pip install -r requirements.txt ``` @@ -51,6 +51,7 @@ Alternatively, set `BLS_USER_AGENT` to an identifying User-Agent that includes a | `CENSUS_PRICE_PARITY_DIR` | Alternate Regional Price Parity directory (default `data/price_parities/`); same treatment | | `BLS_USER_AGENT` | Identifying User-Agent for `scripts/update_price_indexes.py` | | `CENSUS_VRE_DIR` | Cache for downloaded Census Variance Replicate Estimate files (default `.cache/vre/`) | +| `CENSUS_WWW2_MIN_INTERVAL` / `CENSUS_WWW2_ATTEMPTS` / `CENSUS_WWW2_BACKOFF_BASE` / `CENSUS_WWW2_REJECTED_RETRY_DELAY` | census.gov file downloads (`census/http.py`): seconds between requests to one host (default 1), attempts on HTTP 429/5xx or connection errors (4), backoff base in seconds (2; doubled per retry, capped at 60, honours `Retry-After`), and the wait before the single retry of a firewall "Request Rejected" page (30). A rejection that persists names the cache path where a browser-downloaded copy of the file is picked up | | `CENSUS_LIVE_TESTS=1` | Enable live API contract tests and the live validation battery | ## Quick start @@ -238,7 +239,7 @@ Built on GeoPandas, Shapely (GEOS), pyogrio (GDAL), pyproj, and PySAL. Outputs a | Census boundaries matched to each ACS release | `census.boundaries.get_boundaries` / `get_boundaries` | Cartographic (1:500k) for maps, TIGER/Line for areal work; vintage mismatches are errors | | Journal maps that show uncertainty | `census.maps.choropleth` / `make_map` | Quantile classes by default; legend reports class separability (Sun and Wong 2010); low-reliability areas hatched; uncertainty and significance maps | | Spatial autocorrelation | `census.spatial.morans_i`, `local_morans` / `spatial_autocorrelation` | Reproduces PySAL references; replicate SE of Moran's I is conservative (coverage 0.987); local clusters use two-sided permutation p-values (9999 permutations, BH): false-positive rates at nominal under randomness | -| Spatial regression | `census.spatial.spatial_regression` | Lag (GMM default) and error (ML default) models with LM diagnostics (spreg); simulated coverage under ACS outcome error: rho 0.89-0.90 (GMM lag), lambda attenuated | +| Spatial regression | `census.spatial.spatial_regression` / `spatial_regression` | Lag (GMM default) and error (ML default) models with LM diagnostics (spreg); simulated coverage under ACS outcome error: rho 0.89-0.90 (GMM lag), lambda attenuated (0.003-0.85). Error model option `correct_outcome_error=True, se_col=...` corrects lambda for the outcome's sampling error (spatial Fay-Herriot likelihood, REML): lambda coverage 0.86-0.93. It assumes independent errors across areas, which ACS replicates support for rates, shares and medians ([covariance audit](docs/audits/acs_sampling_covariance.md)); it warns for population/household totals and is biased by positively correlated errors ([validation](docs/audits/spatial_regression_validation.md)) | | Spatial Fay–Herriot | `census.sae_spatial.spatial_fay_herriot` / `spatial_small_area_estimates` | Reproduces the R `sae` grapes example; coverage 0.891 | | Population-weighted tract crosswalks | `census.block_weights`, `harmonize_panel(weight="auto")` | Decennial block weights; Delaware test: mean error 444 vs 624 people with land-area weights | | Block groups and custom polygons | `geo_level="block group"`, `census.custom_areas` / `custom_area_mapping` | Area or block-population allocation | diff --git a/census/__init__.py b/census/__init__.py index 59ef3df..760bf59 100644 --- a/census/__init__.py +++ b/census/__init__.py @@ -1,7 +1,7 @@ # SPDX-License-Identifier: AGPL-3.0-or-later """Headless Census ACS data client and processing library.""" -__version__ = "0.14.0" +__version__ = "0.15.0" from census.api import ( DATASETS, diff --git a/census/block_weights.py b/census/block_weights.py index b9c4ed5..18d7eaf 100644 --- a/census/block_weights.py +++ b/census/block_weights.py @@ -71,6 +71,7 @@ import io import json import os +import warnings import zipfile from typing import Any, Dict, Iterable, List, Optional, Sequence @@ -78,6 +79,7 @@ import pandas as pd from .api import BASE_URL, DEFAULT_API_KEY +from .http import adopt_manual_file, payload_problem, polite_get, quarantine from .replicate import USER_AGENT, _read_cached, _sha256, _sidecar_path, _write_cached from .geo import get_resource_path @@ -148,29 +150,51 @@ def _state(state: Any) -> str: return code -def _get(url: str, params: Optional[Dict[str, str]] = None, session: Any = None, timeout: float = 300.0) -> bytes: - import requests - - getter = session if session is not None else requests - try: - resp = getter.get(url, params=params, headers={"User-Agent": USER_AGENT}, timeout=timeout) - except requests.RequestException as exc: - raise BlockDataError(f"download failed for {url}: {exc}") from exc +def _get(url: str, params: Optional[Dict[str, str]] = None, session: Any = None, timeout: float = 300.0, + cache_path: Optional[str] = None) -> bytes: + resp = polite_get(url, user_agent=USER_AGENT, error=BlockDataError, session=session, timeout=timeout, + params=params, cache_path=cache_path) if resp.status_code != 200: raise BlockDataError(f"HTTP {resp.status_code} for {url}") - raw = resp.content or b"" - if b"Request Rejected" in raw[:2000]: - raise BlockDataError(f"census.gov rejected the request for {url}; retry later") - return raw + return resp.content or b"" + +def _cached_or_fetch(path: str, url: str, fetch: Any, content_type: str, adopt: bool = False, + zip_member: Optional[str] = None) -> bytes: + """Cached bytes or ``fetch()``; with ``adopt``, a file placed at ``path`` by hand is used (raw downloads). -def _cached_or_fetch(path: str, url: str, fetch: Any, content_type: str) -> bytes: + Raw downloads (``adopt``) are checked with census.http.payload_problem (a + zip with a ``zip_member`` member, never an HTML page): a manually placed + file that fails is moved aside to ``.invalid`` (no sidecar) and + BlockDataError names it; a cached file that fails is moved aside and + fetched again; a fetched payload that fails raises before anything is + cached. + """ + if adopt: + def check(p: str) -> None: + problem = payload_problem(p, zip_member) + if problem: + raise BlockDataError(f"Manually placed file {p} is not usable: {problem}.") + + adopt_manual_file(path, url, _sidecar_path(path), validate=check) raw = _read_cached(path) + if raw is not None and adopt: + problem = payload_problem(raw, zip_member) + if problem: + dest = quarantine(path, _sidecar_path(path)) + warnings.warn(f"cached file {path} is not usable ({problem}); moved aside to {dest} and fetching it " + "again", UserWarning, stacklevel=2) + raw = None if raw is not None: return raw if _offline(): raise BlockDataError(f"{os.path.basename(path)} not cached and CENSUS_BLOCK_OFFLINE is set") raw = fetch() + if adopt: + problem = payload_problem(raw, zip_member) + if problem: + raise BlockDataError(f"{url} returned content that is not the data file: {problem}. Nothing was " + f"cached; download it in a web browser and save it at {path}") _write_cached(path, raw, url, content_type) return raw @@ -210,7 +234,8 @@ def load_block_relationships(state: Any, session: Any = None) -> pd.DataFrame: st = _state(state) url = REL_URL.format(state=st) path = os.path.join(block_cache_dir(), os.path.basename(url)) - raw = _cached_or_fetch(path, url, lambda: _get(url, session=session), "application/zip") + raw = _cached_or_fetch(path, url, lambda: _get(url, session=session, cache_path=path), "application/zip", + adopt=True, zip_member=".txt") try: with zipfile.ZipFile(io.BytesIO(raw)) as zf: name = next(n for n in zf.namelist() if n.lower().endswith(".txt")) @@ -538,15 +563,10 @@ def _write_assignment_cache(path: str, df: pd.DataFrame, url: str) -> None: def _fetch_zcta2020(session: Any = None) -> None: """Stream the national 2020 ZCTA-to-block file once; cache one compact file per state.""" - import requests - if _offline(): raise BlockDataError("2020 ZCTA block file not cached and CENSUS_BLOCK_OFFLINE is set") - getter = session if session is not None else requests - try: - resp = getter.get(ZCTA_2020_URL, headers={"User-Agent": USER_AGENT}, timeout=600, stream=True) - except requests.RequestException as exc: - raise BlockDataError(f"download failed for {ZCTA_2020_URL}: {exc}") from exc + resp = polite_get(ZCTA_2020_URL, user_agent=USER_AGENT, error=BlockDataError, session=session, timeout=600, + stream=True) if resp.status_code != 200: raise BlockDataError(f"HTTP {resp.status_code} for {ZCTA_2020_URL}") lines = (ln.decode("utf-8-sig") if isinstance(ln, bytes) else ln for ln in resp.iter_lines()) diff --git a/census/boundaries.py b/census/boundaries.py index 686fd94..d0f0ba5 100644 --- a/census/boundaries.py +++ b/census/boundaries.py @@ -57,9 +57,10 @@ import pandas as pd from .geo import get_resource_path +from .http import adopt_manual_file, polite_get from .geo_crosswalk import tract_vintage -USER_AGENT = "acs-batch-downloader/boundaries (research toolkit; python-requests)" +USER_AGENT = "census-research-engine/boundaries (research toolkit; python-requests)" BASE_URL = "https://www2.census.gov/geo/tiger" SOURCE_CRS = "EPSG:4269" DETAILS = ("cb", "tiger") @@ -352,9 +353,21 @@ def _sidecar(path: str) -> str: return path + ".json" -def _read_cached(path: str) -> Optional[Dict[str, Any]]: - """Sidecar metadata if ``path`` exists and matches its SHA-256.""" +def _check_zip(path: str) -> None: + with open(path, "rb") as f: + if f.read(2) != b"PK": + raise BoundaryError(f"Manually placed file {path} is not a zip archive.") + + +def _read_cached(path: str, url: Optional[str] = None) -> Optional[Dict[str, Any]]: + """Sidecar metadata if ``path`` exists and matches its SHA-256. + + With ``url``, a zip placed at ``path`` by hand (no sidecar yet) is adopted + with ``source: "manual download"`` (census.http.adopt_manual_file). + """ side = _sidecar(path) + if url is not None: + adopt_manual_file(path, url, side, validate=_check_zip) if not (os.path.exists(path) and os.path.exists(side)): return None try: @@ -381,17 +394,11 @@ def write_cache_entry(path: str, raw: bytes, url: str, content_type: str = "appl return meta -def _download(url: str, session: Any = None, timeout: float = 300.0) -> Tuple[bytes, str]: - import requests - - getter = session if session is not None else requests - try: - resp = getter.get(url, headers={"User-Agent": USER_AGENT}, timeout=timeout) - except requests.RequestException as exc: # pragma: no cover - network failure - raise BoundaryError(f"Download failed for {url}: {exc}") from exc +def _download(url: str, session: Any = None, timeout: float = 300.0, + cache_path: Optional[str] = None) -> Tuple[bytes, str]: + resp = polite_get(url, user_agent=USER_AGENT, error=BoundaryError, session=session, timeout=timeout, + cache_path=cache_path) raw = resp.content or b"" - if b"Request Rejected" in raw[:2000]: - raise BoundaryError(f"census.gov rejected the request for {url}; wait and retry later") if resp.status_code == 404: raise BoundaryError(f"Not published (HTTP 404): {url}") if resp.status_code != 200: @@ -438,9 +445,9 @@ def get_boundaries(level: str, year: int, state: Optional[Union[str, int]] = Non file_state = st if LEVELS[lvl]["per_state"] else None url = boundary_url(lvl, year, file_state, detail) path = os.path.join(cache_dir or geo_cache_dir(), str(year), url.rsplit("/", 1)[-1]) - meta = None if refresh else _read_cached(path) + meta = None if refresh else _read_cached(path, url) if meta is None: - raw, ctype = _download(url, session=session) + raw, ctype = _download(url, session=session, cache_path=path) meta = write_cache_entry(path, raw, url, ctype or "application/zip") gdf = gpd.read_file(path) gdf = _normalize_geoid_column(gdf) @@ -448,7 +455,7 @@ def get_boundaries(level: str, year: int, state: Optional[Union[str, int]] = Non gdf = gdf[gdf["GEOID"].str.startswith(st)] gdf = gdf.sort_values("GEOID", kind="mergesort").reset_index(drop=True) gdf.attrs.update(level=lvl, year=year, detail=detail, state=st, source_url=url, - sha256=meta.get("sha256"), cache_path=path, + sha256=meta.get("sha256"), cache_path=path, acquisition=meta.get("source", "download"), description=boundary_description(lvl, year, detail)) return gdf diff --git a/census/errata.py b/census/errata.py index cd19e17..e373420 100644 --- a/census/errata.py +++ b/census/errata.py @@ -276,14 +276,16 @@ def check_urls( import requests + from census.http import polite_get + session = requests.Session() - session.headers["User-Agent"] = "acs-batch-downloader/errata-check (research toolkit)" def get(url: str) -> int: # type: ignore[no-redef] time.sleep(delay) try: - return session.get(url, timeout=30, allow_redirects=True).status_code - except requests.RequestException: + return polite_get(url, user_agent="census-research-engine/errata-check (research toolkit)", + session=session, timeout=30).status_code + except RuntimeError: # connection failure after retries, or the firewall page return 0 failures = [] diff --git a/census/http.py b/census/http.py new file mode 100644 index 0000000..e015d9d --- /dev/null +++ b/census/http.py @@ -0,0 +1,334 @@ +# SPDX-License-Identifier: AGPL-3.0-or-later +"""Polite file downloads from the census.gov file servers (www2.census.gov, www.census.gov). + +Every module that downloads a file from census.gov (VRE tables, boundary +files, PUMS, block relationship files, validation sources, errata pages) +goes through :func:`polite_get`, which + +* identifies the client honestly with the caller's own User-Agent string + (``census-research-engine/ ...``); it never poses as a browser; +* keeps a process-wide minimum interval between requests to the same host + (``CENSUS_WWW2_MIN_INTERVAL`` seconds, default 1); +* retries HTTP 429/500/502/503/504 and connection errors or timeouts with + exponential backoff and jitter (``CENSUS_WWW2_ATTEMPTS`` attempts, default + 4; delay ``CENSUS_WWW2_BACKOFF_BASE`` x 2^k seconds, default base 2, capped + at 60, times a random factor in [0.5, 1]), honouring a ``Retry-After`` + header (capped at 300 s); +* recognises the census.gov firewall page ("Request Rejected ... Your support + ID is: N", served with HTTP 200), retries it at most once after + ``CENSUS_WWW2_REJECTED_RETRY_DELAY`` seconds (default 30), and then raises + the caller's error class with the support ID, the URL and, when the caller + passes it, the exact cache path where a manually downloaded copy is picked up. + +The firewall page is not (only) a rate limit: on 2026-11-09 www2 rejected one +specific file (``replicate_estimates/2019/data/5-year/140/B01003_26.csv.zip``) +on every request, whatever the pacing or client, while every other file in +the same directory downloaded normally. Retrying in a loop does not help, so +the helper never loops; the message tells the user to fetch the URL in a +browser and place it at the cache path, where :func:`adopt_manual_file` +records it (SHA-256, ``source: "manual download"``) on first read. +""" + +import email.utils +import hashlib +import json +import os +import random +import re +import threading +import time +from typing import Any, Callable, Dict, Mapping, Optional, Type +from urllib.parse import urlsplit + +RETRY_STATUSES = frozenset({429, 500, 502, 503, 504}) +DEFAULT_ATTEMPTS = 4 +DEFAULT_BACKOFF_BASE = 2.0 +BACKOFF_CAP = 60.0 +RETRY_AFTER_CAP = 300.0 +DEFAULT_MIN_INTERVAL = 1.0 +DEFAULT_REJECTED_RETRY_DELAY = 30.0 +MANUAL_SOURCE = "manual download" + +ENV_MIN_INTERVAL = "CENSUS_WWW2_MIN_INTERVAL" +ENV_ATTEMPTS = "CENSUS_WWW2_ATTEMPTS" +ENV_BACKOFF_BASE = "CENSUS_WWW2_BACKOFF_BASE" +ENV_REJECTED_RETRY_DELAY = "CENSUS_WWW2_REJECTED_RETRY_DELAY" + +_REJECTED_MARKER = b"Request Rejected" +_SUPPORT_ID = re.compile(rb"support\s+ID\s+is:?\s*(?:<[^>]*>\s*)*([0-9A-Za-z-]+)", re.IGNORECASE) + +# Indirections so tests can drive time and jitter without sleeping. +_sleep: Callable[[float], None] = time.sleep +_monotonic: Callable[[], float] = time.monotonic +_jitter: Callable[[], float] = lambda: random.uniform(0.5, 1.0) # noqa: E731 + +_pace_lock = threading.Lock() +_last_request: Dict[str, float] = {} + + +def _env_float(name: str, default: float) -> float: + try: + return max(0.0, float(os.environ[name])) + except (KeyError, ValueError): + return default + + +def min_interval() -> float: + return _env_float(ENV_MIN_INTERVAL, DEFAULT_MIN_INTERVAL) + + +def attempts() -> int: + return max(1, int(_env_float(ENV_ATTEMPTS, DEFAULT_ATTEMPTS))) + + +def backoff_delay(attempt: int) -> float: + """Delay before retry number ``attempt`` (0-based): base x 2^attempt, capped, times jitter.""" + base = _env_float(ENV_BACKOFF_BASE, DEFAULT_BACKOFF_BASE) + return min(BACKOFF_CAP, base * 2**attempt) * _jitter() + + +def retry_after_seconds(value: Any) -> Optional[float]: + """Seconds from a ``Retry-After`` header (delta-seconds or HTTP date), capped; None if absent/invalid.""" + if value in (None, ""): + return None + text = str(value).strip() + try: + secs = float(text) + except ValueError: + try: + when = email.utils.parsedate_to_datetime(text) + except (TypeError, ValueError): + return None + if when is None: + return None + secs = when.timestamp() - time.time() + return min(RETRY_AFTER_CAP, max(0.0, secs)) + + +def rejection_support_id(body: bytes) -> Optional[str]: + """None if ``body`` is not the census.gov firewall page; else its support ID ('' if none shown).""" + head = body[:4000] + if _REJECTED_MARKER not in head: + return None + m = _SUPPORT_ID.search(head) + return m.group(1).decode("ascii", "replace") if m else "" + + +def _pace(host: str) -> None: + """Wait for this request's slot on ``host``: at least ``min_interval()`` after the previous slot. + + The slot is reserved under the lock and the wait happens outside it, so a + thread waiting for one host never blocks requests to another host, and + concurrent requests to the same host get successive slots. + """ + gap = min_interval() + with _pace_lock: + now = _monotonic() + last = _last_request.get(host) + slot = now if last is None or gap <= 0 else max(now, last + gap) + _last_request[host] = slot + if slot > now: + _sleep(slot - now) + + +def _close(resp: Any) -> None: + """Release a response that will not be returned (a streamed body would otherwise hold its connection).""" + close = getattr(resp, "close", None) + if callable(close): + try: + close() + except Exception: # noqa: BLE001 - best effort; the retry proceeds either way + pass + + +def _raise(error: Type[Exception], message: str, **attrs: Any) -> None: + exc = error(message) + for k, v in attrs.items(): + setattr(exc, k, v) + raise exc + + +def rejected_message(url: str, support_id: str, cache_path: Optional[str], retried_after: Optional[float]) -> str: + sid = f"support ID {support_id}" if support_id else "no support ID shown" + again = (f", also on one retry {retried_after:g} s later" if retried_after is not None else "") + where = (f"save it at exactly this path:\n {cache_path}\nThe toolkit then uses that file and records its " + f"SHA-256 with source '{MANUAL_SOURCE}'." + if cache_path else "use the downloaded file instead of the URL.") + return (f"census.gov rejected the request for {url} with its firewall 'Request Rejected' page ({sid}){again}. " + "This rejection can be persistent and specific to one URL (other files on the same server may " + "download normally), so retrying will not help and the toolkit does not retry further. " + f"Download the file in a web browser from {url} and {where}") + + +def polite_get( + url: str, + *, + user_agent: str, + error: Type[Exception] = RuntimeError, + session: Any = None, + timeout: float = 300.0, + params: Optional[Mapping[str, Any]] = None, + headers: Optional[Mapping[str, str]] = None, + stream: bool = False, + cache_path: Optional[str] = None, +) -> Any: + """GET ``url`` politely (see module docstring) and return the response. + + Non-retryable statuses (and a retryable one after the last attempt) are + returned for the caller to judge, so callers keep their own 404/HTTP + messages. Raises ``error`` after the last connection failure, and on a + firewall rejection that persists through its single retry; the exception + carries ``url``, ``support_id``, ``cache_path`` and ``rejected`` attributes. + ``session`` is anything with a requests-style ``get``; default ``requests``. + """ + import requests # local: keeps importers usable without network deps + + getter = session if session is not None else requests + kwargs: Dict[str, Any] = {"headers": {**(headers or {}), "User-Agent": user_agent}, "timeout": timeout} + if params is not None: + kwargs["params"] = params + if stream: + kwargs["stream"] = True + host = urlsplit(url).netloc.lower() + n = attempts() + attempt = 0 + rejected_retry: Optional[float] = None + while True: + _pace(host) + try: + resp = getter.get(url, **kwargs) + except (requests.ConnectionError, requests.Timeout) as exc: + if attempt + 1 >= n: + _raise(error, f"Download failed for {url} after {n} attempt(s): {exc}", + url=url, support_id=None, cache_path=cache_path, rejected=False) + _sleep(backoff_delay(attempt)) + attempt += 1 + continue + except requests.RequestException as exc: + _raise(error, f"Download failed for {url}: {exc}", url=url, support_id=None, + cache_path=cache_path, rejected=False) + status = getattr(resp, "status_code", 200) + if status in RETRY_STATUSES and attempt + 1 < n: + hdrs = getattr(resp, "headers", None) or {} + wait = retry_after_seconds(hdrs.get("Retry-After")) + _close(resp) + _sleep(wait if wait is not None else backoff_delay(attempt)) + attempt += 1 + continue + if not stream or "html" in str((getattr(resp, "headers", None) or {}).get("Content-Type", "")).lower(): + sid = rejection_support_id(resp.content or b"") + if sid is not None: + _close(resp) + if rejected_retry is None: + rejected_retry = _env_float(ENV_REJECTED_RETRY_DELAY, DEFAULT_REJECTED_RETRY_DELAY) + _sleep(rejected_retry) + continue + _raise(error, rejected_message(url, sid, cache_path, rejected_retry), url=url, support_id=sid, + cache_path=cache_path, rejected=True) + return resp + + +def _sha256_file(path: str) -> str: + h = hashlib.sha256() + with open(path, "rb") as f: + for chunk in iter(lambda: f.read(1 << 20), b""): + h.update(chunk) + return h.hexdigest() + + +def payload_problem(source: Any, zip_member_suffix: Optional[str] = None) -> Optional[str]: + """Why ``source`` (bytes or a file path) is not the file a census.gov URL serves, or None. + + With ``zip_member_suffix`` (e.g. ``".csv"``) it must be a zip archive with + a member whose name ends with that suffix; otherwise (text files) it must + not be an HTML/XML page: the firewall's 'Request Rejected' page, an error + page or a web page saved from a browser. + """ + import io + import zipfile + + if isinstance(source, (bytes, bytearray)): + head = bytes(source[:4000]) + else: + with open(source, "rb") as f: + head = f.read(4000) + if not head: + return "the file is empty" + if rejection_support_id(head) is not None: + return "it is the census.gov firewall 'Request Rejected' page, not the data file" + text = head.lstrip(b"\xef\xbb\xbf \t\r\n").lower() + if text.startswith(b"<") or b" str: + """Move an invalid cached file aside to ``.invalid`` and drop its sidecar; return the new path. + + The next run then downloads (or adopts) the file afresh instead of reusing + bad content forever. + """ + dest = path + ".invalid" + os.replace(path, dest) + side = sidecar or path + ".json" + if os.path.exists(side): + os.remove(side) + return dest + + +def adopt_manual_file( + path: str, + url: str, + sidecar: Optional[str] = None, + validate: Optional[Callable[[str], None]] = None, +) -> Optional[Dict[str, Any]]: + """Record a file the user placed at a cache ``path`` by hand; return its new sidecar metadata. + + Returns None when ``path`` does not exist or already has a sidecar (the + caller's normal cache check applies). ``validate(path)`` may raise to + refuse a file that is not what the URL serves (for example a saved HTML + page); the sidecar is then not written, the file is moved aside to + ``.invalid`` (:func:`quarantine`, so it does not block later runs) + and the validator's exception is re-raised with both paths named. The + sidecar holds the source URL, + SHA-256, size and ``source: "manual download"``, so provenance records + say the file was not fetched by the toolkit. + """ + side = sidecar or path + ".json" + if not os.path.isfile(path) or os.path.exists(side): + return None + if validate is not None: + try: + validate(path) + except Exception as exc: + dest = quarantine(path, side) + note = (f" The file was moved aside to {dest} (no sidecar written). Download {url} again in a web " + f"browser and save it at {path}.") + try: + new = type(exc)(f"{exc}{note}") + except Exception: # noqa: BLE001 - exception class with another signature: keep the original + raise exc + for k, v in getattr(exc, "__dict__", {}).items(): + setattr(new, k, v) + raise new from exc + meta = { + "source_url": url, + "sha256": _sha256_file(path), + "size_bytes": os.path.getsize(path), + "content_type": "", + "source": MANUAL_SOURCE, + "adopted_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), + } + with open(side, "w", encoding="utf-8") as f: + json.dump(meta, f, indent=2) + return meta diff --git a/census/lisa_validation.py b/census/lisa_validation.py new file mode 100644 index 0000000..acf5a16 --- /dev/null +++ b/census/lisa_validation.py @@ -0,0 +1,568 @@ +# SPDX-License-Identifier: AGPL-3.0-or-later +"""LISA under ACS sampling error: five scenarios, four methods (PLAN_ROUND_16 workstream B). + +Validation code only. The candidate methods below are prototypes for this +study; none of them changes the behavior of :func:`census.spatial.local_morans` +or any other tool. Results and conclusions: ``docs/audits/lisa_validation.md``. + +Design (12 x 12 lattice, rook contiguity, 144 areas) +---------------------------------------------------- +True values theta_i ~ N(LV_MU, LV_TAU^2) independently, so the truth has no +spatial clustering, except in the cluster scenarios, where a LV_BLOCK x +LV_BLOCK block in the middle of the map is raised by LV_SHIFT (a hot spot). +Published estimates y = theta + e. Scenarios (PLAN_ROUND_16 B1): + +1. ``independent_homogeneous``: e_i ~ N(0, s^2) independent, s = LV_SE for + every area (median tract CV x LV_MU); +2. ``heterogeneous_independent``: e_i ~ N(0, s_i^2) independent, s_i the + tract-profile CVs (x LV_MU) sorted along the lattice diagonal: variances + vary smoothly across space, errors do not covary. This is the scenario of + the Round 15 finding (study 'local_morans_size', "38% of maps"); +3. ``correlated``: e ~ N(0, S R S), S = s I (homogeneous variances, so the + scenario isolates covariance from heterogeneity), R a correlation + matrix with mean first-order (rook) neighbour correlation r1; +4. ``cluster_independent``: hot spot, errors as in 1; +5. ``cluster_correlated``: hot spot, errors as in 3. + +Error correlation (assumption, not an ACS measurement): R is the SAR(1) +covariance (I - lam W)^-1 (I - lam W')^-1 (W the row-standardized rook +matrix) rescaled to unit diagonal, with lam solved so that the mean rook +neighbour correlation equals r1 in :data:`LV_NEIGHBOUR_CORRS` (-0.2 .. 0.5; +r1 = 0 is scenario 1/4). Negative r1 (a possible effect of weighting +controls that hold area totals) gives a checkerboard structure: second-order +neighbours are positively correlated. The grid spans the range in which +workstream C's replicate-based neighbour correlations can be located. + +All r1 values of one repetition share theta and the standard-normal draw z +(e = S L z, L the Cholesky factor of R), so differences between r1 values are +paired (common random numbers). + +Replicates: 80 replicate deviations per map, d_r = S L z_r / 2 (z_r a fresh +standard-normal vector per replicate), then rescaled per area so that +4/80 sum_r d_ir^2 = s_i^2 exactly (as census.simulation.make_replicates); +the replicate maps are y + d_r. Their cross-area covariance is the +scenario's error covariance (up to the per-area rescaling). + +Methods (B2), each applied to the same maps +------------------------------------------- +* ``lisa`` - census.spatial.local_morans with its defaults (two-sided + p-values, 9999 permutations, BH, alpha 0.10). Flag: significant. Cluster + label: HH/LL. +* ``lisa_stable`` - the same call with replicates: an area is flagged when + significant AND replicate_quadrant_share >= LV_STABLE (stated threshold). + replicate_quadrant_share is a stability diagnostic, not a calibrated + probability; this reading measures what filtering by it does, and + ``stable_only`` (HH/LL with share >= LV_STABLE, no significance) shows + what stability alone would flag. +* ``lisa_eblup`` - local_morans (defaults) on spatial Fay-Herriot EBLUPs + (census.sae_spatial, REML, intercept only, the analysis W, D = s_i^2; the + model assumes independent sampling errors). Reported with the smoothing + diagnostics: estimated rho, global Moran's I of the EBLUPs vs the true + values, the hot-spot contrast relative to the truth, and the paired change + in the false-cluster indicator relative to ``lisa`` on the direct + estimates. +* ``wong_cl`` - Wong (2021, IJERPH 18:9848) heuristic hot spot / cold + spot detection, as far as the paper specifies it + (:func:`wong_hot_cold_spots`): seeds are areas whose estimate lies beyond + the analyst's thresholds (here LV_MU +/- LV_WONG_Z x LV_TAU, a stated + domain threshold on the true-value scale); a first-order neighbour of a + member joins the seed's spot when its estimate is not significantly + different from the seed's estimate, compared with the seed (not with the + member), repeated until no neighbour joins. The paper's worked example + fixes the operational rule: a neighbour joins when the two-sided p-value + of the difference is >= CL (Louisiana/Mississippi p = 0.65 joins at + CL 0.648; Alabama/Mississippi p = 0.13 at CL 0.128). The test statistic is + the independent-errors z = (y_s - y_j) / sqrt(s_s^2 + s_j^2) (the paper + says "t-test" without a formula; this is the ACS comparison formula). + Wong's method targets a different estimand from LISA: latent elevation + beyond a threshold, not local similarity beyond chance. Under "no + clustering" it still reports spots wherever extreme values occur, so it + is not scored with LISA's false-positive metrics but against its own + target (:func:`wong_metrics`): ``exceed_recall`` (areas whose true value is + beyond a threshold that are flagged with that label), ``seed_precision`` + (seeds whose true value is beyond the threshold), ``member_near_share`` + (grown, non-seed members whose true value is within LV_WONG_NEAR of their + seed's), ``member_beyond_share``, ``growth_per_seed`` (grown members per + seed: as a function of the SEs and of the error correlation, which the + independence-based test ignores) and, with a hot spot, ``power`` and + ``jaccard`` for the hot label against the block. + +Three questions are kept apart (PLAN_ROUND_16 B4): latent elevation (Wong), +local similarity beyond chance (LISA's significance), stability across +replicates (replicate_quadrant_share). In a simulation the truth is known, +so evidence of a genuine pattern is scored against it. + +Metrics (B3), per map, then averaged over maps with the between-map MC SE +-------------------------------------------------------------------------- +Null areas: every area in scenarios 1-3; areas outside the block in 4-5. +* ``fpr``: flagged null areas / null areas; +* ``fdr``: mean false discovery proportion, flagged null / flagged (0 when + nothing is flagged), i.e. the FDR that BH controls; +* ``false_cluster_map_k2``, ``false_cluster_map_k4`` (the false-cluster map + rate; not Sun et al.'s (2015) false cluster rate, which weights + predefined clusters): 1 if the map has at least one + rook-connected component of areas flagged with the same cluster label + (HH or LL; hot or cold) of size >= k made only of null areas; +* ``power``: block areas flagged with the correct label (HH; hot) / block + areas; +* ``jaccard``: |F n T| / |F u T|, F the areas flagged with the correct + label, T the block; +* ``halo_share``: share of the false discoveries that are rook neighbours of + the block (cluster scenarios; "halo" flags next to a real hot spot). +Sensitivity to W: the ``lisa`` method is also run with queen contiguity, +k-nearest neighbours (k = LV_KNN) and a misspecified W (rook contiguity of a +fixed random permutation of the areas: unrelated to the map) on selected +maps. Decomposition: the same truth analysed without sampling error +(``no_sampling_error``), with independent error, with heterogeneous +variances, with correlated errors at each r1, and with the misspecified W. + +Studies (info rows only, run on demand: ``--studies lisa_validation_null +lisa_validation_clusters``): one row per (scenario, error structure, method, +weights, metric) with ``coverage`` = the metric's mean, ``mc_se`` its MC SE. +""" + +import math +from dataclasses import replace +from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple + +import numpy as np +import pandas as pd +from scipy import linalg as sla, optimize, stats as sps + +from .simulation import (CV_PROFILES, FAST, FULL, STUDIES, SimConfig, _geoids, _lisa_gradient_rank, acs_frame, + draw_cv, lattice_weights) +from .variance import REPLICATE_COUNT, REPLICATE_FACTOR + +LV_SIDE = 12 +LV_MU = 50.0 # mean of the true values +LV_TAU = 10.0 # SD of the independent true values +LV_SE = CV_PROFILES["tract"]["median"] / 100 * LV_MU # homogeneous sampling SE (6.7; reliability ~0.69) +LV_BLOCK = 4 # hot spot: LV_BLOCK x LV_BLOCK block ... +LV_SHIFT = 2.5 * LV_TAU # ... raised by 2.5 true-value SDs +LV_NEIGHBOUR_CORRS = (-0.2, -0.1, 0.0, 0.1, 0.25, 0.5) # mean rook-neighbour error correlation r1 +LV_ALPHA = 0.10 # local_morans default alpha +LV_STABLE = 0.8 # replicate_quadrant_share stability threshold +LV_WONG_Z = 1.645 # Wong thresholds: LV_MU +/- 1.645 LV_TAU (true 5% tails) +LV_WONG_CLS = (0.5, 0.75) # Wong confidence levels (the paper's simulated examples) +LV_KNN = 6 +LV_CLUSTER_K = (2, 4) +LV_W_SENSITIVITY_CORRS = (0.0, 0.25) # r1 values also analysed under queen / kNN / misspecified W + # (as are the no-error and heterogeneous maps) +LISA_REPS_FULL = {"lisa_validation_null": 400, "lisa_validation_clusters": 400} +LISA_REPS_FAST = {"lisa_validation_null": 3, "lisa_validation_clusters": 3} +LV_WONG_NEAR = 0.5 * LV_TAU # a grown member is "near" its seed if |theta_j - theta_s| <= this +METRICS = ("fpr", "fdr", "false_cluster_map_k2", "false_cluster_map_k4", "power", "jaccard", "halo_share") + + +# --------------------------------------------------------------------------- +# Error covariance and replicates +# --------------------------------------------------------------------------- + + +def rook_adjacency(side: int) -> np.ndarray: + """Binary rook adjacency of a side x side lattice (row-major cells).""" + m = side * side + A = np.zeros((m, m)) + for r in range(side): + for c in range(side): + for rr, cc in ((r + 1, c), (r, c + 1)): + if rr < side and cc < side: + A[r * side + c, rr * side + cc] = A[rr * side + cc, r * side + c] = 1.0 + return A + + +def sar_correlation(W: np.ndarray, lam: float) -> np.ndarray: + """Correlation matrix of the SAR(1) process (I - lam W)^-1 eps, eps ~ N(0, I).""" + # scipy (LAPACK) rather than numpy.linalg.inv: see census.sae_spatial._State + B = sla.inv(np.eye(W.shape[0]) - lam * W) + S = B @ B.T + d = np.sqrt(np.diag(S)) + return S / np.outer(d, d) + + +def mean_neighbour_correlation(R: np.ndarray, A: np.ndarray, order: int = 1) -> float: + """Mean correlation over pairs at contiguity order 1 (neighbours) or 2 (neighbours of neighbours).""" + if order == 1: + mask = A > 0 + else: + mask = ((A @ A) > 0) & (A == 0) & ~np.eye(A.shape[0], dtype=bool) + return float(R[mask].mean()) + + +def error_correlation(W: np.ndarray, A: np.ndarray, r1: float) -> Dict[str, Any]: + """SAR error correlation with mean rook-neighbour correlation r1 (r1 = 0: identity). + + Returns lam, the correlation matrix R, its Cholesky factor L and the + achieved first- and second-order mean correlations. + """ + m = W.shape[0] + if r1 == 0: + R = np.eye(m) + lam = 0.0 + else: + lam = optimize.brentq(lambda t: mean_neighbour_correlation(sar_correlation(W, t), A) - r1, + -0.999 if r1 < 0 else 1e-9, -1e-9 if r1 < 0 else 0.999, xtol=1e-10) + R = sar_correlation(W, lam) + return {"lam": float(lam), "R": R, "L": sla.cholesky(R, lower=True), "r1": mean_neighbour_correlation(R, A, 1), + "r2": mean_neighbour_correlation(R, A, 2)} + + +def correlated_replicates(se: np.ndarray, L: np.ndarray, Z: np.ndarray) -> np.ndarray: + """(n, 80) replicate deviations d_r = S L z_r / 2, rescaled so 4/80 sum_r d_r^2 = se^2 per area. + + ``Z``: (n, 80) standard normals. The cross-area correlation of the + deviations is that of L L' (the scenario's error correlation). + """ + dev = (L @ Z) * se[:, None] / 2.0 + scale = np.sqrt(REPLICATE_FACTOR * (dev ** 2).sum(axis=1, keepdims=True)) + return np.where(scale > 0, dev / np.where(scale > 0, scale, 1.0), 0.0) * se[:, None] + + +# --------------------------------------------------------------------------- +# Wong (2021) heuristic hot spot / cold spot detection +# --------------------------------------------------------------------------- + + +def wong_hot_cold_spots(y: np.ndarray, se: np.ndarray, neighbours: Sequence[Sequence[int]], high: float, + low: float, confidence: float) -> Tuple[np.ndarray, np.ndarray, np.ndarray]: + """Wong (2021) heuristic hot spots / cold spots: (labels, seed mask, seed_of). + + labels: 'hot', 'cold', 'both' (claimed by a hot and a cold seed; the + paper does not say how spots from different seeds meet) or ''. + seed_of: for each area, the first seed (in area order) whose spot it + joined (a seed is its own), -1 for areas in no spot. + + Seeds: y > high (hot) or y < low (cold). Growing (paper sec. 4.2): a + first-order neighbour of a member of a seed's spot joins it when its + estimate is not significantly different from the SEED's estimate; the + comparison is repeated outward until every candidate differs. The + operational rule reproduces the paper's worked example (sec. 5.1): join + when the two-sided p-value of z = (y_s - y_j) / sqrt(se_s^2 + se_j^2) is + >= ``confidence`` (so lowering the confidence level grows the spots, as + the paper describes). Independent errors are assumed by the test. + """ + y, se = np.asarray(y, float), np.asarray(se, float) + n = len(y) + hot, cold = np.zeros(n, bool), np.zeros(n, bool) + seed_of = np.full(n, -1) + seeds_hot, seeds_cold = y > high, y < low + for s in np.flatnonzero(seeds_hot | seeds_cold): + member, seen, frontier = {int(s)}, {int(s)}, [int(s)] + while frontier: + nxt = [] + for i in frontier: + for j in neighbours[i]: + j = int(j) + if j in seen: + continue + seen.add(j) + sd = math.sqrt(se[s] ** 2 + se[j] ** 2) + p = 2 * sps.norm.sf(abs(y[s] - y[j]) / sd) if sd > 0 else float(y[s] == y[j]) + if p >= confidence: + member.add(j) + nxt.append(j) + frontier = nxt + mem = np.array(sorted(member)) + (hot if seeds_hot[s] else cold)[mem] = True + seed_of[mem[seed_of[mem] < 0]] = s + seeds = seeds_hot | seeds_cold + seed_of[seeds] = np.flatnonzero(seeds) + labels = np.where(hot & cold, "both", np.where(hot, "hot", np.where(cold, "cold", ""))) + return labels, seeds, seed_of + + +# --------------------------------------------------------------------------- +# Metrics +# --------------------------------------------------------------------------- + + +def components(labels: np.ndarray, neighbours: Sequence[Sequence[int]], keep: Sequence[str]) -> List[np.ndarray]: + """Connected components (via ``neighbours``) of areas sharing a label in ``keep``.""" + labels = np.asarray(labels) + seen = np.zeros(len(labels), bool) + out = [] + for s in range(len(labels)): + if seen[s] or labels[s] not in keep: + continue + comp, stack = [], [s] + seen[s] = True + while stack: + i = stack.pop() + comp.append(i) + for j in neighbours[i]: + if not seen[j] and labels[j] == labels[s]: + seen[j] = True + stack.append(j) + out.append(np.array(sorted(comp))) + return out + + +def map_metrics(flag: np.ndarray, labels: np.ndarray, truth: np.ndarray, correct: str, cluster_labels: Sequence[str], + neighbours: Sequence[Sequence[int]], halo: Optional[np.ndarray] = None) -> Dict[str, float]: + """B3 metrics for one map (see the module docstring); power/jaccard are NaN without true clusters.""" + flag, truth = np.asarray(flag, bool), np.asarray(truth, bool) + null = ~truth + false = flag & null + out = {"fpr": float(false.sum() / null.sum()), "fdr": float(false.sum() / max(int(flag.sum()), 1))} + comps = components(np.where(flag, labels, ""), neighbours, cluster_labels) + for k in LV_CLUSTER_K: + out[f"false_cluster_map_k{k}"] = float(any(len(c) >= k and not truth[c].any() for c in comps)) + if truth.any(): + hit = flag & (np.asarray(labels) == correct) + out["power"] = float((hit & truth).sum() / truth.sum()) + out["jaccard"] = float((hit & truth).sum() / (hit | truth).sum()) + out["halo_share"] = (float((false & halo).sum() / false.sum()) if false.any() and halo is not None + else float("nan")) + else: + out["power"] = out["jaccard"] = out["halo_share"] = float("nan") + return out + + +def wong_metrics(labels: np.ndarray, seeds: np.ndarray, seed_of: np.ndarray, theta: np.ndarray, high: float, + low: float, truth: np.ndarray) -> Dict[str, float]: + """Wong (2021) against its own target, latent elevation beyond the thresholds (see the module docstring). + + LISA's false-positive metrics are not computed: under "no clustering" + Wong's spots are areas beyond the thresholds, which is its estimand. + """ + nan = float("nan") + labels = np.asarray(labels) + true_hot, true_cold = theta > high, theta < low + hit = ((labels == "hot") & true_hot) | ((labels == "cold") & true_cold) + beyond = true_hot | true_cold + seed_ok = np.where(labels == "hot", true_hot, true_cold) + grown = (labels != "") & ~seeds + near = np.abs(theta - theta[np.maximum(seed_of, 0)]) <= LV_WONG_NEAR + out = {"exceed_recall": float(hit[beyond].mean()) if beyond.any() else nan, + "seed_precision": float(seed_ok[seeds].mean()) if seeds.any() else nan, + "member_near_share": float(near[grown].mean()) if grown.any() else nan, + "member_beyond_share": float(seed_ok[grown].mean()) if grown.any() else nan, + "growth_per_seed": float(grown.sum() / seeds.sum()) if seeds.any() else nan, + "flagged_share": float((labels != "").mean())} + if truth.any(): + hot_flag = labels == "hot" + out["power"] = float((hot_flag & truth).sum() / truth.sum()) + out["jaccard"] = float((hot_flag & truth).sum() / (hot_flag | truth).sum()) + return out + + +def _moran(v: np.ndarray, W: np.ndarray) -> float: + z = v - v.mean() + return float(len(v) / W.sum() * (z @ W @ z) / (z @ z)) + + +# --------------------------------------------------------------------------- +# Weights for the sensitivity analysis +# --------------------------------------------------------------------------- + + +def _lattice_coords(side: int) -> np.ndarray: + r, c = np.divmod(np.arange(side * side), side) + return np.column_stack([c, r]).astype(float) + + +def analysis_weights(side: int, ids: Sequence[str], seed: int) -> Dict[str, Any]: + """libpysal W for 'rook', 'queen', f'knn{LV_KNN}' and 'misspecified' (rook of a fixed random permutation).""" + import libpysal + + ids = list(ids) + out = {"rook": lattice_weights(side, ids)} + nb: Dict[str, List[str]] = {} + for r in range(side): + for c in range(side): + nb[ids[r * side + c]] = [ids[rr * side + cc] for rr in (r - 1, r, r + 1) for cc in (c - 1, c, c + 1) + if (rr, cc) != (r, c) and 0 <= rr < side and 0 <= cc < side] + out["queen"] = libpysal.weights.W(nb, id_order=ids, silence_warnings=True) + out[f"knn{LV_KNN}"] = libpysal.weights.KNN(_lattice_coords(side), k=LV_KNN, ids=ids) + # area i is placed at lattice cell pos[i]; its neighbours are the areas at the rook cells around pos[i] + pos = np.random.default_rng(seed).permutation(len(ids)) + at = np.argsort(pos) # at[cell] = area placed there + idx = {gid: k for k, gid in enumerate(ids)} + rook = out["rook"] + nb_m = {ids[i]: [ids[at[idx[q]]] for q in rook.neighbors[ids[pos[i]]]] for i in range(len(ids))} + out["misspecified"] = libpysal.weights.W(nb_m, id_order=ids, silence_warnings=True) + for w in out.values(): + w.transform = "r" + return out + + +# --------------------------------------------------------------------------- +# Studies +# --------------------------------------------------------------------------- + + +class _Acc: + """Per-map metric values keyed by (cell, method, weights).""" + + def __init__(self): + self.vals: Dict[Tuple[str, str, str], Dict[str, List[float]]] = {} + + def add(self, cell: str, method: str, wname: str, metrics: Dict[str, float]) -> None: + d = self.vals.setdefault((cell, method, wname), {}) + for k, v in metrics.items(): + d.setdefault(k, []).append(float(v)) + + +def _summary_rows(acc: _Acc, cells: Dict[str, Dict[str, Any]], side: int) -> List[Dict[str, Any]]: + rows = [] + for (cell, method, wname), d in acc.vals.items(): + info = cells[cell] + for metric, vals in d.items(): + v = np.asarray(vals, float) + v = v[np.isfinite(v)] + if not v.size: + continue + rows.append({ + "method": f"{method}, {wname} W, {side}x{side} lattice, {info['label']}", + "target": f"{metric} (coverage column = mean of the per-map {metric})", + "reps": int(v.size), "nominal": float("nan"), "coverage": float(v.mean()), + "mc_se": float(v.std(ddof=1) / math.sqrt(v.size)) if v.size > 1 else float("nan"), + "status": "info", "scenario": info["scenario"], "errors": info["errors"], + "neighbour_corr": info.get("r1", float("nan")), "sar_lambda": info.get("lam", float("nan")), + "second_order_corr": info.get("r2", float("nan")), "lv_method": method, "weights": wname, + "metric": metric, "value": float(v.mean()), + }) + return rows + + +def _lisa_flags(out: pd.DataFrame) -> Tuple[np.ndarray, np.ndarray]: + return out["significant"].to_numpy(bool), out["quadrant"].to_numpy(str) + + +def _run(cfg: SimConfig, reps: int, clusters: bool, offset: int, side: int = LV_SIDE) -> List[Dict[str, Any]]: + from .sae_spatial import _fit + from .spatial import local_morans + from .simulation import replicate_long + + m = side * side + g = _geoids(m) + ws = analysis_weights(side, g, cfg.seed + 1600) + rook = ws["rook"] + W = rook.full()[0] + A = rook_adjacency(side) + nbrs = [np.flatnonzero(A[i]) for i in range(m)] + r_, c_ = np.divmod(np.arange(m), side) + lo = (side - LV_BLOCK) // 2 + block = (r_ >= lo) & (r_ < lo + LV_BLOCK) & (c_ >= lo) & (c_ < lo + LV_BLOCK) + truth = block if clusters else np.zeros(m, bool) + halo = (A[:, block].sum(axis=1) > 0) & ~block + structures = {r1: error_correlation(W, A, r1) for r1 in LV_NEIGHBOUR_CORRS} + grad = _lisa_gradient_rank(side) + hi, lo_t = LV_MU + LV_WONG_Z * LV_TAU, LV_MU - LV_WONG_Z * LV_TAU + X1 = np.ones((m, 1)) + scen_ind, scen_cor = (("cluster_independent", "cluster_correlated") if clusters + else ("independent_homogeneous", "correlated")) + cells: Dict[str, Dict[str, Any]] = { + "truth": {"scenario": "no_sampling_error", "errors": "none", + "label": ("hot spot" if clusters else "no clustering") + ", no sampling error (true values)"}} + for r1, st in structures.items(): + key = f"r1={r1:g}" + cells[key] = {"scenario": scen_ind if r1 == 0 else scen_cor, + "errors": "independent" if r1 == 0 else "correlated", + "r1": st["r1"], "r2": st["r2"], "lam": st["lam"], + "label": (("hot spot, " if clusters else "no clustering, ") + + (f"independent homogeneous errors (SE {LV_SE:g})" if r1 == 0 else + f"correlated errors, SAR lambda {st['lam']:.3f}, " + f"mean neighbour corr {st['r1']:.2f}"))} + if not clusters: + cells["heterogeneous"] = {"scenario": "heterogeneous_independent", "errors": "independent", + "label": "no clustering, heterogeneous independent errors (tract CVs sorted along " + "the diagonal)"} + acc = _Acc() + rng = np.random.default_rng(cfg.seed + offset) + for _ in range(reps): + theta = LV_MU + LV_TAU * rng.standard_normal(m) + (LV_SHIFT * block if clusters else 0.0) + z = rng.standard_normal(m) + Zr = rng.standard_normal((m, REPLICATE_COUNT)) + se_het = np.sort(draw_cv(m, "tract", rng))[grad] / 100 * LV_MU + base_seed = int(rng.integers(2 ** 31 - 1000)) + maps: List[Tuple[str, np.ndarray, Optional[np.ndarray], Optional[np.ndarray]]] = [("truth", theta, None, None)] + se_h = np.full(m, LV_SE) + for r1, st in structures.items(): + maps.append((f"r1={r1:g}", theta + se_h * (st["L"] @ z), se_h, st["L"])) + if not clusters: + maps.append(("heterogeneous", theta + se_het * z, se_het, np.eye(m))) + contrast_true = theta[block].mean() - theta[~block].mean() if clusters else float("nan") + for k, (cell, y, se, L) in enumerate(maps): + seed = base_seed + k + frame = acs_frame(g, {"y": y}, {"y": se if se is not None else np.zeros(m)}) + reps_frame = None + if se is not None: + dev = correlated_replicates(se, L, Zr) + reps_frame = replicate_long(g, ["y"], y[:, None], (y[:, None] + dev)[:, None, :]) + out = local_morans(frame, "y", rook, seed=seed, replicates=reps_frame) + sig, quad = _lisa_flags(out) + mm = map_metrics(sig, quad, truth, "HH", ("HH", "LL"), nbrs, halo) + mm["moran_I"] = _moran(y, W) + acc.add(cell, "lisa", "rook", mm) + r1 = float(cell.split("=")[1]) if cell.startswith("r1=") else None + if cell in ("truth", "heterogeneous") or r1 in LV_W_SENSITIVITY_CORRS: + for wname, w in ws.items(): + if wname == "rook": + continue + o = local_morans(frame, "y", w, seed=seed) + s2, q2 = _lisa_flags(o) + acc.add(cell, "lisa", wname, map_metrics(s2, q2, truth, "HH", ("HH", "LL"), nbrs, halo)) + if se is None: + continue + # replicate_quadrant_share as a filter and alone (diagnostic readings) + share = out["replicate_quadrant_share"].to_numpy(float) + stable = share >= LV_STABLE + st_m = map_metrics(sig & stable, quad, truth, "HH", ("HH", "LL"), nbrs, halo) + st_m["stable_share_of_significant"] = float(stable[sig].mean()) if sig.any() else float("nan") + st_m["share_mean_sig_null"] = float(share[sig & ~truth].mean()) if (sig & ~truth).any() else float("nan") + st_m["share_mean_sig_true"] = float(share[sig & truth].mean()) if (sig & truth).any() else float("nan") + acc.add(cell, "lisa_stable", "rook", st_m) + hhll = np.isin(quad, ("HH", "LL")) + acc.add(cell, "stable_only", "rook", + map_metrics(stable & hhll, quad, truth, "HH", ("HH", "LL"), nbrs, halo)) + # spatial Fay-Herriot EBLUPs, then LISA + fit = _fit(y, X1, se ** 2, W, "REML") + eb = fit["eblup"] + oe = local_morans(acs_frame(g, {"y": eb}, {"y": se}), "y", rook, seed=seed) + se_sig, se_quad = _lisa_flags(oe) + em = map_metrics(se_sig, se_quad, truth, "HH", ("HH", "LL"), nbrs, halo) + em["rho_hat"] = fit["rho"] + em["moran_I"] = _moran(eb, W) + em["moran_I_truth"] = _moran(theta, W) + for kk in LV_CLUSTER_K: + em[f"induced_false_cluster_map_k{kk}"] = (em[f"false_cluster_map_k{kk}"] + - mm[f"false_cluster_map_k{kk}"]) + if clusters: + em["contrast_ratio"] = float((eb[block].mean() - eb[~block].mean()) / contrast_true) + mm_c = float((y[block].mean() - y[~block].mean()) / contrast_true) + acc.add(cell, "lisa", "rook", {"contrast_ratio": mm_c}) + acc.add(cell, "lisa_eblup", "rook", em) + # Wong (2021), scored against its own target (latent elevation beyond the thresholds) + for cl in LV_WONG_CLS: + lab, seeds, seed_of = wong_hot_cold_spots(y, se, nbrs, hi, lo_t, cl) + acc.add(cell, f"wong_cl{cl:g}", "rook", wong_metrics(lab, seeds, seed_of, theta, hi, lo_t, truth)) + return _summary_rows(acc, cells, side) + + +def study_lisa_validation_null(cfg: SimConfig, reps: int, side: int = LV_SIDE) -> List[Dict[str, Any]]: + """Scenarios 1-3 (no true clustering): independent homogeneous, heterogeneous independent, correlated errors. + + ``side`` < LV_SIDE only for quick tests (the error-correlation grid is solved for that lattice). + """ + return _run(cfg, reps, clusters=False, offset=1600, side=side) + + +def study_lisa_validation_clusters(cfg: SimConfig, reps: int, side: int = LV_SIDE) -> List[Dict[str, Any]]: + """Scenarios 4-5 (a 4x4 hot spot): independent and correlated errors (``side`` as in the null study).""" + return _run(cfg, reps, clusters=True, offset=1601, side=side) + + +LISA_STUDIES: Dict[str, Callable[[SimConfig, int], List[Dict[str, Any]]]] = { + "lisa_validation_null": study_lisa_validation_null, + "lisa_validation_clusters": study_lisa_validation_clusters, +} +STUDIES.update(LISA_STUDIES) + + +def lisa_config(fast: bool = True, **overrides: Any) -> SimConfig: + """Config that runs only the Round 16 LISA validation studies.""" + base = FAST if fast else FULL + reps = {k: 0 for k in base.reps} + reps.update(LISA_REPS_FAST if fast else LISA_REPS_FULL) + return replace(base, reps=reps, **overrides) diff --git a/census/pipeline.py b/census/pipeline.py index 92e7040..a01f4ea 100644 --- a/census/pipeline.py +++ b/census/pipeline.py @@ -103,7 +103,7 @@ from census.stats import add_reliability_columns, clean_census_sentinels, parse_variable_code from census.tables import load_variable_metadata -PACKAGE_VERSION = "0.14.0" +PACKAGE_VERSION = "0.15.0" CONCORDANCE_MODES = ("fail", "strict", "code") MAX_ISSUES_IN_RESULT = 100 diff --git a/census/pums.py b/census/pums.py index 5480776..18132cc 100644 --- a/census/pums.py +++ b/census/pums.py @@ -58,11 +58,12 @@ from .api import PRODUCT_1YR, PRODUCT_5YR, normalize_product from .geo import STATE_FILES, STATE_MAPPING, get_resource_path +from .http import adopt_manual_file, payload_problem, polite_get, quarantine from .results import (SOURCE_MODEL, SOURCE_SAMPLING_REPLICATE, ModelResult, check_refit_count, check_replicate_policy, replicate_completeness) from .variance import REPLICATE_COUNT, REPLICATE_FACTOR, Z90 -USER_AGENT = "acs-batch-downloader/pums (research toolkit; python-requests)" +USER_AGENT = "census-research-engine/pums (research toolkit; python-requests)" PUMS_DATA_BASE = "https://www2.census.gov/programs-surveys/acs/data/pums" PUMS_DOCS_BASE = "https://www2.census.gov/programs-surveys/acs/tech_docs/pums" PUMS_API_TEMPLATE = "https://api.census.gov/data/{year}/acs/{dataset}/pums" @@ -195,13 +196,34 @@ def _write_sidecar(path: str, meta: Dict[str, Any]) -> None: json.dump(meta, f, indent=2) -def read_cache_entry(path: str) -> Optional[Dict[str, Any]]: +def _expected_member(url: str) -> Optional[str]: + """``.csv`` for the zipped PUMS files (state CSV zips); None for plain text files (dictionaries, CSV/TXT).""" + return ".csv" if url.lower().endswith(".zip") else None + + +def _check_manual(url: str) -> Callable[[str], None]: + def check(path: str) -> None: + problem = payload_problem(path, _expected_member(url)) + if problem: + raise PumsError(f"Manually placed file {path} is not usable: {problem}.") + return check + + +def read_cache_entry(path: str, url: Optional[str] = None) -> Optional[Dict[str, Any]]: """Sidecar metadata if ``path`` exists and matches its recorded SHA-256. + With ``url``, a file placed at ``path`` by hand (no sidecar yet) is + adopted with ``source: "manual download"`` (census.http.adopt_manual_file) + after checking it is what the URL serves (a zip with a CSV member for + ``.zip`` URLs, not an HTML page); a file that fails is moved aside to + ``.invalid`` and PumsError names both paths (no sidecar written). + State PUMS zips run to hundreds of MB, so the full SHA-256 is recomputed only when the file's size or mtime differs from the values recorded in the sidecar (``size_bytes``, ``mtime_ns``); a verified hash refreshes them. """ + if url is not None: + adopt_manual_file(path, url, _sidecar(path), validate=_check_manual(url)) if not (os.path.exists(path) and os.path.exists(_sidecar(path))): return None try: @@ -236,17 +258,11 @@ def write_cache_entry(path: str, raw: bytes, url: str, content_type: str = "") - return meta -def _download(url: str, session: Any = None, timeout: float = 600.0) -> Tuple[bytes, str]: - import requests - - getter = session if session is not None else requests - try: - resp = getter.get(url, headers={"User-Agent": USER_AGENT}, timeout=timeout) - except requests.RequestException as exc: # pragma: no cover - network failure - raise PumsError(f"Download failed for {url}: {exc}") from exc +def _download(url: str, session: Any = None, timeout: float = 600.0, + cache_path: Optional[str] = None) -> Tuple[bytes, str]: + resp = polite_get(url, user_agent=USER_AGENT, error=PumsError, session=session, timeout=timeout, + cache_path=cache_path) raw = resp.content or b"" - if b"Request Rejected" in raw[:2000]: - raise PumsError(f"census.gov rejected the request for {url}; wait and retry later") if resp.status_code == 404: raise PumsError(f"Not published (HTTP 404): {url}") if resp.status_code != 200: @@ -256,11 +272,29 @@ def _download(url: str, session: Any = None, timeout: float = 600.0) -> Tuple[by def cached_download(url: str, subdir: str = "", *, cache_dir: Optional[str] = None, refresh: bool = False, session: Any = None) -> Tuple[str, Dict[str, Any]]: - """Path and sidecar metadata of ``url`` in the PUMS cache, downloading if needed.""" + """Path and sidecar metadata of ``url`` in the PUMS cache, downloading if needed. + + Downloads are checked before they are cached (a zip with a CSV member for + ``.zip`` URLs; never an HTML page such as the census.gov firewall page); + a failing download raises PumsError and nothing is written. A cached file + whose content fails the same check (e.g. cached before the check existed) + is moved aside to ``.invalid`` with a warning and downloaded again. + """ path = os.path.join(cache_dir or pums_cache_dir(), subdir, url.rsplit("/", 1)[-1]) - meta = None if refresh else read_cache_entry(path) + meta = None if refresh else read_cache_entry(path, url) + if meta is not None: + problem = payload_problem(path, _expected_member(url)) + if problem: + dest = quarantine(path, _sidecar(path)) + warnings.warn(f"cached PUMS file {path} is not usable ({problem}); moved aside to {dest} and " + "downloading it again", UserWarning, stacklevel=2) + meta = None if meta is None: - raw, ctype = _download(url, session=session) + raw, ctype = _download(url, session=session, cache_path=path) + problem = payload_problem(raw, _expected_member(url)) + if problem: + raise PumsError(f"{url} returned content that is not the data file: {problem}. Nothing was cached; " + f"download it in a web browser and save it at {path}") meta = write_cache_entry(path, raw, url, ctype) return path, meta diff --git a/census/replicate.py b/census/replicate.py index 75e89a7..3598c31 100644 --- a/census/replicate.py +++ b/census/replicate.py @@ -62,6 +62,7 @@ import pandas as pd from census.geo import get_resource_path +from census.http import adopt_manual_file, polite_get from census.variance import ( METHOD_REPLICATE, REPLICATE_COLUMNS, @@ -135,7 +136,7 @@ ).split() CONTROLLED_CME = "*****" -USER_AGENT = "acs-batch-downloader/replicate (research toolkit; python-requests)" +USER_AGENT = "census-research-engine/replicate (research toolkit; python-requests)" # Columns of a replicate frame, in order (PLAN_ROUND_7.md section 4, B3). REPLICATE_FRAME_COLUMNS: List[str] = ["GEOID", "NAME", "variable", "estimate", "moe"] + list( @@ -623,9 +624,27 @@ def _sidecar_path(path: str) -> str: return path + ".json" -def _read_cached(path: str) -> Optional[bytes]: - """Cached bytes if the file exists and matches its sidecar SHA-256.""" +def _read_cached( + path: str, url: Optional[str] = None, validate: Optional[Callable[[bytes], Any]] = None +) -> Optional[bytes]: + """Cached bytes if the file exists and matches its sidecar SHA-256. + + With ``url``, a file placed at ``path`` by hand (no sidecar yet, e.g. after + a firewall rejection) is adopted: ``validate(raw)`` must accept it, then a + sidecar with its SHA-256 and ``source: "manual download"`` is written. + """ sidecar = _sidecar_path(path) + if url is not None: + + def check(p: str) -> None: + if validate is not None: + with open(p, "rb") as f: + try: + validate(f.read()) + except Exception as exc: # any parse failure: not the file the URL serves + raise VREDownloadError(f"Manually placed file {p} is not usable: {exc}") from exc + + adopt_manual_file(path, url, sidecar, validate=check) if not (os.path.exists(path) and os.path.exists(sidecar)): return None with open(path, "rb") as f: @@ -656,22 +675,15 @@ def _write_cached(path: str, raw: bytes, url: str, content_type: str) -> Dict[st return meta -def _download(url: str, session: Any = None, timeout: float = 300.0) -> Tuple[bytes, str]: - import requests # local import keeps the pure functions importable without network deps - - getter = session if session is not None else requests - try: - resp = getter.get(url, headers={"User-Agent": USER_AGENT}, timeout=timeout) - except requests.RequestException as exc: # pragma: no cover - network failure - raise VREDownloadError(f"Download failed for {url}: {exc}") from exc +def _download( + url: str, session: Any = None, timeout: float = 300.0, cache_path: Optional[str] = None +) -> Tuple[bytes, str]: + # Retries, pacing and the census.gov firewall page ("Request Rejected", + # which can persist for one specific URL) are handled by census.http. + resp = polite_get(url, user_agent=USER_AGENT, error=VREDownloadError, session=session, timeout=timeout, + cache_path=cache_path) raw = resp.content or b"" content_type = str(resp.headers.get("Content-Type", "")) - if b"Request Rejected" in raw[:2000]: - # census.gov's firewall answers rapid scripted requests with this page. - raise VREDownloadError( - f"census.gov rejected the request for {url} (firewall 'Request Rejected' page). " - "Wait and retry later; do not retry in a tight loop." - ) if resp.status_code == 404: raise VREDownloadError(f"Not published (HTTP 404): {url}") if resp.status_code != 200: @@ -758,9 +770,9 @@ def vre_tables( if not url: return frozen path = os.path.join(cache_dir or vre_cache_dir(year), os.path.basename(url)) - raw = None if refresh else _read_cached(path) + raw = None if refresh else _read_cached(path, url, _parse_table_list_csv) if raw is None and refresh: - raw, ctype = _download(url, session=session, timeout=60) + raw, ctype = _download(url, session=session, timeout=60, cache_path=path) _write_cached(path, raw, url, ctype) if raw is None: return frozen @@ -939,12 +951,12 @@ def fetch_vre( raise ValueError(f"Summary level {sumlev} is not nested in states; omit state_code.") url = vre_url(table, year, sumlev, state) path = os.path.join(cache_dir or vre_cache_dir(year), sumlev, url.rsplit("/", 1)[-1]) - raw = None if refresh else _read_cached(path) + raw = None if refresh else _read_cached(path, url, _decompress) meta: Dict[str, Any] if raw is None: if session is None and client is not None: session = getattr(client, "session", None) - raw, ctype = _download(url, session=session) + raw, ctype = _download(url, session=session, cache_path=path) _decompress(raw) # validate before caching meta = _write_cached(path, raw, url, ctype) else: @@ -962,6 +974,7 @@ def fetch_vre( source_url=url, sha256=meta.get("sha256"), cache_path=path, + acquisition=meta.get("source", "download"), ) if check_moe: report = verify_published_moe(frame) diff --git a/census/replicate_covariance.py b/census/replicate_covariance.py new file mode 100644 index 0000000..7b1d784 --- /dev/null +++ b/census/replicate_covariance.py @@ -0,0 +1,681 @@ +# SPDX-License-Identifier: AGPL-3.0-or-later +"""Sampling covariance between areas' ACS estimates from the 80 VRE replicates. + +PLAN_ROUND_16.md workstream C. The successive-difference replication (SDR) +variance of one estimate, Var(X) = 4/80 sum_r (X_r - X)^2 (census.variance), +extends to the covariance of two estimates published in the same release: + + Cov(X, Y) = 4/80 * sum_{r=1..80} (X_r - X)(Y_r - Y) + +(ACS Variance Replicate Estimate Tables documentation, formula (1) applied to +a sum; census.replicate.VRE_DOCUMENTATION). The 4/80 factor cancels in a +correlation, which is the uncentered cosine of the two replicate-deviation +vectors d_i = X_{i,r} - X_i. + +What 80 replicates can and cannot say +------------------------------------- +* The replicate covariance matrix of n areas is 4/80 D D^T with D the (n, 80) + deviation matrix, so its rank is at most 80. For hundreds or thousands of + tracts it is a rank-limited, finite-sample estimate, not the covariance. +* A single pairwise correlation from 80 replicates is very noisy: if one + area's deviation vector is spherically symmetric in R^80 and independent of + the other, (1 + r) / 2 ~ Beta(79/2, 79/2), so SD(r) = 1/sqrt(80) = 0.112 + and |r| > 0.22 happens 5% of the time (:func:`null_correlation`). Pairwise + correlations are therefore never reported as individually established; only + structured summaries over many pairs are (:func:`summarize_pair_correlations`). +* The 80 ACS replicates are not 80 free dimensions: in 2020-2024 Michigan and + New York tract data every area's deviations nearly vanish along the same 3 + replicate-space directions (:func:`replicate_spectrum`; a property of the + replicate-factor design), so the empirical null SD of r between independent + areas is about 1/sqrt(77) = 0.114-0.116 and |r| > 0.22 happens ~6% of the + time (docs/audits/acs_sampling_covariance.md). +* Uncertainty of those summaries comes from (a) the 80 replicates, by a + bootstrap over replicate columns (an approximation: SDR replicate + deviations are close to, but not exactly, iid across replicates), and (b) + the areas, by a delete-one-group jackknife (e.g. counties) that drops every + pair touching the group. :func:`split_half_agreement` measures how well the + per-pair correlations reproduce between two disjoint halves of the + replicates (reliability); :func:`holdout_model_comparison` scores structured + and shrinkage covariance models on replicates held out of the fit. + +Everything here is pure numpy/scipy; deviations are passed in, so the module +does not depend on census.replicate or on the network. +""" + +import math +from typing import Any, Dict, Mapping, Optional, Tuple + +import numpy as np +import pandas as pd +from scipy import sparse +from scipy import stats as sps + +from census.variance import REPLICATE_COUNT, REPLICATE_FACTOR + +__all__ = [ + "replicate_deviations", + "replicate_covariance", + "replicate_correlation", + "pair_correlations", + "null_correlation", + "permutation_null", + "contiguity_orders", + "pair_orders", + "summarize_pair_correlations", + "split_half_agreement", + "aggregation_variance_ratio", + "ledoit_wolf_intensity", + "holdout_model_comparison", + "replicate_spectrum", + "rank_report", +] + + +# --------------------------------------------------------------------------- +# Deviations, covariance, correlation +# --------------------------------------------------------------------------- + + +def replicate_deviations(estimates: Any, replicates: Any) -> np.ndarray: + """(n, R) replicate deviations X_{i,r} - X_i. + + ``estimates`` has shape (n,), ``replicates`` (n, R). Non-finite values are + refused (drop such areas first). + """ + est = np.asarray(estimates, dtype=float).reshape(-1) + reps = np.asarray(replicates, dtype=float) + if reps.ndim != 2 or reps.shape[0] != est.size: + raise ValueError(f"replicates must have shape (n, R) with n = {est.size}, got {reps.shape}") + if not (np.isfinite(est).all() and np.isfinite(reps).all()): + raise ValueError("estimates and replicates must be finite; drop incomplete areas first") + return reps - est[:, None] + + +def _factor(n_replicates: int) -> float: + # 4/80 for the ACS; for a subset of R replicates the same per-replicate + # scaling (4/R) keeps the estimator unbiased under the SDR design. + return 4.0 / n_replicates + + +def replicate_covariance(estimates: Any, replicates: Any) -> np.ndarray: + """(n, n) replicate covariance 4/R * D D^T (R = 80 for ACS VRE files). + + Its diagonal is census.variance.replicate_variance. Rank <= R. + """ + d = replicate_deviations(estimates, replicates) + return _factor(d.shape[1]) * (d @ d.T) + + +def replicate_correlation(estimates: Any, replicates: Any) -> np.ndarray: + """(n, n) replicate correlation; rows/columns of zero-variance areas are NaN.""" + d = replicate_deviations(estimates, replicates) + norm = np.sqrt(np.sum(d * d, axis=1)) + with np.errstate(divide="ignore", invalid="ignore"): + z = d / norm[:, None] + c = z @ z.T + c[norm == 0, :] = np.nan + c[:, norm == 0] = np.nan + return np.clip(c, -1.0, 1.0) + + +def pair_correlations(deviations: np.ndarray, i: Any, j: Any, weights: Optional[np.ndarray] = None) -> np.ndarray: + """Replicate correlation of the pairs (i[k], j[k]), without the n x n matrix. + + ``weights`` (length R) reweights replicate columns (bootstrap counts). + Pairs with a zero-variance member give NaN. + """ + d = np.asarray(deviations, dtype=float) + i = np.asarray(i, dtype=np.int64) + j = np.asarray(j, dtype=np.int64) + w = np.ones(d.shape[1]) if weights is None else np.asarray(weights, dtype=float) + num = np.einsum("kr,kr,r->k", d[i], d[j], w) + ss = (d * d) @ w + with np.errstate(divide="ignore", invalid="ignore"): + return np.clip(num / np.sqrt(ss[i] * ss[j]), -1.0, 1.0) + + +# --------------------------------------------------------------------------- +# Noise floor +# --------------------------------------------------------------------------- + + +def null_correlation(n_replicates: int = REPLICATE_COUNT, alpha: float = 0.05) -> Dict[str, float]: + """Null distribution of one pairwise replicate correlation. + + If one deviation vector is spherically symmetric in R^R and independent of + the other, the (uncentered) correlation is a coordinate of a uniform point + on the unit sphere: (1 + r)/2 ~ Beta((R-1)/2, (R-1)/2), mean 0, SD + 1/sqrt(R). Returns the SD and the two-sided ``alpha`` critical |r|. Real + SDR deviations of small areas can have fewer effective dimensions, so + check this against :func:`permutation_null` and against pairs known to be + independent (e.g. areas in different states). + """ + a = (n_replicates - 1) / 2.0 + crit = 2.0 * float(sps.beta.ppf(1.0 - alpha / 2.0, a, a)) - 1.0 + return {"n_replicates": int(n_replicates), "sd": 1.0 / math.sqrt(n_replicates), "alpha": float(alpha), + "critical_abs_r": crit} + + +def permutation_null(deviations: np.ndarray, i: Any, j: Any, n_perm: int = 20, seed: int = 0, + alpha: float = 0.05) -> Dict[str, float]: + """Empirical null for pairwise correlations: permute replicate indices of j. + + Each draw applies one random derangement (a permutation without fixed + points, so no replicate stays paired with itself) of the R replicate + columns to the second member of every pair, which destroys any real + covariance but keeps each area's own deviation distribution. Returns mean, SD and the two-sided + ``alpha`` quantile of |r| over all pairs and draws. + """ + d = np.asarray(deviations, dtype=float) + i = np.asarray(i, dtype=np.int64) + j = np.asarray(j, dtype=np.int64) + rng = np.random.default_rng(seed) + out = [] + norm = np.sqrt(np.sum(d * d, axis=1)) + ok = (norm[i] > 0) & (norm[j] > 0) + i, j = i[ok], j[ok] + zi = d[i] / norm[i, None] + zj = d[j] / norm[j, None] + for _ in range(int(n_perm)): + p = rng.permutation(d.shape[1]) + while np.any(p == np.arange(p.size)): + p = rng.permutation(d.shape[1]) + out.append(np.sum(zi * zj[:, p], axis=1)) + r = np.concatenate(out) if out else np.array([]) + return {"n_pairs": int(i.size), "n_perm": int(n_perm), "mean": float(np.mean(r)) if r.size else math.nan, + "sd": float(np.std(r)) if r.size else math.nan, + "critical_abs_r": float(np.quantile(np.abs(r), 1.0 - alpha)) if r.size else math.nan} + + +# --------------------------------------------------------------------------- +# Pair construction +# --------------------------------------------------------------------------- + + +def _adjacency(neighbors: Any, n: Optional[int] = None) -> sparse.csr_matrix: + if sparse.issparse(neighbors): + a = sparse.csr_matrix(neighbors, dtype=bool) + elif hasattr(neighbors, "sparse") and hasattr(neighbors, "id_order"): # libpysal W + a = sparse.csr_matrix(neighbors.sparse, dtype=bool) + else: + a = sparse.csr_matrix(np.asarray(neighbors), dtype=bool) + if a.shape[0] != a.shape[1] or (n is not None and a.shape[0] != n): + raise ValueError(f"adjacency must be square n x n, got {a.shape}") + a = ((a + a.T) > 0).tolil() + a.setdiag(False) + a = a.tocsr() + a.eliminate_zeros() + return a + + +def contiguity_orders(neighbors: Any, max_order: int = 3) -> sparse.csr_matrix: + """Sparse matrix of graph distance (contiguity order) 1..max_order between areas. + + ``neighbors``: a libpysal W, a sparse or dense adjacency matrix. Entry + (i, j) is k when j is a k-th order neighbour of i (shortest path of k + contiguity steps); pairs further apart (or disconnected) are absent. + """ + a = _adjacency(neighbors) + n = a.shape[0] + a = a.astype(np.int32) + reach = sparse.identity(n, dtype=np.int32, format="csr") + order = sparse.csr_matrix((n, n), dtype=np.int16) + for k in range(1, int(max_order) + 1): + nxt = reach + reach @ a + nxt.data = (nxt.data > 0).astype(np.int32) + new = (nxt - reach).tocsr() + new.data = (new.data > 0).astype(np.int32) + new.eliminate_zeros() + order = order + new.astype(np.int16) * k + reach = nxt.tocsr() + order.eliminate_zeros() + return order.tocsr() + + +def pair_orders(order_matrix: sparse.spmatrix, i: Any, j: Any) -> np.ndarray: + """Contiguity order of each pair (0 = beyond the matrix's max order).""" + m = sparse.csr_matrix(order_matrix) + return np.asarray(m[np.asarray(i), np.asarray(j)]).reshape(-1).astype(int) + + +# --------------------------------------------------------------------------- +# Structured summaries and their reliability +# --------------------------------------------------------------------------- + + +def _bootstrap_weights(rng: np.random.Generator, n_replicates: int) -> np.ndarray: + return rng.multinomial(n_replicates, np.full(n_replicates, 1.0 / n_replicates)).astype(float) + + +def _group_jackknife(r: np.ndarray, gi: np.ndarray, gj: np.ndarray) -> float: + groups = np.unique(np.concatenate([gi, gj])) + if groups.size < 3: + return math.nan + means = [] + for g in groups: + keep = (gi != g) & (gj != g) + if keep.any(): + means.append(float(np.mean(r[keep]))) + if len(means) < 3: + return math.nan + m = np.asarray(means) + k = m.size + return float(math.sqrt((k - 1) / k * np.sum((m - m.mean()) ** 2))) + + +def summarize_pair_correlations( + deviations: np.ndarray, + i: Any, + j: Any, + labels: Any, + *, + groups: Optional[Any] = None, + n_boot: int = 200, + seed: int = 0, + alpha: float = 0.05, +) -> pd.DataFrame: + """Mean/median replicate correlation per pair class, with its uncertainty. + + ``labels[k]`` is the class of pair (i[k], j[k]) (e.g. contiguity order, + within/across county). ``groups`` (length n, e.g. county of each area) + enables a delete-one-group jackknife SE of the class mean. Columns: + + * ``mean``, ``median``, ``sd`` of the pairwise correlations; + * ``se_boot`` / ``se_boot_median``: bootstrap over the R replicate columns + (``n_boot`` draws; replicate-sampling uncertainty); + * ``se_jackknife``: delete-one-group jackknife (area heterogeneity plus + replicate noise), NaN without ``groups``; + * ``se`` = the larger of the two, ``ci_low``/``ci_high`` = mean -/+ z se; + * ``null_sd`` = 1/sqrt(R) and ``excess_sd`` = sqrt(max(0, sd^2 - null_sd^2)), + the spread of the pairs' correlations beyond pure replicate noise; + * ``share_beyond_critical``: share of pairs with |r| above the null + two-sided ``alpha`` critical value (``alpha`` expected under the null). + """ + d = np.asarray(deviations, dtype=float) + i = np.asarray(i, dtype=np.int64) + j = np.asarray(j, dtype=np.int64) + labels = np.asarray(labels) + if not (i.size == j.size == labels.size): + raise ValueError("i, j and labels must have the same length") + n_rep = d.shape[1] + null = null_correlation(n_rep, alpha) + z = float(sps.norm.ppf(1.0 - alpha / 2.0)) + g = None if groups is None else np.asarray(groups) + rng = np.random.default_rng(seed) + boot_w = [_bootstrap_weights(rng, n_rep) for _ in range(int(n_boot))] + rows = [] + for lab in pd.unique(labels): + sel = labels == lab + ii, jj = i[sel], j[sel] + r = pair_correlations(d, ii, jj) + ok = np.isfinite(r) + ii, jj, r = ii[ok], jj[ok], r[ok] + if not r.size: + continue + bm, bmed = [], [] + if boot_w: + prod = d[ii] * d[jj] + sq = d * d + for w in boot_w: + ss = sq @ w + with np.errstate(divide="ignore", invalid="ignore"): + rb = (prod @ w) / np.sqrt(ss[ii] * ss[jj]) + rb = rb[np.isfinite(rb)] + bm.append(float(np.mean(rb))) + bmed.append(float(np.median(rb))) + se_boot = float(np.std(bm, ddof=1)) if len(bm) > 1 else math.nan + se_boot_med = float(np.std(bmed, ddof=1)) if len(bmed) > 1 else math.nan + se_jk = _group_jackknife(r, g[ii], g[jj]) if g is not None else math.nan + se = float(np.nanmax([se_boot, se_jk])) if np.isfinite([se_boot, se_jk]).any() else math.nan + mean = float(np.mean(r)) + sd = float(np.std(r, ddof=1)) if r.size > 1 else math.nan + rows.append({ + "class": lab, "n_pairs": int(r.size), "n_areas": int(np.unique(np.concatenate([ii, jj])).size), + "mean": mean, "median": float(np.median(r)), "sd": sd, + "se_boot": se_boot, "se_boot_median": se_boot_med, "se_jackknife": se_jk, "se": se, + "ci_low": mean - z * se, "ci_high": mean + z * se, + "null_sd": null["sd"], + "excess_sd": float(math.sqrt(max(0.0, sd ** 2 - null["sd"] ** 2))) if np.isfinite(sd) else math.nan, + "share_beyond_critical": float(np.mean(np.abs(r) > null["critical_abs_r"])), + }) + return pd.DataFrame(rows) + + +def split_half_agreement( + deviations: np.ndarray, + i: Any, + j: Any, + labels: Optional[Any] = None, + *, + split: str = "random", + seed: int = 0, +) -> pd.DataFrame: + """Agreement of pairwise correlations between two disjoint halves of the replicates. + + ``split``: ``random`` (seeded), ``first_last`` (1-40 vs 41-80) or + ``odd_even``. Per class: the two halves' class means, and the Pearson + correlation across pairs of the half-A and half-B pairwise correlations + (``pair_reliability_half``) with its Spearman-Brown step-up to all R + replicates (``pair_reliability_full``). A reliability near 0 means the + individual pairwise correlations are mostly replicate noise even if the + class mean is well determined. + """ + d = np.asarray(deviations, dtype=float) + n_rep = d.shape[1] + if split == "random": + perm = np.random.default_rng(seed).permutation(n_rep) + a, b = np.sort(perm[: n_rep // 2]), np.sort(perm[n_rep // 2:]) + elif split == "first_last": + a, b = np.arange(n_rep // 2), np.arange(n_rep // 2, n_rep) + elif split == "odd_even": + a, b = np.arange(0, n_rep, 2), np.arange(1, n_rep, 2) + else: + raise ValueError("split must be 'random', 'first_last' or 'odd_even'") + i = np.asarray(i, dtype=np.int64) + j = np.asarray(j, dtype=np.int64) + labels = np.zeros(i.size, dtype=int) if labels is None else np.asarray(labels) + ra = pair_correlations(d[:, a], i, j) + rb = pair_correlations(d[:, b], i, j) + rows = [] + for lab in pd.unique(labels): + sel = (labels == lab) & np.isfinite(ra) & np.isfinite(rb) + x, y = ra[sel], rb[sel] + rel = float(np.corrcoef(x, y)[0, 1]) if x.size > 2 and x.std() > 0 and y.std() > 0 else math.nan + full = 2 * rel / (1 + rel) if np.isfinite(rel) and rel > -1 else math.nan + rows.append({"class": lab, "split": split, "n_pairs": int(x.size), + "mean_half_a": float(np.mean(x)) if x.size else math.nan, + "mean_half_b": float(np.mean(y)) if y.size else math.nan, + "pair_reliability_half": rel, "pair_reliability_full": full}) + return pd.DataFrame(rows) + + +def aggregation_variance_ratio(deviations: np.ndarray, groups: Any, *, n_boot: int = 200, + seed: int = 0) -> Tuple[pd.DataFrame, Dict[str, float]]: + """Var(sum of areas in a group) / sum of the areas' variances, per group. + + For counts that add up (tracts in a county) the ratio is + 1 + sum_{i != j} Cov_ij / sum_i Var_i: below 1 means the areas' sampling + errors are negatively correlated on balance (e.g. weighting controls that + fix the group total), above 1 positively. ``implied_mean_corr`` is + sum_{i != j} Cov_ij / sum_{i != j} SD_i SD_j. This summary pools all pairs + of a group in one number, so it is far less noisy than any pairwise + correlation. Returns the per-group frame and a pooled summary (ratio of + summed variances) with a replicate-bootstrap SE. + """ + d = np.asarray(deviations, dtype=float) + g = np.asarray(groups) + n_rep = d.shape[1] + f = _factor(n_rep) + codes, uniq = pd.factorize(g) + k = uniq.size + gsum = np.zeros((k, n_rep)) + np.add.at(gsum, codes, d) + var_i = f * np.sum(d * d, axis=1) + sd_i = np.sqrt(var_i) + var_sum = f * np.sum(gsum * gsum, axis=1) + sum_var = np.bincount(codes, weights=var_i, minlength=k) + sum_sd = np.bincount(codes, weights=sd_i, minlength=k) + sum_sd2 = np.bincount(codes, weights=sd_i ** 2, minlength=k) + cross_sd = sum_sd ** 2 - sum_sd2 + n_areas = np.bincount(codes, minlength=k) + with np.errstate(divide="ignore", invalid="ignore"): + frame = pd.DataFrame({ + "group": uniq, "n_areas": n_areas, "var_of_sum": var_sum, "sum_of_var": sum_var, + "ratio": var_sum / sum_var, "implied_mean_corr": (var_sum - sum_var) / cross_sd, + }) + multi = n_areas > 1 + + def pooled(w: np.ndarray) -> Tuple[float, float]: + vs = np.sum((gsum * gsum) @ w * multi) + sv = np.sum(((d * d) @ w) * multi[codes]) + ss = np.sqrt((d * d) @ w) + ssum = np.bincount(codes, weights=ss, minlength=k) + ssq = np.bincount(codes, weights=ss ** 2, minlength=k) + cs = np.sum((ssum ** 2 - ssq) * multi) + return float(vs / sv), float((vs - sv) / cs) + + ratio, corr = pooled(np.ones(n_rep)) + rng = np.random.default_rng(seed) + boots = np.array([pooled(_bootstrap_weights(rng, n_rep)) for _ in range(int(n_boot))]).reshape(-1, 2) + summary = { + "n_groups": int(multi.sum()), "n_areas": int(n_areas[multi].sum()), + "pooled_ratio": ratio, "pooled_ratio_se_boot": float(np.std(boots[:, 0], ddof=1)) if len(boots) > 1 else math.nan, + "pooled_implied_mean_corr": corr, + "pooled_implied_mean_corr_se_boot": float(np.std(boots[:, 1], ddof=1)) if len(boots) > 1 else math.nan, + "median_group_ratio": float(np.median(frame["ratio"][multi])) if multi.any() else math.nan, + } + return frame, summary + + +# --------------------------------------------------------------------------- +# Regularized and structured covariance models +# --------------------------------------------------------------------------- + + +def _standardized(d: np.ndarray) -> Tuple[np.ndarray, np.ndarray]: + """Observations x_r = 2 d_r (E x x^T = Sigma under 4/R scaling), standardized. + + Returns (T, n) standardized observations and the per-area SDs. + """ + n_rep = d.shape[1] + x = math.sqrt(_factor(n_rep) * n_rep) * d.T # sqrt(4) * d for any R + sd = np.sqrt(np.mean(x * x, axis=0)) + if np.any(sd == 0): + raise ValueError("zero-variance area(s); drop them before fitting covariance models") + return x / sd, sd + + +def ledoit_wolf_intensity(deviations: np.ndarray) -> Dict[str, float]: + """Ledoit-Wolf (2004) shrinkage of the replicate correlation toward the identity. + + Known-mean (deviations are centred on the full-sample estimate) version on + standardized observations z_r (T = R observations of n areas): sample + correlation S = Z^T Z / T, target I, intensity + min(1, beta^2 / delta^2) with delta^2 = ||S - I||_F^2 / n and + beta^2 = sum_r ||z_r z_r^T - S||_F^2 / (T^2 n). Computed through the + T x T Gram matrix, so n can be large. Intensity near 1 means the data + cannot support the unrestricted off-diagonal structure. + """ + z, _ = _standardized(np.asarray(deviations, dtype=float)) + t, n = z.shape + gram = z @ z.T # (T, T) + s_norm2 = float(np.sum(gram ** 2)) / t ** 2 # ||S||_F^2 + delta2 = (s_norm2 - 2.0 * float(np.trace(gram)) / t + n) / n # ||S - I||^2 / n (diag(S) = 1) + zz = np.diag(gram) # ||z_r||^2 + quad = np.sum(gram ** 2, axis=1) / t # z_r^T S z_r + beta2 = float(np.sum(zz ** 2 - 2.0 * quad + s_norm2)) / (t ** 2 * n) + beta2 = min(beta2, delta2) + intensity = 1.0 if delta2 <= 0 else beta2 / delta2 + return {"intensity": float(intensity), "delta2": float(delta2), "beta2": float(beta2), "n": int(n), "T": int(t)} + + +def _loglik_dense(r_mat: np.ndarray, z_test: np.ndarray) -> Tuple[float, bool]: + """Mean over test observations of -0.5 (logdet R + z^T R^-1 z); projects to PD if needed.""" + projected = False + try: + chol = np.linalg.cholesky(r_mat) + except np.linalg.LinAlgError: + vals, vecs = np.linalg.eigh(r_mat) + vals = np.clip(vals, 1e-3, None) + r_mat = (vecs * vals) @ vecs.T + s = np.sqrt(np.diag(r_mat)) + r_mat = r_mat / np.outer(s, s) + chol = np.linalg.cholesky(r_mat) + projected = True + logdet = 2.0 * float(np.sum(np.log(np.diag(chol)))) + sol = np.linalg.solve(chol, z_test.T) # L^-1 z + quad = np.sum(sol * sol, axis=0) + return float(np.mean(-0.5 * (logdet + quad))), projected + + +def _loglik_lowrank(a: float, z_train: np.ndarray, z_test: np.ndarray) -> float: + """Test log-likelihood for R = a I + (1 - a) Z^T Z / T via Woodbury.""" + t, n = z_train.shape + b = 1.0 - a + a = max(a, 1e-9) + if b <= 0: + return float(np.mean(-0.5 * np.sum(z_test * z_test, axis=1))) + m = np.eye(t) * (a * t / b) + z_train @ z_train.T + sign, logdet_m = np.linalg.slogdet(m) + logdet = n * math.log(a) + logdet_m - t * math.log(a * t / b) + u = z_train @ z_test.T # (T, m) + quad = (np.sum(z_test * z_test, axis=1) - np.sum(u * np.linalg.solve(m, u), axis=0)) / a + return float(np.mean(-0.5 * (logdet + quad))) + + +def holdout_model_comparison( + deviations: np.ndarray, + models: Mapping[str, np.ndarray], + *, + split: str = "random", + seed: int = 0, + remove_directions: Any = "auto", +) -> Dict[str, Any]: + """Score covariance models on replicates held out of the fit. + + Replicates are split in two halves (``split`` as in + :func:`split_half_agreement`); each model is fitted on one half and scored + on the other (both directions, averaged) by the Gaussian log-likelihood of + the held-out standardized replicate observations, relative to the + independence (diagonal) model, in nats per replicate. Per-area SDs come + from the training half for every model. + + ``models`` maps a name to an (n, n) integer class matrix: off-diagonal + entries >= 0 share one correlation parameter per class (estimated as the + training-half mean correlation of the class, the least-squares fit of the + structure); -1 fixes the correlation at 0; the diagonal is ignored. + A Ledoit-Wolf shrinkage model (identity target) is always included. + A fitted structure that is not positive definite is projected (eigenvalues + clipped at 1e-3) and flagged. + + The 80 replicates are not exchangeable draws: real ACS deviations of every + area nearly vanish along a few directions of replicate space (a property + of the replicate-factor construction; :func:`replicate_spectrum`). Then + the held-out half's component along such a direction is minus the + training half's for every area at once - a dependence between the halves + that a low-rank model (Ledoit-Wolf) exploits and that is not covariance + between areas. ``remove_directions="auto"`` (default) projects each half + orthogonally to these directions (restricted to the half) and to the + constant vector; an (k, R) array gives the directions explicitly; None + disables it. + """ + d = np.asarray(deviations, dtype=float) + n, n_rep = d.shape + if split == "random": + perm = np.random.default_rng(seed).permutation(n_rep) + halves = (np.sort(perm[: n_rep // 2]), np.sort(perm[n_rep // 2:])) + elif split == "first_last": + halves = (np.arange(n_rep // 2), np.arange(n_rep // 2, n_rep)) + elif split == "odd_even": + halves = (np.arange(0, n_rep, 2), np.arange(1, n_rep, 2)) + else: + raise ValueError("split must be 'random', 'first_last' or 'odd_even'") + scores: Dict[str, list] = {name: [] for name in list(models) + ["ledoit_wolf"]} + params: Dict[str, list] = {name: [] for name in models} + flags: Dict[str, bool] = {name: False for name in models} + lw_int = [] + base = [] + if isinstance(remove_directions, str) and remove_directions == "auto": + directions = np.vstack([replicate_spectrum(d)["null_directions"], np.ones((1, n_rep))]) + elif remove_directions is None: + directions = np.zeros((0, n_rep)) + else: + directions = np.atleast_2d(np.asarray(remove_directions, dtype=float)) + + def half(idx: np.ndarray) -> np.ndarray: + h = d[:, idx] + if not directions.shape[0]: + return h + q, rr = np.linalg.qr(directions[:, idx].T) + q = q[:, np.abs(np.diag(rr)) > 1e-10] + return h - (h @ q) @ q.T + + for tr, te in (halves, halves[::-1]): + z_tr, sd = _standardized(half(tr)) + x_te = math.sqrt(4.0) * half(te).T + z_te = x_te / sd + diag = float(np.mean(-0.5 * np.sum(z_te * z_te, axis=1))) + base.append(diag) + c_tr = (z_tr.T @ z_tr) / z_tr.shape[0] + for name, cls in models.items(): + cls = np.asarray(cls) + if cls.shape != (n, n): + raise ValueError(f"class matrix for {name!r} must be {n} x {n}") + off = ~np.eye(n, dtype=bool) + codes = cls[off] + vals = c_tr[off] + use = codes >= 0 + k = int(codes[use].max()) + 1 if use.any() else 0 + rho = (np.bincount(codes[use], weights=vals[use], minlength=k) + / np.maximum(np.bincount(codes[use], minlength=k), 1)) if k else np.zeros(0) + r_mat = np.zeros((n, n)) + r_mat[off] = np.where(use, rho[np.clip(codes, 0, None)] if k else 0.0, 0.0) + np.fill_diagonal(r_mat, 1.0) + ll, projected = _loglik_dense(r_mat, z_te) + scores[name].append(ll - diag) + params[name].append(rho.tolist()) + flags[name] = flags[name] or projected + lw = ledoit_wolf_intensity(half(tr)) + lw_int.append(lw["intensity"]) + scores["ledoit_wolf"].append(_loglik_lowrank(lw["intensity"], z_tr, z_te) - diag) + return { + "n_areas": int(n), "n_replicates": int(n_rep), "split": split, + "directions_removed": int(directions.shape[0]), + "gain_nats_per_replicate": {k: float(np.mean(v)) for k, v in scores.items()}, + "gain_by_direction": {k: [float(x) for x in v] for k, v in scores.items()}, + "fitted_rho": params, "projected_to_pd": flags, + "ledoit_wolf_intensity": [float(x) for x in lw_int], + } + + +def replicate_spectrum(deviations: np.ndarray, threshold: float = 0.25) -> Dict[str, Any]: + """Eigenvalues of the replicate-space Gram matrix of standardized deviations. + + For n areas with iid deviations the eigenvalues of Z^T Z / n (R x R; Z the + areas' deviations scaled to unit mean square) lie above the + Marchenko-Pastur lower edge (1 - sqrt(R/n))^2 (relative to their mean). + Directions with eigenvalue below ``threshold`` x that edge are near-null: + almost every area's deviation vector is orthogonal to them, a property of + the replicate design shared by all areas, which lowers the effective + number of replicates for a correlation to about R - k. Detection needs + n >= 4 R areas (otherwise no directions are returned and ``note`` says + so; pass directions found on a larger set, e.g. tracts, instead). + Returns the sorted eigenvalues relative to their mean, the (k, R) + near-null directions and R - k. + """ + d = np.asarray(deviations, dtype=float) + n_rep = d.shape[1] + norm = np.sqrt(np.mean(d * d, axis=1, keepdims=True)) + z = d[norm[:, 0] > 0] / norm[norm[:, 0] > 0] + n = z.shape[0] + _, s, vt = np.linalg.svd(z, full_matrices=False) + ev = s ** 2 / n + ratio = ev / np.mean(ev) + out = {"eigen_ratio": np.sort(ratio), "null_directions": np.zeros((0, n_rep)), "n_null": 0, + "effective_replicates": int(n_rep), "mp_lower_edge": math.nan, "note": ""} + if n < 4 * n_rep: + out["note"] = f"{n} areas < 4 x {n_rep} replicates: near-null directions not detected" + return out + edge = (1.0 - math.sqrt(n_rep / n)) ** 2 + null = ratio < threshold * edge + out.update(null_directions=vt[null], n_null=int(null.sum()), effective_replicates=int(n_rep - null.sum()), + mp_lower_edge=float(edge)) + return out + + +def rank_report(deviations: np.ndarray) -> Dict[str, Any]: + """How much an unrestricted covariance could be estimated from these replicates.""" + d = np.asarray(deviations, dtype=float) + n, n_rep = d.shape + sv = np.linalg.svd(d, compute_uv=False) + tol = sv.max() * max(n, n_rep) * np.finfo(float).eps if sv.size else 0.0 + rank = int(np.sum(sv > tol)) + energy = np.cumsum(sv ** 2) / np.sum(sv ** 2) if sv.size else np.array([]) + return { + "n_areas": int(n), "n_replicates": int(n_rep), "rank": rank, + "max_rank": int(min(n, n_rep)), + "unrestricted_correlations": int(n * (n - 1) // 2), + "replicate_values": int(n * n_rep), + "share_zero_eigenvalues": float(1.0 - rank / n) if n else math.nan, + "top5_share": float(energy[min(4, energy.size - 1)]) if energy.size else math.nan, + } diff --git a/census/report_tables.py b/census/report_tables.py index 72ebd28..3337e91 100644 --- a/census/report_tables.py +++ b/census/report_tables.py @@ -216,6 +216,10 @@ def regression_table( disp += [top, bottom] disp.append({"Term": "N", **{m: f"{results[m].n_obs:,}" for m in models}}) notes = [] + for m, res in results.items(): + if getattr(res, "effects", None) is not None: + notes.append(f"{m}: spatial lag model; its coefficients are not marginal effects: report the " + "direct/indirect/total effects (effects_table) first.") for m, res in results.items(): info = infos[m] # the interval text already carries the REPLICATES INCOMPLETE note: do not repeat it @@ -236,6 +240,50 @@ def regression_table( notes, citation, label) +def effects_table( + result: ModelResult, *, uncertainty: str = "se", digits: int = 3, + title: str = "Spatial lag model: direct, indirect and total effects", citation: str = "", label: str = "", +) -> ReportTable: + """Direct, indirect and total effects of a spatial lag ModelResult (``result.effects``). + + One row per covariate, columns Direct / Indirect / Total, each estimate + with ``(SE)`` or ``[lower, upper]`` beneath (the interval target of the + fit). These are the marginal effects of the lag model; its coefficients + (regression_table) are not. No significance markers. + """ + eff = getattr(result, "effects", None) + if eff is None: + raise ValueError("result has no effects table (only spatial lag models from census.spatial do)") + if uncertainty not in ("se", "ci"): + raise ValueError("uncertainty must be 'se' or 'ci'") + info = interval_description(result) + basis = info["basis_column"] + disp = [] + for v in dict.fromkeys(eff["variable"]): + top, bottom = {"Variable": v}, {"Variable": ""} + for e in ("direct", "indirect", "total"): + r = eff[(eff["variable"] == v) & (eff["effect"] == e)].iloc[0] + top[e.capitalize()] = _fmt(r["estimate"], digits) + se = r.get(basis) + ok = se is not None and np.isfinite(float(se)) + bottom[e.capitalize()] = ((f"({_fmt(se, digits)})" if ok else "(SE undefined)") if uncertainty == "se" + else (f"[{_fmt(r['ci_lower'], digits)}, {_fmt(r['ci_upper'], digits)}]" + if ok else "[undefined]")) + disp += [top, bottom] + data = eff.assign(se_basis=basis, confidence_level=result.confidence_level, method=result.method, + n_obs=result.n_obs) + notes = ["Effects of a one-unit change in each covariate in every area, averaged over areas (LeSage & Pace " + "2009): direct = own-area effect including feedback through neighbours; indirect = spillover " + "summed over all other areas; total = direct + indirect. The model coefficients (beta) are not " + "marginal effects.", + f"{result.method}; {info['text']}.", + ("Standard errors in parentheses" if uncertainty == "se" else "Confidence intervals in brackets") + + f" ({basis}; delta method from the fit's (beta, rho) covariance" + + (", plus replicate refits" if "sampling_replicate" in result.uncertainty_sources else "") + ")."] + return ReportTable(title, pd.DataFrame(disp, columns=["Variable", "Direct", "Indirect", "Total"]), data, + notes, citation, label) + + def descriptive_table( frame: pd.DataFrame, variables: Sequence[str], *, labels: Optional[Mapping[str, str]] = None, cv_threshold: float = 30.0, digits: int = 2, title: str = "Descriptive statistics", diff --git a/census/results.py b/census/results.py index c007919..4c9333e 100644 --- a/census/results.py +++ b/census/results.py @@ -278,19 +278,28 @@ class ModelResult: notes: List[str] = field(default_factory=list) diagnostics: Dict[str, Any] = field(default_factory=dict) provenance: Dict[str, Any] = field(default_factory=dict) + # Spatial lag models (census.spatial): direct/indirect/total effects, the + # marginal effects (the lag-model beta is not one). COEFFICIENT_COLUMNS + # plus "variable" and "effect"; None for every other model. + effects: Optional[pd.DataFrame] = None def summary(self, info: Optional[Dict[str, Any]] = None) -> str: """One line; includes the REPLICATES INCOMPLETE note (from the interval text) at most once.""" info = info or interval_description(self) + lead = ("EFFECTS FIRST: report the direct/indirect/total effects table (the marginal effects); " + "the lag-model coefficients (beta) are not marginal effects. " if self.effects is not None else "") return ( - f"{self.model_type} '{self.formula}', n={self.n_obs}, method={self.method}, " + f"{lead}{self.model_type} '{self.formula}', n={self.n_obs}, method={self.method}, " f"SEs include: {', '.join(self.uncertainty_sources)}; {info['text']}" ) def to_dict(self) -> Dict[str, Any]: info = interval_description(self) + head: Dict[str, Any] = {"model_type": self.model_type, "formula": self.formula, "n_obs": self.n_obs} + if self.effects is not None: # effects before the coefficients (PLAN_ROUND_16 A2) + head["effects"] = self.effects return _jsonable({ - "model_type": self.model_type, "formula": self.formula, "n_obs": self.n_obs, + **head, "coefficients": self.coefficients, "method": self.method, "uncertainty_sources": self.uncertainty_sources, "confidence_level": self.confidence_level, "notes": self.notes, diff --git a/census/simulation.py b/census/simulation.py index 6fa52b6..40bcb94 100644 --- a/census/simulation.py +++ b/census/simulation.py @@ -1092,11 +1092,16 @@ def designs_config(fast: bool = True, **overrides: Any) -> SimConfig: def group_config(group: str, fast: bool) -> SimConfig: - """Configuration of a study group: "main" (FULL/FAST), "spatial" or "designs".""" + """Configuration of a study group: "main" (FULL/FAST), "spatial", "designs", "lisa" or "spreg_validation".""" if group == "spatial": return spatial_config(fast=fast) if group == "designs": return designs_config(fast=fast) + if group == "lisa": + return _lisa.lisa_config(fast=fast) + if group == "spreg_validation": + from .simulation_spreg import spreg_validation_config + return spreg_validation_config(fast=fast) if group != "main": raise ValueError(f"unknown study group {group!r}") return FAST if fast else FULL @@ -1231,3 +1236,23 @@ def serial_config(fast: bool = True, **overrides: Any) -> SimConfig: reps = {k: 0 for k in base.reps} reps.update(SERIAL_REPS_FAST if fast else SERIAL_REPS_FULL) return replace(base, reps=reps, **overrides) + + +# --------------------------------------------------------------------------- +# Round 16 (PLAN_ROUND_16 workstream B): LISA under sampling error +# --------------------------------------------------------------------------- +# Info rows only, on demand (scripts/inference_validation.py --studies NAME); +# not in any default report. Design and methods: census.lisa_validation, which +# registers its studies in STUDIES when imported. +from . import lisa_validation as _lisa # noqa: E402 + +# Round 16 (PLAN_ROUND_16 workstream A): spatial regression validation +# --------------------------------------------------------------------------- +# Info-only studies in census.simulation_spreg (group "spreg_validation"), +# registered with no default reps; run with +# run_study(spreg_validation_config(fast=...), studies=[...]) or +# scripts/inference_validation.py --studies NAME. + +from .simulation_spreg import SPREG_VALIDATION_STUDIES # noqa: E402 (registers the studies) + +STUDIES.update(SPREG_VALIDATION_STUDIES) diff --git a/census/simulation_spreg.py b/census/simulation_spreg.py new file mode 100644 index 0000000..b2a9b1f --- /dev/null +++ b/census/simulation_spreg.py @@ -0,0 +1,862 @@ +# SPDX-License-Identifier: AGPL-3.0-or-later +"""Round 16 workstream A: validation of spatial regression under ACS sampling error (info only). + +PLAN_ROUND_16 section 1. Seeded simulations on a 12 x 12 lattice with +known truth; every row has status ``info`` (gating is an owner decision). +Results and conclusions: docs/audits/spatial_regression_validation.md. + +Studies (registered in census.simulation.STUDIES, group "spreg_validation"; +run with ``scripts/inference_validation.py --studies NAME``): + +* ``spatial_error_attenuation`` (A1, independent sampling errors): + spatial error DGP y* = 10 + beta x + (I - lambda W)^-1 e at lambda 0, + 0.3, 0.6, weights rook / queen / kNN (k = SPREG16_KNN_K); published + y = y* + d with independent d_i ~ N(0, s_i^2), s_i = CV_i x mean(y*), CVs + from the tract profile scaled by 0, 0.5, 1, 2. Fitted: the tool default + (spreg ML_Error, uncorrected) and the engine's existing spatial + Fay-Herriot likelihood (census.sae_spatial) with + V = sigma^2 [(I - lambda W)'(I - lambda W)]^-1 + diag(s^2), REML and ML, + whose rho is the SEM's lambda corrected for independent outcome error. + These rows call the shipped estimator itself, + census.spatial.error_model_known_variances (the function behind + ``spatial_regression(correct_outcome_error=True)``); :func:`me_sem_fit` + is used only for the dense-Sigma oracle. +* ``spatial_error_attenuation_correlated`` (A1, correlated sampling + errors): as above at CV x 1, with d ~ N(0, S R(gamma) S), R(gamma) the + correlation matrix of a SAR(gamma) process on the same W (gamma 0.3, 0.6; + a stated assumption, not an estimate of the ACS structure). Fitted: the + uncorrected ML_Error, the spatial Fay-Herriot correction (which assumes + independence), and an oracle given the true error covariance. +* ``spatial_dgp_recovery`` (A2/A3): SEM, SAR lag, spatial Durbin + (y = rho W y + X beta + W X theta + e), an omitted spatially correlated + covariate, and Juhl's (2021) omitted-variable DGP, with and without + outcome sampling error; the tool's four estimators (lag GMM/ML, error + ML/GMM) through spatial_regression. Reports recovery of the disturbance + parameter (lambda), the spatial lag parameter (rho), beta, and the + direct/indirect/total effects with their coverage (A2: lag-model + effects from spatial_regression's effects table). +* ``common_factor_lr`` (A4): likelihood-ratio test of the SEM as the + restricted SDM (theta = -rho beta; SDM by ML lag with W X, SEM by ML + error) in Juhl's (2021, Political Analysis 29(2):193-211) design: size + and power with chi-square(1) and Juhl's residual-bootstrap critical + values. ``common_factor_lr_acs``: the same test under ACS-like outcome + error. + +Every (study, cell) draws from its own generator seeded from cfg.seed and the +cell's position, so results do not depend on run order or process. +Monte Carlo SEs: coverage and rates sqrt(p(1-p)/R); bias sd/sqrt(R); RMSE +by the delta method, SE(MSE)/(2 RMSE). +""" + +import math +import warnings +from dataclasses import replace +from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple + +import numpy as np +from scipy import linalg as sla, optimize, stats as sps + +from .simulation import (FAST, FULL, NOMINAL, SimConfig, _geoids, acs_frame, draw_cv, lattice_weights, + sar_inverse) + +SPREG16_SIDE = 12 +SPREG16_KNN_K = 6 +SPREG16_WEIGHTS = ("rook", "queen", "knn") +SPREG16_LAMBDAS = (0.0, 0.3, 0.6) +SPREG16_CV_SCALES = (0.0, 0.5, 1.0, 2.0) # x tract-profile CVs (0 = no sampling error) +SPREG16_GAMMAS = (0.3, 0.6) # SAR strength of the sampling-error correlation +SPREG16_BOUNDARY = 0.98 # |lambda| at or above: boundary estimate +SPREG16_LR_ALPHA = 0.05 +SPREG16_LR_BOOT = 100 # bootstrap draws per test (Juhl 2021 sec. 3.3) +SPREG16_LR_BOOT_REPS = 200 # repetitions per cell that also run the bootstraps +# Juhl (2021) design: y = x beta + (I - rho W)^-1 (x gamma + v), beta 2, x, v ~ N(0, 1), queen W +SPREG16_JUHL_SIDES = (7, 10, 15, 20) +SPREG16_JUHL_RHOS = (0.0, 0.2, 0.4, 0.6, 0.8) +SPREG16_JUHL_GAMMAS_LR = (0.0, 0.5, 1.0) # 0: SEM (H0 true); > 0: SDM-like (H0 false) +SPREG16_JUHL_BOOT_GAMMAS = (0.0, 0.5) # cells that also run the bootstrap +SPREG16_JUHL_BETA = 2.0 +SPREG16_ACS_LR_GAMMAS = (0.0, 0.25) # ACS extension (12 x 12 queen, beta 0.5, x ~ N(50, 24^2)) +SPREG16_ACS_LR_RHOS = (0.3, 0.6) +SPREG16_SDM_RHO = 0.4 +SPREG16_SDM_THETA = 0.3 +SPREG16_OMIT_CORR = 0.5 # corr(x, z) for the omitted covariate z +SPREG16_OMIT_RHO = 0.7 # SAR autocorrelation of z +SPREG16_EFFECT_DRAWS = 1000 # draws for simulated (LeSage & Pace) effect intervals +# spreg's OLS-based RS tests (Koley & Bera 2024): none of them tests SEM vs SDM (the common factor) +SPREG16_RS_TESTS = {"lml": "rho = 0 (lag), assuming theta = 0", "lmwx": "theta = 0 (WX), assuming rho = 0", + "lmspdurbin": "rho = 0 and theta = 0 (joint)", "rlmwx": "theta = 0, robust to rho", + "rlmdurlag": "rho = 0, robust to theta"} +SPREG16_JUHL_RHO = 0.5 # Juhl (2021) Eq. 5 DGP in the A3 set ... +SPREG16_JUHL_GAMMA = 0.25 # ... with gamma / beta = 0.5 +SPREG16_REPS_FULL = {"spatial_error_attenuation": 300, "spatial_error_attenuation_correlated": 300, + "spatial_dgp_recovery": 400, "common_factor_lr": 1000, "common_factor_lr_acs": 500} +SPREG16_REPS_FAST = {"spatial_error_attenuation": 3, "spatial_error_attenuation_correlated": 3, + "spatial_dgp_recovery": 3, "common_factor_lr": 6, "common_factor_lr_acs": 4} + + +# --------------------------------------------------------------------------- +# Weights and error structures +# --------------------------------------------------------------------------- + + +def lattice_w(side: int, kind: str, ids: Optional[Sequence[str]] = None): + """Row-standardized libpysal W on a side x side lattice: rook, queen or kNN. + + kNN uses cell centres with a fixed tiny jitter (seed 0, 1e-6) so ties + among equidistant cells are broken the same way on every machine. + """ + import libpysal + + ids = list(ids) if ids is not None else _geoids(side * side) + if kind == "rook": + return lattice_weights(side, ids) + r, c = np.divmod(np.arange(side * side), side) + if kind == "queen": + nb = {ids[i]: [ids[j] for j in range(side * side) + if j != i and abs(r[i] - r[j]) <= 1 and abs(c[i] - c[j]) <= 1] for i in range(side * side)} + w = libpysal.weights.W(nb, id_order=ids, silence_warnings=True) + elif kind == "knn": + xy = np.column_stack([c, r]).astype(float) + 1e-6 * np.random.default_rng(0).standard_normal((side * side, 2)) + w = libpysal.weights.KNN(xy, k=SPREG16_KNN_K, ids=ids) + else: + raise ValueError(f"unknown lattice weights {kind!r}") + w.transform = "r" + return w + + +def sar_correlation(W: np.ndarray, gamma: float) -> np.ndarray: + """Correlation matrix of a SAR(gamma) process on W: C = [(I - gW)'(I - gW)]^-1 scaled to unit diagonal.""" + B = np.eye(W.shape[0]) - gamma * W + C = sla.cho_solve(sla.cho_factor(B.T @ B), np.eye(W.shape[0])) + d = np.sqrt(np.diag(C)) + R = C / np.outer(d, d) + return 0.5 * (R + R.T) + + +def neighbour_correlation(R: np.ndarray, W: np.ndarray) -> float: + """Mean correlation over first-order neighbour pairs (W_ij > 0).""" + return float(R[W > 0].mean()) + + +# --------------------------------------------------------------------------- +# Estimators used only by the studies +# --------------------------------------------------------------------------- + + +def me_sem_fit(y: np.ndarray, X: np.ndarray, W: np.ndarray, Sigma: np.ndarray, + starts: Sequence[float] = (0.0, 0.5), method: str = "REML") -> Dict[str, Any]: + """REML/ML spatial error model with known measurement-error covariance ``Sigma`` (study-only). + + y = X beta + u + d, u = lambda W u + e, e ~ N(0, A I), d ~ N(0, Sigma), + so V = A C(lambda) + Sigma with C = [(I - lambda W)'(I - lambda W)]^-1. + With diagonal Sigma this is exactly the engine's spatial Fay-Herriot + model (census.sae_spatial: same likelihoods, REML default or ML; Molina + & Marhuenda 2015 eq. (6), independent sampling errors of known + variance) and lambda is its rho; a dense Sigma is the study's oracle for + correlated errors. beta by GLS (profiled); (log A, lambda) by L-BFGS-B + with analytic scores, lambda in [-0.99, 0.99], from each lambda in + ``starts``. SEs from the expected information 0.5 tr(M V_k M V_l), + M = P (REML) or V^-1 (ML), as census.sae_spatial's rho_se, and + (X'V^-1X)^-1 (beta). + + Its parameterization, bounds, starts and convergence rule differ from the + shipped estimator (census.spatial.error_model_known_variances), so the A1 + diagonal-Sigma rows do not use it: here it is the dense-Sigma oracle (and + the null fit of the common-factor bootstrap). + """ + method = method.upper() + if method not in ("REML", "ML"): + raise ValueError("method must be REML or ML") + n = len(y) + X = np.column_stack([np.ones(n), X]) if not np.allclose(X[:, 0], 1) else X + I = np.eye(n) + lam_b = 0.99 + + def state(A, lam): + B = I - lam * W + C = sla.cho_solve(sla.cho_factor(B.T @ B), I) + C = 0.5 * (C + C.T) + V = A * C + Sigma + cf = sla.cho_factor(V) + Vi = sla.cho_solve(cf, I) + Vi = 0.5 * (Vi + Vi.T) + ViX = Vi @ X + Q = sla.inv(X.T @ ViX) + beta = Q @ (ViX.T @ y) + r = y - X @ beta + Vr = Vi @ r + dC = C @ (W + W.T - 2 * lam * W.T @ W) @ C + logdet = 2.0 * float(np.sum(np.log(np.diag(cf[0])))) + M = Vi - ViX @ Q @ ViX.T if method == "REML" else Vi + return {"C": C, "Vi": Vi, "Q": Q, "beta": beta, "Vr": Vr, "dC": dC, "logdet": logdet, "r": r, "M": M} + + def nll(t): + A, lam = math.exp(t[0]), float(t[1]) + try: + s = state(A, lam) + except (np.linalg.LinAlgError, sla.LinAlgError, ValueError): + return 1e300, np.zeros(2) + if method == "REML": + ll = -0.5 * ((n - X.shape[1]) * math.log(2 * math.pi) + s["logdet"] - np.linalg.slogdet(s["Q"])[1] + + s["r"] @ s["Vr"]) + else: + ll = -0.5 * (n * math.log(2 * math.pi) + s["logdet"] + s["r"] @ s["Vr"]) + g = [] + for Vk in (s["C"], A * s["dC"]): # P y = V^-1 r (GLS residual r) under REML and ML + g.append(-0.5 * np.sum(s["M"] * Vk.T) + 0.5 * s["Vr"] @ Vk @ s["Vr"]) + return -ll, -np.array([A * g[0], g[1]]) + + var_y = float(np.var(y, ddof=1)) + a0 = math.log(max(var_y - float(np.mean(np.diag(Sigma))), 0.1 * var_y)) + best = None + for l0 in starts: + res = optimize.minimize(nll, np.array([a0, l0]), jac=True, method="L-BFGS-B", + bounds=[(math.log(var_y) - 25, math.log(var_y) + 5), (-lam_b, lam_b)], + options={"ftol": 1e-13, "gtol": 1e-8, "maxiter": 500}) + if best is None or res.fun < best.fun: + best = res + A, lam = math.exp(float(best.x[0])), float(best.x[1]) + s = state(A, lam) + M = [s["M"] @ s["C"], s["M"] @ (A * s["dC"])] + F = np.array([[0.5 * np.sum(M[a] * M[b].T) for b in range(2)] for a in range(2)]) + try: + with warnings.catch_warnings(): # singular at the A = 0 boundary: se_lambda is then not meaningful + warnings.simplefilter("ignore", sla.LinAlgWarning) + Fi = sla.inv(F) + se_lam = math.sqrt(Fi[1, 1]) if Fi[1, 1] > 0 else float("nan") + except (np.linalg.LinAlgError, sla.LinAlgError, ValueError): + se_lam = float("nan") + return {"lambda": lam, "A": A, "beta": s["beta"], "se_beta": np.sqrt(np.diag(s["Q"])), "se_lambda": se_lam, + "converged": bool(best.success) and np.isfinite(best.fun), "loglik": -float(best.fun), + "boundary": abs(lam) >= SPREG16_BOUNDARY or A <= 1e-8 * var_y} + + +def _logdet_terms(eig: np.ndarray, a: float) -> float: + """log|I - a W| from the eigenvalues of W (real part of the complex sum).""" + return float(np.sum(np.log(np.abs(1.0 - a * eig)))) + + +def sem_ml(y: np.ndarray, X: np.ndarray, Wd: np.ndarray, eig: np.ndarray) -> Dict[str, float]: + """Concentrated ML of the spatial error model (same likelihood as spreg.ML_Error, method 'full'). + + X includes the constant. The full log-likelihood includes the constants + -n/2 (log 2 pi + 1), as in spreg; lambda bounded to (-1, 1). + """ + n = len(y) + Wy, WX = Wd @ y, Wd @ X + + def nll(lam): + ys, xs = y - lam * Wy, X - lam * WX + b = np.linalg.lstsq(xs, ys, rcond=None)[0] + e = ys - xs @ b + return 0.5 * n * math.log(float(e @ e) / n) - _logdet_terms(eig, lam) + + res = optimize.minimize_scalar(nll, bounds=(-1.0, 1.0), method="bounded", options={"xatol": 1e-7}) + lam = float(res.x) + ys, xs = y - lam * Wy, X - lam * WX + b = np.linalg.lstsq(xs, ys, rcond=None)[0] + e = ys - xs @ b + return {"lambda": lam, "beta": b, "sig2": float(e @ e) / n, + "loglik": -float(res.fun) - 0.5 * n * (math.log(2 * math.pi) + 1)} + + +def sdm_ml(y: np.ndarray, X: np.ndarray, Wd: np.ndarray, eig: np.ndarray) -> Dict[str, Any]: + """Concentrated ML of the spatial Durbin model y = rho W y + X beta + W X1 theta + e. + + X includes the constant; W X1 lags the non-constant columns (as spreg.ML_Lag + with slx_lags=1). Same likelihood as spreg.ML_Lag (method 'full'), rho in (-1, 1). + """ + n = len(y) + Z = np.column_stack([X, Wd @ X[:, 1:]]) + Wy = Wd @ y + b0 = np.linalg.lstsq(Z, y, rcond=None)[0] + b1 = np.linalg.lstsq(Z, Wy, rcond=None)[0] + e0, e1 = y - Z @ b0, Wy - Z @ b1 + + def nll(rho): + e = e0 - rho * e1 + return 0.5 * n * math.log(float(e @ e) / n) - _logdet_terms(eig, rho) + + res = optimize.minimize_scalar(nll, bounds=(-1.0, 1.0), method="bounded", options={"xatol": 1e-7}) + rho = float(res.x) + e = e0 - rho * e1 + return {"rho": rho, "coef": b0 - rho * b1, "sig2": float(e @ e) / n, + "loglik": -float(res.fun) - 0.5 * n * (math.log(2 * math.pi) + 1)} + + +def common_factor_lr(y: np.ndarray, X: np.ndarray, Wd: np.ndarray, eig: np.ndarray) -> Tuple[float, Dict, Dict]: + """LR = 2 (logL_SDM - logL_SEM) for H0: theta = -rho beta (k - 1 restrictions); (LR, sdm, sem).""" + sdm, sem = sdm_ml(y, X, Wd, eig), sem_ml(y, X, Wd, eig) + return max(2.0 * (sdm["loglik"] - sem["loglik"]), 0.0), sdm, sem + + +# --------------------------------------------------------------------------- +# Truth and summaries +# --------------------------------------------------------------------------- + + +def true_effects(W: np.ndarray, rho: float, beta: float, theta: float = 0.0) -> Dict[str, float]: + """Average direct/indirect/total effects of x for y = rho W y + x beta + W x theta + e. + + S = (I - rho W)^-1 (beta I + theta W): direct = tr(S)/n, total = 1'S1/n, + indirect = total - direct (LeSage & Pace 2009). rho = 0, theta = 0 gives (beta, 0, beta). + """ + n = W.shape[0] + S = sar_inverse(W, rho) @ (beta * np.eye(n) + theta * W) + direct, total = float(np.trace(S) / n), float(S.sum() / n) + return {"direct": direct, "indirect": total - direct, "total": total} + + +def _metric_row(method: str, target: str, est: Sequence[float], truth: Optional[float], + covered: Optional[Sequence[bool]], extra: Dict[str, Any], ses: Optional[Sequence[float]] = None + ) -> Dict[str, Any]: + """Info row: coverage (if intervals), bias, RMSE and their Monte Carlo SEs.""" + e = np.asarray(est, float) + e = e[np.isfinite(e)] + r = int(e.size) + row: Dict[str, Any] = {"method": method, "target": target, "reps": r, "nominal": NOMINAL, + "coverage": float("nan"), "mc_se": float("nan"), "status": "info"} + if covered is not None and len(covered): + c = np.asarray(covered, bool) + cov = float(c.mean()) + row.update(coverage=cov, mc_se=math.sqrt(cov * (1 - cov) / c.size), reps=int(c.size)) + row["mean_estimate"] = float(e.mean()) if r else float("nan") + if truth is not None and r: + d = e - truth + mse = float(np.mean(d ** 2)) + rmse = math.sqrt(mse) + row.update(truth=truth, bias=float(d.mean()), + bias_mc_se=float(d.std(ddof=1) / math.sqrt(r)) if r > 1 else float("nan"), + rmse=rmse, rmse_mc_se=(float(np.std(d ** 2, ddof=1) / math.sqrt(r)) / (2 * rmse) + if r > 1 and rmse > 0 else float("nan"))) + else: + row.update(truth=truth if truth is not None else float("nan"), bias=float("nan")) + if ses is not None: + s = np.asarray(ses, float) + s = s[np.isfinite(s)] + if s.size and r > 1 and e.std(ddof=1) > 0: + row["se_sd_ratio"] = float(s.mean() / e.std(ddof=1)) + row.update(extra) + return row + + +def _rate(method: str, target: str, hits: Sequence[bool], nominal_rate: float, extra: Dict[str, Any]) -> Dict[str, Any]: + v = np.asarray(hits, bool) + p = float(v.mean()) if v.size else float("nan") + row = {"method": method, "target": target, "reps": int(v.size), "nominal": 1 - nominal_rate, + "coverage": 1 - p, "mc_se": math.sqrt(p * (1 - p) / v.size) if v.size else float("nan"), + "status": "info", "size": p, "nominal_size": nominal_rate} + row.update(extra) + return row + + +def _z() -> float: + return float(sps.norm.ppf(0.5 + NOMINAL / 2)) + + +# --------------------------------------------------------------------------- +# A1: lambda attenuation +# --------------------------------------------------------------------------- + + +def _uncorrected_sem(y: np.ndarray, x: np.ndarray, w) -> Tuple[float, float, float, float]: + """The tool default for model='error' (spreg ML_Error, method 'full'): (beta, se, lambda, se).""" + from .spatial import _spreg_fit + + _, b, se, lam = _spreg_fit(y[:, None], x[:, None], w, "error", "ml", ["x"], "y", "full", {}) + return float(b[1]), float(se[1]), lam, float(se[2]) + + +def _shipped_correction(y: np.ndarray, x: np.ndarray, W: np.ndarray, s: np.ndarray, method: str + ) -> Tuple[float, float, float, float, bool]: + """The tool's correct_outcome_error estimator (census.spatial.error_model_known_variances, diagonal s^2): + (beta, se, lambda, se, boundary). Non-convergence and singular fits raise RuntimeError (a fit failure).""" + from .spatial import error_model_known_variances + + f = error_model_known_variances(y, np.column_stack([np.ones(len(y)), x]), W, s ** 2, method) + return (float(f["beta"][1]), float(math.sqrt(f["vcov_beta"][1, 1])), f["lambda"], f["se_lambda"], + f["boundary"]) + + +def _attenuation_cell(cfg: SimConfig, reps: int, seed: int, kind: str, lam: float, cv_scale: float, + gamma: Optional[float]) -> List[Dict[str, Any]]: + side = SPREG16_SIDE + m = side * side + g = _geoids(m) + w = lattice_w(side, kind, g) + W = w.full()[0] + Binv = sar_inverse(W, lam) + mean_y = 10.0 + cfg.beta * cfg.x_mean + Lr = np.linalg.cholesky(sar_correlation(W, gamma)) if gamma else None + rng = np.random.default_rng(seed) + z = _z() + # (label, method, Sigma kind): the engine's spatial Fay-Herriot likelihood (diagonal Sigma) + # by REML (its default) and ML, and the full-Sigma oracle under correlated errors + corrected = [] + if cv_scale > 0: + corrected += [("spatial Fay-Herriot REML rho (diagonal SE^2: independence assumed)", "REML", "diag"), + ("spatial Fay-Herriot ML rho (diagonal SE^2: independence assumed)", "ML", "diag")] + if gamma: + corrected.append(("oracle: SEM + known correlated error covariance (REML, true Sigma)", "REML", "full")) + names = ["uncorrected ML_Error [tool default]"] + [c[0] for c in corrected] + rec = {k: {"lam": [], "lam_se": [], "lam_cov": [], "b": [], "b_se": [], "b_cov": [], "fail": 0, "boundary": []} + for k in names} + for _ in range(reps): + x = cfg.x_mean + cfg.x_sd * rng.standard_normal(m) + ystar = 10.0 + cfg.beta * x + Binv @ (cfg.sigma_u * rng.standard_normal(m)) + s = draw_cv(m, "tract", rng) / 100 * mean_y * cv_scale + zz = rng.standard_normal(m) + d = s * (Lr @ zz if Lr is not None else zz) + y = ystar + d + fits: Dict[str, Tuple[float, float, float, float, bool]] = {} + try: + b, bse, lh, lse = _uncorrected_sem(y, x, w) + fits[names[0]] = (b, bse, lh, lse, abs(lh) >= SPREG16_BOUNDARY) + except Exception: # noqa: BLE001 - counted and reported + rec[names[0]]["fail"] += 1 + X = x[:, None] + for k, meth, kind_s in corrected: + try: + if kind_s == "diag": # the shipped estimator (correct_outcome_error=True), same function + fits[k] = _shipped_correction(y, x, W, s, meth) + continue + Sig = (s[:, None] * Lr) @ (s[:, None] * Lr).T + f = me_sem_fit(y, X, W, Sig, method=meth) + if not f["converged"]: + raise ValueError("not converged") + fits[k] = (float(f["beta"][1]), float(f["se_beta"][1]), f["lambda"], f["se_lambda"], f["boundary"]) + except Exception: # noqa: BLE001 - counted and reported + rec[k]["fail"] += 1 + for k, (b, bse, lh, lse, bd) in fits.items(): + r = rec[k] + r["lam"].append(lh) + r["lam_se"].append(lse) + r["lam_cov"].append(bool(np.isfinite(lse) and abs(lh - lam) <= z * lse)) + r["b"].append(b) + r["b_se"].append(bse) + r["b_cov"].append(bool(abs(b - cfg.beta) <= z * bse)) + r["boundary"].append(bool(bd)) + rows = [] + errs = ("none" if cv_scale == 0 else f"independent, CV x {cv_scale:g}" if not gamma + else f"correlated SAR(gamma={gamma:g}) on W, CV x {cv_scale:g}") + ncorr = neighbour_correlation(sar_correlation(W, gamma), W) if gamma else 0.0 + for k in names: + r = rec[k] + bd = np.asarray(r["boundary"], bool) + base = {"weights": kind, "lambda": lam, "cv_scale": cv_scale, "gamma": gamma or 0.0, + "neighbour_error_corr": ncorr, "estimator": k, "fit_failures": r["fail"], + "boundary_rate": float(bd.mean()) if bd.size else float("nan"), + "boundary_mc_se": float(math.sqrt(bd.mean() * (1 - bd.mean()) / bd.size)) if bd.size else float("nan")} + lab = f"{k}; {kind} W, lambda={lam:g}, sampling error {errs}" + rows.append(_metric_row(lab, f"lambda ({lam:g}), disturbance parameter", r["lam"], lam, r["lam_cov"], + {**base, "term": "lambda", "parameter_class": "disturbance"}, r["lam_se"])) + rows.append(_metric_row(lab, f"beta ({cfg.beta:g})", r["b"], cfg.beta, r["b_cov"], + {**base, "term": "x", "parameter_class": "substantive"}, r["b_se"])) + return rows + + +def study_spatial_error_attenuation(cfg: SimConfig, reps: int) -> List[Dict[str, Any]]: + """A1 under independent sampling errors: W x lambda x CV scale (docstring of the module).""" + rows: List[Dict[str, Any]] = [] + for wi, kind in enumerate(SPREG16_WEIGHTS): + for li, lam in enumerate(SPREG16_LAMBDAS): + for ci, cs in enumerate(SPREG16_CV_SCALES): + seed = cfg.seed + 1600 + 100 * wi + 10 * li + ci + rows += _attenuation_cell(cfg, reps, seed, kind, lam, cs, None) + return rows + + +def study_spatial_error_attenuation_correlated(cfg: SimConfig, reps: int) -> List[Dict[str, Any]]: + """A1 under spatially correlated sampling errors (SAR(gamma) correlation on W, CV x 1).""" + rows: List[Dict[str, Any]] = [] + for wi, kind in enumerate(SPREG16_WEIGHTS): + for li, lam in enumerate(SPREG16_LAMBDAS): + for gi, gam in enumerate(SPREG16_GAMMAS): + seed = cfg.seed + 2000 + 100 * wi + 10 * li + gi + rows += _attenuation_cell(cfg, reps, seed, kind, lam, 1.0, gam) + return rows + + +# --------------------------------------------------------------------------- +# A2/A3: DGP set +# --------------------------------------------------------------------------- + +# (label, kind, parameters); kind: sem, sar, sdm, omitted +SPREG16_DGPS = ( + ("SEM lambda=0.6", "sem", {"lam": 0.6}), + ("SAR lag rho=0.3", "sar", {"rho": 0.3}), + ("SAR lag rho=0.6", "sar", {"rho": 0.6}), + (f"SDM rho={SPREG16_SDM_RHO:g} theta={SPREG16_SDM_THETA:g}", "sdm", {"rho": SPREG16_SDM_RHO, + "theta": SPREG16_SDM_THETA}), + (f"omitted SAR({SPREG16_OMIT_RHO:g}) covariate, corr(x, z)={SPREG16_OMIT_CORR:g}", "omitted", {}), + # Juhl (2021) Eq. 5: a spatially clustered omitted variable correlated with x, + # y = x beta + (I - rho W)^-1 (x gamma + v) = SDM with beta + gamma and theta = -rho beta + (f"Juhl omitted-variable DGP rho={SPREG16_JUHL_RHO:g} gamma={SPREG16_JUHL_GAMMA:g}", "juhl", + {"rho": SPREG16_JUHL_RHO, "gamma": SPREG16_JUHL_GAMMA}), +) + + +def _dgp_truth(cfg: SimConfig, kind: str, p: Dict[str, float], W: np.ndarray) -> Dict[str, float]: + """True average effects of x in a DGP of SPREG16_DGPS.""" + if kind == "juhl": + return true_effects(W, p["rho"], cfg.beta + p["gamma"], -p["rho"] * cfg.beta) + return true_effects(W, p.get("rho", 0.0), cfg.beta, p.get("theta", 0.0)) + + +def _dgp_draw(cfg: SimConfig, kind: str, p: Dict[str, float], W: np.ndarray, inv: Dict[str, np.ndarray], + rng: np.random.Generator) -> Tuple[np.ndarray, np.ndarray]: + m = W.shape[0] + e = cfg.sigma_u * rng.standard_normal(m) + if kind == "omitted": + zf = inv["z"] @ rng.standard_normal(m) + zf = (zf - zf.mean()) / zf.std() + x = cfg.x_mean + cfg.x_sd * (SPREG16_OMIT_CORR * zf + math.sqrt(1 - SPREG16_OMIT_CORR ** 2) + * rng.standard_normal(m)) + return x, 10.0 + cfg.beta * x + cfg.sigma_u * zf + e + x = cfg.x_mean + cfg.x_sd * rng.standard_normal(m) + if kind == "sem": + return x, 10.0 + cfg.beta * x + inv["S"] @ e + if kind == "sar": + return x, inv["S"] @ (10.0 + cfg.beta * x + e) + if kind == "juhl": + return x, 10.0 + cfg.beta * x + inv["S"] @ (p["gamma"] * (x - cfg.x_mean) + e) + return x, inv["S"] @ (10.0 + cfg.beta * x + p["theta"] * (W @ x) + e) + + +def study_spatial_dgp_recovery(cfg: SimConfig, reps: int) -> List[Dict[str, Any]]: + """A2/A3: the tool's lag and error models fitted to SEM, SAR, SDM and omitted-covariate DGPs. + + Rook 12 x 12 lattice; outcome sampling error from the tract CV profile + (scenario 'acs') or none. Each (DGP, scenario) cell has its own seed. + Lag-model effects and their 90% intervals come from spatial_regression's + effects table; the error model's effects are beta (direct = total, indirect + 0 by construction), so its indirect effect has no interval. + """ + from .spatial import SPATIAL_METHODS, SPATIAL_MODELS, DEFAULT_METHODS, spatial_regression + + side = SPREG16_SIDE + m = side * side + g = _geoids(m) + w = lattice_weights(side, g) + W = w.full()[0] + eig = np.linalg.eigvals(W) + rows: List[Dict[str, Any]] = [] + for di, (label, kind, p) in enumerate(SPREG16_DGPS): + rho = p.get("rho", 0.0) + inv = {"S": sar_inverse(W, p.get("lam", rho))} + if kind == "omitted": + inv["z"] = sar_inverse(W, SPREG16_OMIT_RHO) + truth = _dgp_truth(cfg, kind, p, W) + mean_y = float(np.mean(inv["S"] @ np.full(m, 10.0 + cfg.beta * cfg.x_mean + p.get("theta", 0.0) * cfg.x_mean))) \ + if kind in ("sar", "sdm") else 10.0 + cfg.beta * cfg.x_mean + for si, scen in enumerate(("acs", "none")): + rng = np.random.default_rng(cfg.seed + 2400 + 10 * di + si) + rng_sim = np.random.default_rng([cfg.seed, 2400 + 10 * di + si, 1]) # effect-interval draws + fits = [(mod, meth) for mod in SPATIAL_MODELS for meth in SPATIAL_METHODS] + rec = {f: {"beta": [], "beta_cov": [], "sp": [], "sp_cov": [], "fail": 0, + **{e: [] for e in ("direct", "indirect", "total")}, + **{f"{e}_cov": [] for e in ("direct", "indirect", "total")}, + **{f"{e}_simcov": [] for e in ("direct", "indirect", "total")}, + **{f"{e}_se": [] for e in ("direct", "indirect", "total")}} for f in fits} + rs_rej: Dict[str, List[bool]] = {k: [] for k in SPREG16_RS_TESTS} + for _ in range(reps): + x, ystar = _dgp_draw(cfg, kind, p, W, inv, rng) + ses: Dict[str, np.ndarray] = {} + y = ystar + if scen == "acs": + ses["y"] = draw_cv(m, "tract", rng) / 100 * mean_y + y = ystar + ses["y"] * rng.standard_normal(m) + frame = acs_frame(g, {"y": y, "x": x}, ses) + for k, rej in _rs_rejections(y, x, w).items(): + rs_rej[k].append(rej) + for f in fits: + mod, meth = f + r = rec[f] + try: + res = spatial_regression("y ~ x", frame, w, model=mod, method=meth) + except Exception: # noqa: BLE001 - counted and reported + r["fail"] += 1 + continue + tab = res.coefficients.set_index("term") + sp = "rho" if mod == "lag" else "lambda" + sp_truth = rho if (mod == "lag" and kind in ("sar", "sdm", "juhl")) else \ + p.get("lam") if (mod == "error" and kind == "sem") else None + r["beta"].append(float(tab.loc["x", "estimate"])) + r["beta_cov"].append(bool(tab.loc["x", "ci_lower"] <= cfg.beta <= tab.loc["x", "ci_upper"])) + r["sp"].append(float(tab.loc[sp, "estimate"])) + if sp_truth is not None: + r["sp_cov"].append(bool(tab.loc[sp, "ci_lower"] <= sp_truth <= tab.loc[sp, "ci_upper"])) + if mod == "lag": + eff = res.effects.set_index("effect") + lo, hi = _simulated_effect_ci(res, eig, rng_sim) + for i, e in enumerate(("direct", "indirect", "total")): + r[e].append(float(eff.loc[e, "estimate"])) + r[f"{e}_se"].append(float(eff.loc[e, "se_model"])) + r[f"{e}_cov"].append(bool(eff.loc[e, "ci_lower"] <= truth[e] <= eff.loc[e, "ci_upper"])) + r[f"{e}_simcov"].append(bool(lo[i] <= truth[e] <= hi[i])) + else: + b = tab.loc["x"] + for e in ("direct", "total"): + r[e].append(float(b["estimate"])) + r[f"{e}_se"].append(float(b["se_model"])) + r[f"{e}_cov"].append(bool(b["ci_lower"] <= truth[e] <= b["ci_upper"])) + r["indirect"].append(0.0) + scen_label = "ACS-like outcome error" if scen == "acs" else "no sampling error" + for f in fits: + mod, meth = f + r = rec[f] + dflt = meth == DEFAULT_METHODS[mod] + est = f"spatial_regression {mod} {'ML' if meth == 'ml' else 'GMM'}{' [tool default]' if dflt else ''}" + lab = f"{est}; DGP {label}, rook 12x12, {scen_label}" + base = {"dgp": kind, "dgp_label": label, "scenario": scen, "model": mod, "estimator": meth, + "tool_default": dflt, "fit_failures": r["fail"]} + sp = "rho" if mod == "lag" else "lambda" + sp_truth = rho if (mod == "lag" and kind in ("sar", "sdm", "juhl")) else \ + p.get("lam") if (mod == "error" and kind == "sem") else None + pclass = "disturbance" if mod == "error" else "spatial lag (substantive spillover)" + rows.append(_metric_row(lab, f"{sp} ({'no true value in this DGP' if sp_truth is None else sp_truth})", + r["sp"], sp_truth, r["sp_cov"] if sp_truth is not None else None, + {**base, "term": sp, "parameter_class": pclass})) + rows.append(_metric_row(lab, f"beta coefficient ({cfg.beta:g}; the marginal effect only when " + "rho = 0 and theta = 0)", r["beta"], cfg.beta, r["beta_cov"], + {**base, "term": "x", "parameter_class": "coefficient"})) + for e in ("direct", "indirect", "total"): + has_ci = mod == "lag" or e != "indirect" + rows.append(_metric_row(lab, f"{e} effect of x ({truth[e]:.3f})", r[e], truth[e], + r[f"{e}_cov"] if has_ci else None, + {**base, "term": f"x ({e})", "parameter_class": "substantive effect", + "interval": "delta method [tool]" if mod == "lag" else "beta interval"}, + r[f"{e}_se"] if has_ci else None)) + if mod == "lag": + rows.append(_metric_row(lab, f"{e} effect of x ({truth[e]:.3f}), simulated interval", + r[e], truth[e], r[f"{e}_simcov"], + {**base, "term": f"x ({e})", "parameter_class": "substantive effect", + "interval": f"simulated percentile ({SPREG16_EFFECT_DRAWS} draws)"})) + for k, v in rs_rej.items(): + rows.append(_rate(f"OLS-based RS test {k} (spreg; Koley & Bera 2024); DGP {label}, rook 12x12, " + f"{scen_label}", f"rejection rate at {SPREG16_LR_ALPHA:g} (H0: {SPREG16_RS_TESTS[k]})", + v, SPREG16_LR_ALPHA, {"dgp": kind, "dgp_label": label, "scenario": scen, + "term": k, "parameter_class": "specification test"})) + return rows + + +def _rs_rejections(y: np.ndarray, x: np.ndarray, w) -> Dict[str, bool]: + """Rejections at SPREG16_LR_ALPHA of spreg's OLS-based RS tests (SPREG16_RS_TESTS).""" + import spreg + + from .spatial import _silence_spreg_output + + _silence_spreg_output() + + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + ols = spreg.OLS(y[:, None], x[:, None]) + lm = spreg.LMtests(ols, w, tests=["lml", "lmwx", "lmspdurbin", "rlmwx", "rlmdurlag"]) + return {k: float(np.ravel(getattr(lm, k))[1]) < SPREG16_LR_ALPHA for k in SPREG16_RS_TESTS} + + +def _simulated_effect_ci(res, eig: np.ndarray, rng: np.random.Generator) -> Tuple[np.ndarray, np.ndarray]: + """LeSage & Pace (2009) simulated 90% intervals of (direct, indirect, total) for the first covariate. + + Draws (beta, rho) from N(estimate, coefficient_vcov) of the lag fit and + takes percentiles of the effects; draws with |rho| >= 1 give NaN effects + and are dropped. + """ + cv = res.diagnostics["coefficient_vcov"] + est = res.coefficients.set_index("term").loc[cv["terms"], "estimate"].to_numpy(float) + draws = rng.multivariate_normal(est, np.asarray(cv["matrix"], float), size=SPREG16_EFFECT_DRAWS) + b, r = draws[:, 1], draws[:, -1] + ok = np.abs(r) < 1 + b, r = b[ok], r[ok] + direct = np.real(np.mean(1.0 / (1.0 - r[:, None] * eig[None, :]), axis=1)) # census.spatial.lag_multipliers + total = 1.0 / (1.0 - r) + vals = np.column_stack([b * direct, b * (total - direct), b * total]) + q = (1 - NOMINAL) / 2 + return np.quantile(vals, q, axis=0), np.quantile(vals, 1 - q, axis=0) + + +# --------------------------------------------------------------------------- +# A4: common-factor LR test +# --------------------------------------------------------------------------- + + +def lr_residual_bootstrap(y: np.ndarray, X: np.ndarray, Wd: np.ndarray, eig: np.ndarray, lr: float, + sem: Dict[str, Any], rng: np.random.Generator, n_boot: int = SPREG16_LR_BOOT) -> bool: + """Juhl (2021, sec. 3.3) residual bootstrap of the common-factor test: True if H0 is rejected. + + Fit the SEM (``sem``: sem_ml of the data); its innovations + e = (I - lambda W)(y - X beta), centred, are resampled with replacement + into n_boot samples y_b = X beta + (I - lambda W)^-1 e_b generated under + the SEM; the statistic is recomputed on each and H0 rejected when the + observed LR exceeds the (1 - alpha) quantile of the bootstrap LRs. + """ + n = len(y) + B = np.eye(n) - sem["lambda"] * Wd + Binv = sla.inv(B) + mu = X @ sem["beta"] + e = B @ (y - mu) + e = e - e.mean() + stats_b = np.array([common_factor_lr(mu + Binv @ rng.choice(e, n, replace=True), X, Wd, eig)[0] + for _ in range(n_boot)]) + return bool(lr > np.quantile(stats_b, 1 - SPREG16_LR_ALPHA)) + + +def _me_bootstrap(y: np.ndarray, X: np.ndarray, Wd: np.ndarray, eig: np.ndarray, lr: float, s: np.ndarray, + rng: np.random.Generator, n_boot: int = SPREG16_LR_BOOT) -> bool: + """ADAPTATION (not in Juhl): parametric bootstrap under the SEM plus known independent sampling error. + + The null model is the spatial Fay-Herriot fit (me_sem_fit, REML, diagonal + s^2); bootstrap samples are X beta + (I - lambda W)^-1 N(0, A I) + N(0, s^2). + """ + n = len(y) + f = me_sem_fit(y, X[:, 1:], Wd, np.diag(s ** 2)) + Binv = sar_inverse(Wd, f["lambda"]) + mu = X @ f["beta"] + sd = math.sqrt(f["A"]) + stats_b = np.array([common_factor_lr(mu + Binv @ (sd * rng.standard_normal(n)) + s * rng.standard_normal(n), + X, Wd, eig)[0] for _ in range(n_boot)]) + return bool(lr > np.quantile(stats_b, 1 - SPREG16_LR_ALPHA)) + + +def _lr_rows(label: str, scen_label: str, h0: bool, base: Dict[str, Any], tests: Dict[str, List[bool]] + ) -> List[Dict[str, Any]]: + what = "size (rejection rate, H0 true)" if h0 else "power (rejection rate, H0 false)" + names = {"chi2": "chi-square(1) critical value", + "bootstrap_residual": f"Juhl residual bootstrap (B={SPREG16_LR_BOOT})", + "bootstrap_me": f"ME-aware parametric bootstrap (B={SPREG16_LR_BOOT}; adaptation)"} + return [_rate(f"common-factor LR, {names[k]}; {label}, {scen_label}", f"{what} at {SPREG16_LR_ALPHA:g}", + v, SPREG16_LR_ALPHA, {**base, "critical": k}) + for k, v in tests.items() if v] + + +def study_common_factor_lr(cfg: SimConfig, reps: int) -> List[Dict[str, Any]]: + """A4: size and power of the LR common-factor test in Juhl's (2021) design, no sampling error. + + y = x beta + (I - rho W)^-1 (x gamma + v), beta = 2, x, v ~ N(0, 1), + row-standardized queen lattices of side 7, 10, 15, 20 (n 49-400), + rho 0-0.8, gamma 0 (the SEM: H0 theta = -rho beta true) and 0.5, 1 (an + SDM with beta + gamma and theta = -rho beta: H0 false). LR = 2 (logL_SDM + - logL_SEM) by ML with the full Jacobian (sem_ml / sdm_ml, the + likelihoods of spreg ML_Error and ML_Lag with slx_lags=1). Critical + values: chi-square(1); on the first SPREG16_LR_BOOT_REPS repetitions of + the gamma cells in SPREG16_JUHL_BOOT_GAMMAS also Juhl's residual + bootstrap. Each (side, rho, gamma) cell has its own seed. + """ + crit = float(sps.chi2.ppf(1 - SPREG16_LR_ALPHA, 1)) + rows: List[Dict[str, Any]] = [] + for zi, side in enumerate(SPREG16_JUHL_SIDES): + w = lattice_w(side, "queen") + W = w.full()[0] + n = w.n + eig = np.linalg.eigvals(W) + for ri, rho in enumerate(SPREG16_JUHL_RHOS): + S = sar_inverse(W, rho) + for gi, gam in enumerate(SPREG16_JUHL_GAMMAS_LR): + rng = np.random.default_rng(cfg.seed + 3000 + 100 * zi + 10 * ri + gi) + tests: Dict[str, List[bool]] = {"chi2": [], "bootstrap_residual": []} + lrs, fail = [], 0 + fitted: Dict[str, List[float]] = {"sem_lambda": [], "sem_beta": [], "sdm_rho": [], "sdm_beta": [], + "sdm_theta": []} + for rep in range(reps): + x = rng.standard_normal(n) + y = SPREG16_JUHL_BETA * x + S @ (gam * x + rng.standard_normal(n)) + X = np.column_stack([np.ones(n), x]) + try: + lr, sdm, sem = common_factor_lr(y, X, W, eig) + for k, v in (("sem_lambda", sem["lambda"]), ("sem_beta", sem["beta"][1]), + ("sdm_rho", sdm["rho"]), ("sdm_beta", sdm["coef"][1]), + ("sdm_theta", sdm["coef"][2])): + fitted[k].append(float(v)) + boot = (lr_residual_bootstrap(y, X, W, eig, lr, sem, rng) + if gam in SPREG16_JUHL_BOOT_GAMMAS and rep < SPREG16_LR_BOOT_REPS else None) + except Exception: # noqa: BLE001 - counted and reported + fail += 1 + continue + lrs.append(lr) + tests["chi2"].append(lr > crit) + if boot is not None: + tests["bootstrap_residual"].append(boot) + label = f"Juhl DGP, queen {side}x{side} (n={n}), rho={rho:g}, gamma={gam:g}" + base = {"design": "juhl", "n_areas": n, "rho": rho, "gamma": gam, "scenario": "none", + "h0_true": gam == 0, "fit_failures": fail, "lr_df": 1, + "lr_mean": float(np.mean(lrs)) if lrs else float("nan"), + # mean ML estimates: SEM lambda and beta vs the truth (rho, beta = 2) show how an + # SEM misreads an SDM-like DGP (Lacombe & LeSage 2015); SDM: beta + gamma, -rho beta + **{f"{k}_mean": float(np.mean(v)) if v else float("nan") for k, v in fitted.items()}} + rows += _lr_rows(label, "no sampling error", gam == 0, base, tests) + return rows + + +def study_common_factor_lr_acs(cfg: SimConfig, reps: int) -> List[Dict[str, Any]]: + """A4 extension: the common-factor LR test with ACS-like outcome sampling error. + + Queen 12 x 12 lattice on the scale of the other studies: y* = 10 + beta x + + (I - rho W)^-1 (gamma (x - x_mean) + e), beta = cfg.beta, x ~ N(x_mean, + x_sd^2), e ~ N(0, sigma_u^2); gamma 0 (SEM, H0 true) or + SPREG16_ACS_LR_GAMMAS[1] (H0 false); rho in SPREG16_ACS_LR_RHOS. The + published y adds independent error with tract-profile CVs (scenario + 'acs') or none. Under 'acs' the SEM null is itself misspecified (the + data follow an SEM plus independent noise). Critical values: chi-square(1), + Juhl's residual bootstrap and, under 'acs', a parametric bootstrap from + the spatial Fay-Herriot fit with the known SEs (an adaptation), both on + the first SPREG16_LR_BOOT_REPS repetitions. + """ + side = SPREG16_SIDE + w = lattice_w(side, "queen") + W = w.full()[0] + m = w.n + eig = np.linalg.eigvals(W) + crit = float(sps.chi2.ppf(1 - SPREG16_LR_ALPHA, 1)) + mean_y = 10.0 + cfg.beta * cfg.x_mean + rows: List[Dict[str, Any]] = [] + for ri, rho in enumerate(SPREG16_ACS_LR_RHOS): + S = sar_inverse(W, rho) + for gi, gam in enumerate(SPREG16_ACS_LR_GAMMAS): + for si, scen in enumerate(("none", "acs")): + rng = np.random.default_rng(cfg.seed + 3600 + 100 * ri + 10 * gi + si) + tests: Dict[str, List[bool]] = {"chi2": [], "bootstrap_residual": [], "bootstrap_me": []} + lrs, fail = [], 0 + for rep in range(reps): + x = cfg.x_mean + cfg.x_sd * rng.standard_normal(m) + ystar = 10.0 + cfg.beta * x + S @ (gam * (x - cfg.x_mean) + cfg.sigma_u * rng.standard_normal(m)) + s = draw_cv(m, "tract", rng) / 100 * mean_y if scen == "acs" else np.zeros(m) + y = ystar + s * rng.standard_normal(m) + X = np.column_stack([np.ones(m), x]) + try: + lr, _, sem = common_factor_lr(y, X, W, eig) + boot = rep < SPREG16_LR_BOOT_REPS + b_res = lr_residual_bootstrap(y, X, W, eig, lr, sem, rng) if boot else None + b_me = _me_bootstrap(y, X, W, eig, lr, s, rng) if boot and scen == "acs" else None + except Exception: # noqa: BLE001 - counted and reported + fail += 1 + continue + lrs.append(lr) + tests["chi2"].append(lr > crit) + if b_res is not None: + tests["bootstrap_residual"].append(b_res) + if b_me is not None: + tests["bootstrap_me"].append(b_me) + label = f"queen {side}x{side}, rho={rho:g}, gamma={gam:g}" + base = {"design": "acs_extension", "n_areas": m, "rho": rho, "gamma": gam, "scenario": scen, + "h0_true": gam == 0, "fit_failures": fail, "lr_df": 1, + "lr_mean": float(np.mean(lrs)) if lrs else float("nan")} + rows += _lr_rows(label, "ACS-like outcome error" if scen == "acs" else "no sampling error", + gam == 0, base, tests) + return rows + + +SPREG_VALIDATION_STUDIES: Dict[str, Callable[[SimConfig, int], List[Dict[str, Any]]]] = { + "spatial_error_attenuation": study_spatial_error_attenuation, + "spatial_error_attenuation_correlated": study_spatial_error_attenuation_correlated, + "spatial_dgp_recovery": study_spatial_dgp_recovery, + "common_factor_lr": study_common_factor_lr, + "common_factor_lr_acs": study_common_factor_lr_acs, +} + + +def spreg_validation_config(fast: bool = True, **overrides: Any) -> SimConfig: + """Config that runs only the Round 16 workstream A studies.""" + base = FAST if fast else FULL + reps = {k: 0 for k in base.reps} + reps.update(SPREG16_REPS_FAST if fast else SPREG16_REPS_FULL) + return replace(base, reps=reps, **overrides) diff --git a/census/spatial.py b/census/spatial.py index c4fffdc..74ab533 100644 --- a/census/spatial.py +++ b/census/spatial.py @@ -21,7 +21,8 @@ Default: two-sided p-values (2 x esda's folded p_sim) from 9999 permutations. Simulation ('local_morans_size'): per-area and family-wise rates at nominal under randomness; little power on ~150 areas; spatially - patterned sampling error creates spurious clusters. + heterogeneous (independent) sampling error gives spurious significant + areas (Round 16 docs/audits/lisa_validation.md: correlated errors). * :func:`spatial_regression` - spatial lag / spatial error models by ML or GMM (spreg), returned as :class:`census.results.ModelResult` with the OLS Lagrange-multiplier diagnostics that suggest which model fits, and replicate @@ -29,7 +30,9 @@ ML for the error model. Simulation ('spatial_regression_coverage'): under ACS outcome error ML lag intervals for rho fail (GMM lag holds), lambda is attenuated by both estimators, beta intervals hold; LM tests keep their - 0.05 size. + 0.05 size. Option ``correct_outcome_error=True`` (error model, Round 16 + A1) corrects lambda for independent outcome sampling error with the + spatial Fay-Herriot likelihood (census.sae_spatial; known SE^2). References: Moran (1950) Biometrika 37:17-23; Anselin (1995) Geographical Analysis 27:93-115; Anselin, Bera, Florax & Yoon (1996) Regional Science and @@ -50,7 +53,7 @@ import pandas as pd from scipy import sparse, stats as sps -from .results import (SOURCE_MODEL, SOURCE_SAMPLING_REPLICATE, ModelResult, _jsonable, check_refit_count, +from .results import (SOURCE_MODEL, SOURCE_SAMPLING_MODEL, SOURCE_SAMPLING_REPLICATE, ModelResult, _jsonable, check_refit_count, check_replicate_policy, precheck_replicates, replicate_completeness, stochastic_variables) from .variance import REPLICATE_COLUMNS, REPLICATE_COUNT, REPLICATE_FACTOR @@ -84,7 +87,80 @@ "Caveat (simulation 'spatial_regression_coverage'): with ACS sampling error in the outcome, lambda describes " "the spatial correlation of the observed residual, sampling noise included, and understates the true lambda " "(bias -0.16 at 0.3, -0.26 at 0.6; 90% intervals covered 0.56-0.58 and 0.16-0.19, ML and GMM alike); " - "beta intervals covered 0.88-0.91") + "beta intervals covered 0.88-0.91. To correct lambda for independent outcome sampling error, refit with " + "correct_outcome_error=True and se_col= (REML lambda intervals covered 0.86-0.93; " + "docs/audits/spatial_regression_validation.md A1)") +# Error model corrected for outcome sampling error (PLAN_ROUND_16 A1, owner-approved option +# correct_outcome_error=True). Numbers: docs/audits/spatial_regression_validation.md (A1, studies +# 'spatial_error_attenuation' and 'spatial_error_attenuation_correlated', 12x12 lattices, nominal 0.90) and +# docs/audits/acs_sampling_covariance.md (workstream C, VRE replicates). +OUTCOME_ERROR_CORRECTION_NOTE = ( + "lambda corrected for outcome sampling error: spatial error model with the outcome's known sampling " + "variances, V = sigma^2 [(I - lambda W)'(I - lambda W)]^-1 + diag(SE^2) (the spatial Fay-Herriot " + "likelihood of census.sae_spatial; beta by GLS). Assumption: sampling errors are independent across areas " + "and their variances are known (= SE^2 from se_col). Evidence: docs/audits/acs_sampling_covariance.md found " + "neighbour sampling-error correlation of about 0 (|r| <= 0.003 among tracts) for rates, shares, medians and " + "most counts; docs/audits/spatial_regression_validation.md A1 (simulation 'spatial_error_attenuation', " + "independent errors, CV x 0.5-2, rook/queen/kNN; the shipped estimator itself): REML lambda intervals " + "covered 0.86-0.93 at lambda 0.3 and 0.6 (uncorrected ML: 0.003-0.85) and 0.83-0.89 at lambda 0; beta " + "intervals 0.85-0.93") +LAMBDA_MEANING_NOTE = ( + "lambda is the spatial autocorrelation of the disturbances (the diffusion of shocks through unmodelled, " + "spatially clustered factors; Lacombe & LeSage 2015), not a spillover of y or x between areas; beta is the " + "direct = total effect only if there is no substantive spillover") +CORRELATED_ERROR_CAVEAT = ( + "Caveat (simulation 'spatial_error_attenuation_correlated'): positively correlated sampling errors between " + "neighbours bias the correction upward, because the correlation is read as spatial dependence of the " + "disturbances. With SAR(0.6)-correlated errors (mean neighbour error correlation 0.29-0.38) and true lambda " + "0 the corrected lambda averaged +0.19 to +0.22 and its 90% intervals covered 0.59-0.66 (0.65-0.74 at " + "lambda 0.3); with SAR(0.3) errors (0.11-0.17) 0.80-0.83 at lambda 0. ACS VRE replicates showed no positive " + "neighbour correlation (docs/audits/acs_sampling_covariance.md)") +CONTROLLED_TOTAL_CAVEAT = ( + "WARNING: the outcome looks like a controlled total (population, household or housing-unit count). ACS " + "weighting controls correlate the sampling errors of such totals among neighbouring tracts in the same " + "county, and the sign can differ by release (2020-2024: first-order r -0.08 to -0.04 for population, " + "-0.03 to -0.015 for households, tract population errors summing to about 0 within a county; 2015-2019: " + "households +0.01 to +0.03; docs/audits/acs_sampling_covariance.md), so the independence assumption fails. " + "Negative error correlation would likely pull the corrected lambda down and positive correlation push it up; " + "the size of that bias was not simulated. Prefer a rate or share outcome, or report the corrected lambda as " + "a sensitivity analysis") +# Outcome columns treated as controlled totals: ACS cells of total population, households and housing units +# (C: B01003_001 and B25003_001 measured; the others are the same totals in other tables). +CONTROLLED_TOTAL_CELLS = ("B01003_001", "B01001_001", "B11001_001", "B25001_001", "B25002_001", "B25003_001") +_CONTROLLED_TOTAL_NAMES = ("population", "total_population", "pop", "total_pop", "households", + "total_households", "housing_units", "total_housing_units", "occupied_housing_units", + "occupied_units") +CORRECTION_METHODS = ("reml", "ml") +CORRECTION_LAMBDA_BOUNDARY = 0.98 # |lambda| at or above: boundary fit (the A1 studies' definition) +SIGMA2_ZERO_REL = 1e-8 # sigma^2 at or below this share of Var(y): boundary (as the A1 studies) +# correct_outcome_error=True works on dense n x n matrices (O(n^3) per likelihood evaluation); larger frames +# are refused. Measured (rook lattice, one BLAS thread, Python 3.14, scratch benchmark of the whole +# spatial_regression call): see CORRECTION_COST_NOTE. 2,000 keeps one fit near 4 minutes and under 1 GB +# (an MCP call stays interactive); time grows about n^2.6, so 3,000 areas would take about 13 minutes. +CORRECTION_MAX_N = 2000 +CORRECTION_COST_NOTE = ("measured for a full call, one BLAS thread: n=1024 40 s / 0.5 GB peak, n=2025 3.9 min / " + "0.9 GB, n=3025 12.8 min / 1.7 GB") +LAG_EFFECTS_NOTE = ( + "Lag model: the coefficient beta of a covariate is NOT its marginal effect. A change in x in one area " + "changes y there and, through the spatial multiplier (I - rho W)^-1, in every other area. Report the " + "effects table (result.effects): direct = average effect of an area's own x on its own y (feedback " + "included), indirect = average spillover summed over all other areas, total = direct + indirect " + "(LeSage & Pace 2009); delta-method SEs from the fit's (beta, rho) covariance, plus replicate refits " + "when replicates are given") +SPATIAL_PARAMETER_TEST_NOTE = ( + "The p-value of {sp} tests {sp} = 0 within the fitted model; it is not a specification test (it does not " + "choose between the lag, error and Durbin models; Juhl 2021, Political Analysis 29(2), sec. 3.2). " + "lm_tests compare each spatial model with OLS only") +LAG_EFFECTS_CAVEAT = ( + "Caveat (simulation 'spatial_dgp_recovery', 12x12 rook lattice, nominal 0.90): with ACS outcome error the " + "default GMM lag's delta-method effect intervals covered 0.87-0.92 at rho 0.3 but 0.80-0.87 at rho 0.6, " + "where the indirect and total effects are heavy-tailed (rho near 1 in some samples; mean bias +0.8, " + "simulated percentile intervals covered 0.97-0.98). ML lag effects fail under outcome error (indirect " + "0.40 and 0.002), as its rho does. If the data follow a spatial Durbin process (W X effects), the lag " + "model's effects are biased, and GMM's W X instruments are invalid (rho near 1)") +# Effects need the eigenvalues of the dense W (O(n^3), once per fit); above +# this many areas they are not computed (a note says so). +EFFECTS_MAX_N = 5000 NOISY_COVARIATE_CAVEAT = ( "Covariates with sampling error attenuate beta (simulation: bias about -20%, 90% interval coverage " "0.19-0.40); spatial_regression has no measurement-error correction") @@ -93,8 +169,10 @@ "min(1, 2 p_sim), 9999 permutations, BH at alpha 0.10) kept the per-area rate of p_value <= 0.10 at " "0.099 and the family-wise rate (= FDR) at 0.088 under spatial randomness. It has little power: 1.1% of " "areas in a 4x4 hot spot raised by 1.5 SD were flagged, so no significant cluster is not evidence of no " - "clustering. Spatially patterned sampling error (CVs varying smoothly across the map) produced spurious " - "clusters in 38% of maps; check replicate_quadrant_share or rerun on Fay-Herriot estimates. p_sim is " + "clustering. Spatially heterogeneous but independent sampling errors (CVs varying smoothly across the map) " + "gave at least one significant area in 38% of maps (0.5% of areas); that study did not test spatially " + "correlated sampling errors (see docs/audits/lisa_validation.md). replicate_quadrant_share is a stability " + "check, not a probability that a cluster is real. p_sim is " "esda's folded (one-sided, observed-direction) pseudo p-value; sided='folded' with permutations=999 is " "the pre-Round 15 default (per-area rate of p_sim <= 0.10: 0.196)") _CACHE: Dict[str, Any] = {} @@ -443,8 +521,9 @@ def local_morans( per-area rate of p_value <= 0.10 and the family-wise rate are at nominal (the folded 999-permutation reading used before: per-area 0.196, and a family-wise rate of 0.056 only because of its coarse p-value grid). - Power is low on ~150 areas, and spatially patterned sampling error - produces spurious clusters. About 0.6 s for 3,000 areas. + Power is low on ~150 areas, and spatially heterogeneous (independent) + sampling error produces spurious significant areas. About 0.6 s for + 3,000 areas. """ from esda.moran import Moran_Local from .multitest import adjust_pvalues @@ -534,6 +613,10 @@ def _spreg_fit(y, X, w, model: str, method: str, names: List[str], yname: str, m import spreg _silence_spreg_output() + if model == "lag": + # spreg's own impacts (default 'simple') are not used: the effects table comes from lag_effects with the + # cached eigenvalues; computing them on the main fit and every replicate refit is wasted work + kwargs = {"spat_impacts": None, **kwargs} with warnings.catch_warnings(): warnings.simplefilter("ignore") if model == "lag" and method == "ml": @@ -559,6 +642,129 @@ def _spreg_fit(y, X, w, model: str, method: str, names: List[str], yname: str, m return r, betas, se[:k], spatial +def _eigenvalues_dense(W: np.ndarray) -> np.ndarray: + """Eigenvalues of a dense W. A row-standardized W with a symmetric pattern (W = D^-1 C, C = C', as for + contiguity and distance-band weights) is similar to the symmetric D^-1/2 C D^-1/2, whose eigenvalues + eigvalsh returns (real, about 3-10x faster than eigvals); other W (kNN, general weights) use eigvals.""" + d = np.count_nonzero(W, axis=1).astype(float) + C = W * d[:, None] + if np.allclose(C, C.T, rtol=1e-12, atol=1e-12): + r = np.zeros_like(d) + np.sqrt(d, out=r, where=d > 0) + inv = np.divide(1.0, r, out=np.zeros_like(r), where=r > 0) + S = inv[:, None] * C * inv[None, :] + return np.linalg.eigvalsh(0.5 * (S + S.T)) + return np.linalg.eigvals(W) + + +def weights_eigenvalues(w) -> Optional[np.ndarray]: + """Eigenvalues of the (row-standardized) dense W, or None above EFFECTS_MAX_N areas. + + Computed once per fit and reused by every replicate refit: the effect + multipliers at any rho are sums over them (Ord 1975). Cached on the + weights object, keyed by a hash of its current sparse matrix, so repeated + fits on the same weights do not repeat the O(n^3) decomposition. + """ + if w.n > EFFECTS_MAX_N: + return None + sp = w.sparse.tocsr() + h = hashlib.sha1() + for a in (sp.indptr, sp.indices, sp.data): + h.update(np.ascontiguousarray(a).tobytes()) + key = (sp.shape, h.hexdigest()) + cached = getattr(w, "_census_eigenvalues", None) + if cached is not None and cached[0] == key: + return cached[1] + eig = _eigenvalues_dense(np.asarray(w.full()[0], float)) + try: + w._census_eigenvalues = (key, eig) + except AttributeError: # weights objects without attribute assignment: no cache + pass + return eig + + +def lag_multipliers(eig: np.ndarray, rho: float) -> Dict[str, float]: + """Average impact multipliers of the spatial lag model at ``rho`` and their rho-derivatives. + + S(W) = (I - rho W)^-1. direct = tr(S)/n = mean 1/(1 - rho w_i) over the + eigenvalues w_i of W; total = 1' S 1 / n = 1/(1 - rho) for a + row-standardized W without islands (islands: :func:`_lag_effect_values`); + indirect = total - direct (LeSage & Pace 2009, ch. 2.7). NaN outside + |rho| < 1. + """ + nan = {k: float("nan") for k in ("direct", "total", "indirect", "d_direct", "d_total", "d_indirect")} + if not (np.isfinite(rho) and abs(rho) < 1): + return nan + e = np.asarray(eig) + den = 1.0 - rho * e + direct = float(np.real(np.mean(1.0 / den))) + d_direct = float(np.real(np.mean(e / den ** 2))) + total, d_total = 1.0 / (1.0 - rho), 1.0 / (1.0 - rho) ** 2 + return {"direct": direct, "total": total, "indirect": total - direct, + "d_direct": d_direct, "d_total": d_total, "d_indirect": d_total - d_direct} + + +def effect_multipliers(eig: np.ndarray, rho: float, share: float = 1.0) -> Dict[str, float]: + """Average direct / indirect / total multipliers at ``rho`` with islands, and their rho-derivatives. + + As :func:`lag_multipliers`, but total = (1 - s) + s / (1 - rho) where + ``share`` s is the share of areas with neighbours (an island's row of + S = (I - rho W)^-1 is e_i); indirect = total - direct. The multipliers + behind the effects table and ``diagnostics['effects_multipliers']``. + NaN outside |rho| < 1. + """ + m = lag_multipliers(eig, rho) + s = float(share) + total, d_total = (1 - s) + s * m["total"], s * m["d_total"] + return {"direct": m["direct"], "total": total, "indirect": total - m["direct"], + "d_direct": m["d_direct"], "d_total": d_total, "d_indirect": d_total - m["d_direct"]} + + +def _lag_effect_values(params: np.ndarray, eig: np.ndarray, n_x: int, share: np.ndarray) -> np.ndarray: + """Effects [x1 direct, x1 indirect, x1 total, x2 ...] from [const, x1..xk, rho]. + + ``share``: the mean row sum of W (1 for row-standardized W without + islands); with islands, total = beta (1 - s + s / (1 - rho)) where s is + the share of areas with neighbors, since an island's row of S is e_i. + """ + m = effect_multipliers(eig, float(params[-1]), float(share)) + out = [] + for j in range(n_x): + b = float(params[1 + j]) + out += [b * m["direct"], b * m["indirect"], b * m["total"]] + return np.asarray(out, float) + + +def lag_effects(params: np.ndarray, vm: Optional[np.ndarray], names: List[str], eig: np.ndarray, + share: float = 1.0) -> Dict[str, Any]: + """Direct, indirect and total effects of a spatial lag fit with delta-method SEs. + + ``params``: [const, x1..xk, rho] (spreg order); ``vm``: their covariance + (spreg ``vm``; for GM_Lag and ML_Lag rho is the last row). For covariate + k the effects are beta_k times the multipliers of :func:`lag_multipliers`; + the gradient is (multiplier) for beta_k and beta_k x d(multiplier)/d rho + for rho, so Var = g' V[(k, rho)] g. Returns {"terms", "estimate", "se"}. + """ + params = np.asarray(params, float).ravel() + n_x = len(names) + est = _lag_effect_values(params, eig, n_x, np.asarray(share)) + se = np.full(len(est), np.nan) + m = effect_multipliers(eig, float(params[-1]), float(share)) + if vm is not None and np.isfinite(m["direct"]): + V = np.asarray(vm, float) + mult = {e: (m[e], m[f"d_{e}"]) for e in ("direct", "indirect", "total")} + r = V.shape[0] - 1 + for j in range(n_x): + b, k = float(params[1 + j]), 1 + j + sub = V[np.ix_([k, r], [k, r])] + for i, eff in enumerate(("direct", "indirect", "total")): + mu, dmu = mult[eff] + g = np.array([mu, b * dmu]) + se[3 * j + i] = math.sqrt(max(float(g @ sub @ g), 0.0)) + terms = [f"{v} ({eff})" for v in names for eff in ("direct", "indirect", "total")] + return {"terms": terms, "estimate": est, "se": se} + + def lm_diagnostics(y: np.ndarray, X: np.ndarray, w) -> Dict[str, Any]: """OLS Lagrange-multiplier tests (Anselin et al. 1996) and a model suggestion.""" import spreg @@ -587,6 +793,189 @@ def lm_diagnostics(y: np.ndarray, X: np.ndarray, w) -> Dict[str, Any]: return out +def controlled_total_reason(outcome: str) -> Optional[str]: + """Why ``outcome`` looks like an ACS controlled total (population / household / housing-unit count), or None. + + Matches the ACS cells in :data:`CONTROLLED_TOTAL_CELLS` (with or without the + ``E`` suffix) and plain names such as ``population`` or ``households``. + """ + name = str(outcome).strip() + code = name.upper()[:-1] if name.upper().endswith("E") else name.upper() + if code in CONTROLLED_TOTAL_CELLS: + return f"ACS cell {code} is a total controlled to the population / housing-unit estimates" + if name.lower() in _CONTROLLED_TOTAL_NAMES: + return f"the name {name!r} suggests a population / household / housing-unit total" + return None + + +def error_model_known_variances(y: np.ndarray, X: np.ndarray, W: np.ndarray, D: np.ndarray, + method: str = "reml") -> Dict[str, Any]: + """Spatial error model with known independent outcome sampling variances ``D`` (census.sae_spatial). + + y = X beta + u + e, u = lambda W u + eps, eps ~ N(0, sigma^2 I), e ~ N(0, diag(D)): + V = sigma^2 [(I - lambda W)'(I - lambda W)]^-1 + diag(D), the spatial + Fay-Herriot likelihood with rho = lambda (Petrucci & Salvati 2006; + Molina & Marhuenda 2015 eq. (6)). (sigma^2, lambda) by REML or ML + (census.sae_spatial.estimate_variance_rho), beta by GLS with covariance + (X'V^-1X)^-1, SEs of (sigma^2, lambda) from the inverse expected + information 0.5 tr(M V_k M V_l), M = P (REML) or V^-1 (ML). + + The ONE estimator behind ``spatial_regression(correct_outcome_error=True)`` + and the A1 studies' diagonal-Sigma rows (census.simulation_spreg + 'spatial_error_attenuation' and '_correlated' call this function), so the + validated coverage describes the shipped path. ``X`` includes the + constant; ``W`` is dense. Returns ``boundary`` (|lambda| >= + CORRECTION_LAMBDA_BOUNDARY or sigma^2 at 0) with ``sigma2_zero``. Raises + RuntimeError when the optimizer does not converge or V is singular at the + optimum (no fallback to the uncorrected estimate). + """ + from scipy import linalg as sla + from .sae_spatial import _fisher, _fit, _loglik + + meth = method.upper() + try: + fit = _fit(y, X, D, W, meth) + except (np.linalg.LinAlgError, sla.LinAlgError) as exc: + raise RuntimeError( + f"correct_outcome_error: the {meth} fit is singular at the optimum (V = sigma^2 C(lambda) + " + f"diag(SE^2) cannot be factorized: {exc}); no corrected estimate is reported and the uncorrected fit " + "is not substituted. This happens when sigma^2 is estimated at 0 and some SEs are 0 or tiny; check " + "the SE column (units must match the outcome)") from exc + if not fit["converged"]: + raise RuntimeError( + f"correct_outcome_error: the {meth} optimizer for (sigma^2, lambda) did not converge " + f"(stopped at lambda={fit['rho']:.4f}, sigma^2={fit['A']:.4g}); no corrected estimate is reported and " + "the uncorrected fit is not substituted. Check the SE column (units must match the outcome), the " + "weights, and whether the sampling variances exceed the outcome's variance") + s = fit["state"] + A, lam = float(fit["A"]), float(fit["rho"]) + Fi = np.full((2, 2), np.nan) + # at the sigma^2 = 0 boundary (relative to the outcome's variance) lambda is not identified: no SE + sigma2_zero = A <= SIGMA2_ZERO_REL * max(float(np.var(y, ddof=1)), 1e-300) + if not sigma2_zero: + try: + with warnings.catch_warnings(): + warnings.simplefilter("ignore", sla.LinAlgWarning) + Fi = sla.inv(_fisher(s, meth)) + except (np.linalg.LinAlgError, sla.LinAlgError, ValueError): + pass + ll, _ = _loglik(np.array([A, lam]), y, X, D, W, "ML") + return {"A": A, "lambda": lam, "beta": np.asarray(fit["beta"], float), "vcov_beta": np.asarray(fit["Q"], float), + "se_A": float(math.sqrt(Fi[0, 0])) if np.isfinite(Fi[0, 0]) and Fi[0, 0] > 0 else float("nan"), + "se_lambda": float(math.sqrt(Fi[1, 1])) if np.isfinite(Fi[1, 1]) and Fi[1, 1] > 0 else float("nan"), + "loglik": float(ll), "converged": True, "sigma2_zero": bool(sigma2_zero), + "boundary": bool(sigma2_zero or abs(lam) >= CORRECTION_LAMBDA_BOUNDARY)} + + +def _corrected_error_regression(formula: str, d: pd.DataFrame, weights, *, outcome: str, se_col: Optional[str], + method: Optional[str], target: str, level: float, controlled_total: Optional[bool], + ml_method: str, spreg_kwargs: Dict[str, Any]) -> ModelResult: + """spatial_regression(model='error', correct_outcome_error=True): see :func:`spatial_regression`.""" + from .models import _table + + if se_col is None: + raise ValueError("correct_outcome_error=True needs se_col: the column of the outcome's sampling SEs") + if outcome not in d.columns: + raise ValueError(f"correct_outcome_error=True needs a plain outcome column on the left of the formula " + f"(got {outcome!r}); se_col must be that column's SE") + if se_col not in d.columns: + raise KeyError(f"se_col {se_col!r} not in frame") + corr_method = "reml" if method is None else method.lower() + if corr_method not in CORRECTION_METHODS: + raise ValueError(f"with correct_outcome_error=True, method must be one of {CORRECTION_METHODS} " + f"(variance-component estimator; REML is the validated default), not {method!r}") + if len(d) > CORRECTION_MAX_N: + raise ValueError( + f"correct_outcome_error=True is limited to {CORRECTION_MAX_N} areas (got {len(d)}): the known-variance " + f"fit works on dense n x n matrices (O(n^3) time per likelihood evaluation, O(n^2) memory; " + f"{CORRECTION_COST_NOTE}). Fit at county level, or subset the frame to a region (e.g. one or a few " + "states or metro areas) and rebuild the weights for it") + se = pd.to_numeric(d[se_col], errors="coerce").to_numpy(float) + if not np.isfinite(se).all() or (se < 0).any(): + bad = list(np.asarray(d.index)[~(np.isfinite(se) & (se >= 0))][:20]) + raise ValueError(f"se_col {se_col!r} must be finite and >= 0 in every area: {bad}") + if (se == 0).any(): + zero = list(np.asarray(d.index)[se == 0][:20]) + raise ValueError( + f"se_col {se_col!r} is 0 in {int((se == 0).sum())} area(s): {zero}. The known-variance model would treat " + "those estimates as exact (V is singular when sigma^2 is estimated at 0, and an exact area can force a " + "degenerate boundary fit). An ACS SE of 0 usually marks a controlled or suppressed cell: drop those " + "areas (subset the frame and rebuild the weights) or give them their published nonzero SE") + y2, X, names, yname = _design(formula, d) + y = y2[:, 0] + n = len(y) + if n <= X.shape[1] + 3: + raise ValueError(f"need more areas ({n}) than parameters ({X.shape[1] + 3})") + W = np.asarray(weights.full()[0], float) # row-standardized, in w.id_order (= d's row order) + D = se ** 2 + Xc = np.column_stack([np.ones(n), X]) + fit = error_model_known_variances(y, Xc, W, D, corr_method) + terms = ["Intercept"] + names + ["lambda"] + params = pd.Series(list(fit["beta"]) + [fit["lambda"]], index=terms) + se_model = pd.Series(list(np.sqrt(np.diag(fit["vcov_beta"]))) + [fit["se_lambda"]], index=terms) + table = _table(params, se_model, pd.Series(np.nan, index=terms), level, target) + + reason = controlled_total_reason(outcome) if controlled_total is None else ( + "flagged by the caller (controlled_total=True)" if controlled_total else None) + boundary = [] + if abs(fit["lambda"]) >= CORRECTION_LAMBDA_BOUNDARY: + boundary.append(f"|lambda| >= {CORRECTION_LAMBDA_BOUNDARY:g} (near the unit bound; the SAR process is " + "near-singular)") + var_y = float(np.var(y, ddof=1)) + if fit["sigma2_zero"]: + boundary.append("sigma^2 estimated at 0: the sampling variances absorb all residual variation, lambda is " + "not identified and has no interval") + try: + _, _, _, lam_naive = _spreg_fit(y2, X, weights, "error", "ml", names, yname, ml_method, spreg_kwargs) + except Exception: # noqa: BLE001 - comparison only; reported as unavailable + lam_naive = None + k = Xc.shape[1] + 2 + diagnostics: Dict[str, Any] = { + "lm_tests": lm_diagnostics(y2, X, weights), "interval_target": target, + "converged": True, "boundary": bool(boundary), "controlled_total_warning": reason is not None, + "logll": fit["loglik"], "aic": -2 * fit["loglik"] + 2 * k, "schwarz": -2 * fit["loglik"] + k * math.log(n), + "outcome_error_correction": { + "se_col": se_col, "variance_method": corr_method.upper(), "sigma2": fit["A"], + "sigma2_se": fit["se_A"], "lambda": fit["lambda"], "lambda_se": fit["se_lambda"], + "lambda_uncorrected": lam_naive, + "lambda_uncorrected_method": "spreg ML_Error (the default error model, no correction)", + "mean_sampling_variance_share": float(np.mean(D) / var_y) if var_y > 0 else None, + "assumption": "sampling errors independent across areas; variances known (SE^2)", + "boundary_reasons": boundary, "controlled_total_reason": reason, + "evidence": ["docs/audits/spatial_regression_validation.md (A1)", + "docs/audits/acs_sampling_covariance.md (C)"], + }, + } + notes: List[str] = [] + if reason is not None: + notes.append(f"{CONTROLLED_TOTAL_CAVEAT} ({reason}; pass controlled_total=False if it is not)") + notes += [OUTCOME_ERROR_CORRECTION_NOTE, LAMBDA_MEANING_NOTE, CORRELATED_ERROR_CAVEAT] + if lam_naive is not None: + notes.append(f"uncorrected lambda (spreg ML_Error) = {lam_naive:.4f}; corrected = {fit['lambda']:.4f}") + notes += [f"boundary fit: {b}" for b in boundary] + if not np.isfinite(fit["se_lambda"]): + notes.append("lambda has no SE (sigma^2 at 0, or singular information at the fitted values): no interval " + "is reported") + if weights.islands: + notes.append(f"{len(weights.islands)} island(s) in the weights (zero spatial lag)") + if d.attrs.get("extra_rows_dropped"): + notes.append(f"{len(d.attrs['extra_rows_dropped'])} frame row(s) not in the weights were ignored") + notes.append(f"intervals and p-values use the '{target}' target (normal, from se_model: GLS for beta, the " + f"{corr_method.upper()} information for lambda)") + notes.append(SPATIAL_PARAMETER_TEST_NOTE.format(sp="lambda")) + notes.append(NOISY_COVARIATE_CAVEAT) + return ModelResult( + model_type="spatial_error", formula=formula, n_obs=int(n), coefficients=table, + method=f"spatial error model with known outcome sampling variances (spatial Fay-Herriot likelihood, " + f"{corr_method.upper()})", + uncertainty_sources=[SOURCE_MODEL, SOURCE_SAMPLING_MODEL], confidence_level=level, notes=notes, + diagnostics=diagnostics, + provenance={"library": "census.sae_spatial", "spatial_parameter": "lambda", "ml_method": None, + "outcome_error_correction": True, "se_col": se_col, + "weights": getattr(weights, "census_report", None) or weights_report(weights)}, + ) + + def spatial_regression( formula: str, data: pd.DataFrame, @@ -601,6 +990,9 @@ def spatial_regression( ml_method: str = "full", max_failed_replicates: int = 0, replicate_policy: str = "flag", + correct_outcome_error: bool = False, + se_col: Optional[str] = None, + controlled_total: Optional[bool] = None, **spreg_kwargs: Any, ) -> ModelResult: """Spatial lag or spatial error regression (spreg) as a ModelResult. @@ -631,6 +1023,26 @@ def spatial_regression( (default) records failed refits or stochastic variables without replicates in diagnostics["replicates_incomplete"]; "strict" raises ValueError (census.results.replicate_completeness, PLAN_ROUND_11 Q2/D7). + + ``correct_outcome_error=True`` (model='error' only; PLAN_ROUND_16 A1, + owner-approved option, off by default) needs ``se_col``, the outcome's + sampling SEs, and fits the spatial error model with those known + variances: V = sigma^2 [(I - lambda W)'(I - lambda W)]^-1 + diag(SE^2), + the spatial Fay-Herriot likelihood of census.sae_spatial. ``method`` is + then the variance-component estimator, ``reml`` (default, validated) or + ``ml``. lambda (corrected) has a Wald interval from the information + matrix and beta GLS Wald intervals, the interval methods validated by the + 'spatial_error_attenuation' study (REML lambda coverage 0.86-0.93 at + lambda 0.3-0.6 vs 0.003-0.85 uncorrected). It assumes sampling errors + independent across areas (docs/audits/acs_sampling_covariance.md: about 0 + for rates, shares, medians and most counts); positively correlated errors + bias it upward (notes). Controlled totals (population, households, + housing units: :data:`CONTROLLED_TOTAL_CELLS`, or ``controlled_total=True``) + have negatively correlated errors within counties and get a warning note + and ``diagnostics['controlled_total_warning']``. Non-convergence, or a + singular V at the optimum, raises RuntimeError; SEs of 0 and frames above + :data:`CORRECTION_MAX_N` areas (dense O(n^3) fit) are refused with + ValueError. Replicates are not used with this option. """ from .analysis import formula_variables from .models import INTERVAL_TARGETS, _table @@ -640,6 +1052,25 @@ def spatial_regression( model = model.lower() if model not in SPATIAL_MODELS: raise ValueError(f"model must be one of {SPATIAL_MODELS}") + if correct_outcome_error: + if model != "error": + raise ValueError("correct_outcome_error=True applies to model='error' only (the lag model's default " + "GMM estimator already holds its rho intervals under outcome error)") + if replicates is not None and len(replicates): + raise ValueError("correct_outcome_error=True does not use replicates: the outcome's sampling variance " + "enters the model through se_col; fit without replicates") + if target != "population": + raise ValueError("correct_outcome_error=True supports target='population' only (intervals from the " + "model, which includes the known sampling variances)") + outcome = formula.split("~")[0].strip() + d = _aligned(data, outcome if outcome in data.columns else None, weights, geoid_col) + weights.transform = "r" + return _corrected_error_regression( + formula, d, weights, outcome=outcome, se_col=se_col, method=method, target=target, + level=confidence_level if confidence_level <= 1 else confidence_level / 100, + controlled_total=controlled_total, ml_method=ml_method, spreg_kwargs=spreg_kwargs) + if se_col is not None or controlled_total is not None: + raise ValueError("se_col and controlled_total apply only with correct_outcome_error=True") method = DEFAULT_METHODS[model] if method is None else method.lower() if method not in SPATIAL_METHODS: raise ValueError(f"method must be one of {SPATIAL_METHODS}") @@ -674,6 +1105,16 @@ def spatial_regression( "is 'lambda'") if weights.islands: notes.append(f"{len(weights.islands)} island(s) in the weights (zero spatial lag)") + eig = weights_eigenvalues(weights) if model == "lag" else None + share = 1.0 - len(weights.islands) / weights.n + eff = None + if eig is not None: + k = len(terms) + vm_k = np.asarray(r.vm, float)[:k, :k] + eff = lag_effects(betas, vm_k, names, eig, share) + # for simulated (LeSage & Pace 2009) effect intervals: draws from N(estimates, vcov) + diagnostics["coefficient_vcov"] = {"terms": terms, "matrix": vm_k.tolist()} + B = None if replicates is not None and len(replicates): all_used = formula_variables(formula, d) used = [v for v in all_used if v in set(replicates["variable"])] @@ -685,7 +1126,7 @@ def spatial_regression( # failed refits and variables without replicates can be incomplete stoch = stochastic_variables(data, [v for v in all_used if v not in used]) precheck_replicates(replicate_policy, variables_without_replicates=stoch, context="spatial_regression") - B, failed = [], 0 + B, failed, explosive = [], 0, 0 for k in range(REPLICATE_COUNT): dd = d.copy() for v, m in mats.items(): @@ -695,12 +1136,24 @@ def spatial_regression( _, b, _, _ = _spreg_fit(yr, Xr, weights, model, method, names, yname, ml_method, spreg_kwargs) if not np.isfinite(b).all(): raise ValueError("non-finite") + if model == "lag" and not abs(float(b[-1])) < 1: + # (I - rho W)^-1 is not a stationary multiplier: effects undefined, not a usable refit + explosive += 1 + raise ValueError("|rho| >= 1") B.append(b) except Exception: # noqa: BLE001 - counted and reported failed += 1 + context = "spatial_regression" + if explosive: + context += (f" ({explosive} replicate refit(s) estimated |rho| >= 1, where the spatial multiplier and " + "the effects are undefined; counted as failed refits)") + diagnostics["replicate_refits_rho_out_of_bounds"] = explosive replicate_completeness(replicate_policy, failed_refits=failed, variables_without_replicates=stoch, - context="spatial_regression", diagnostics=diagnostics, notes=notes) - check_refit_count(failed, len(B), max_failed_replicates, "spatial_regression") + context=context, diagnostics=diagnostics, notes=notes) + if explosive: + notes.append(f"{explosive} of {REPLICATE_COUNT} replicate refits estimated |rho| >= 1 and were counted " + "as failed refits (not dropped silently); se_sampling uses the remaining refits") + check_refit_count(failed, len(B), max_failed_replicates, context) B = np.vstack(B) se_sampling = pd.Series(np.sqrt(REPLICATE_FACTOR * ((B - betas) ** 2).sum(axis=0)), index=terms) sources.append(SOURCE_SAMPLING_REPLICATE) @@ -711,7 +1164,36 @@ def spatial_regression( "absorbs outcome sampling error), as for census.models.fit_replicate") level = confidence_level if confidence_level <= 1 else confidence_level / 100 table = _table(params, se_model, se_sampling, level, target) + effects = None + if model == "lag": + table["interpretation"] = ["intercept"] + [ + "coefficient on X in y = rho W y + X beta + e: NOT the marginal effect (see effects)"] * len(names) + [ + "spatial lag parameter rho (spillover strength); its p-value tests rho = 0, not the model " + "specification"] + if eff is None: + notes.insert(0, f"{LAG_EFFECTS_NOTE}. Effects were NOT computed: {weights.n} areas exceed " + f"EFFECTS_MAX_N={EFFECTS_MAX_N} (dense eigenvalues of W)") + else: + eff_sampling = pd.Series(np.nan, index=eff["terms"]) + if B is not None: + E = np.vstack([_lag_effect_values(b, eig, len(names), np.asarray(share)) for b in B]) + eff_sampling[:] = np.sqrt(REPLICATE_FACTOR * ((E - eff["estimate"]) ** 2).sum(axis=0)) + effects = _table(pd.Series(eff["estimate"], index=eff["terms"]), pd.Series(eff["se"], index=eff["terms"]), + eff_sampling, level, target) + effects.insert(1, "variable", [v for v in names for _ in range(3)]) + effects.insert(2, "effect", ["direct", "indirect", "total"] * len(names)) + notes[0:0] = [LAG_EFFECTS_NOTE, LAG_EFFECTS_CAVEAT] + if not abs(spatial) < 1: + notes.insert(0, f"rho = {spatial:.4f} is outside (-1, 1): (I - rho W)^-1 is not a stationary spatial " + "multiplier, so the direct, indirect and total effects are undefined (reported as " + "NaN, no intervals). Do not interpret beta or rho as effects; check the " + "specification (a spatial Durbin process or W X instruments correlated with the " + "error push GMM rho toward or past 1), or try method='ml'") + diagnostics["effects_undefined"] = True + diagnostics["effects_multipliers"] = {k: v for k, v in effect_multipliers(eig, spatial, share).items() + if not k.startswith("d_")} notes.append(f"intervals and p-values use the '{target}' target") + notes.append(SPATIAL_PARAMETER_TEST_NOTE.format(sp=sp_name)) notes.append(ERROR_CAVEAT if model == "error" else LAG_ML_CAVEAT if method == "ml" else LAG_GM_NOTE) notes.append(NOISY_COVARIATE_CAVEAT) return ModelResult( @@ -719,5 +1201,8 @@ def spatial_regression( method=est_method, uncertainty_sources=sources, confidence_level=level, notes=notes, diagnostics=diagnostics, provenance={"library": "spreg", "spatial_parameter": sp_name, "ml_method": ml_method if method == "ml" else None, - "weights": getattr(weights, "census_report", None) or weights_report(weights)}, + "weights": getattr(weights, "census_report", None) or weights_report(weights), + **({"coefficient_meaning": "beta is the coefficient on X in y = rho W y + X beta + e, not a " + "marginal effect; see effects"} if model == "lag" else {})}, + effects=effects, ) diff --git a/census/validation.py b/census/validation.py index ceddfd6..9baa68e 100644 --- a/census/validation.py +++ b/census/validation.py @@ -63,6 +63,7 @@ from census import replicate as _replicate from census import decennial as _dec from census.api import BASE_URL, DATASETS, CensusAPIClient +from census.http import polite_get from census.stats import CENSUS_SENTINELS, acs5_overlap_years, parse_variable_code, z_test REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) @@ -74,8 +75,10 @@ # Census API: at most 50 variables per request (NAME counts). MAX_GET_VARIABLES = 48 -USER_AGENT = "acs-batch-downloader/validation (research toolkit; python-requests)" -WWW2_DELAY_SECONDS = 3.0 # census.gov firewall rejects rapid scripted requests +USER_AGENT = "census-research-engine/validation (research toolkit; python-requests)" +# Extra courtesy pause between this harness's www2 requests, on top of the +# process-wide pacing, retries and firewall handling in census.http. +WWW2_DELAY_SECONDS = 3.0 SUMMARY_FILE_URL = ( "https://www2.census.gov/programs-surveys/acs/summary_file/{year}/table-based-SF/data/" @@ -678,13 +681,9 @@ def www2_file(self, name: str, url: str, kind: str) -> bytes: self.urls.append(url) if not self.live: return self.read_fixture(name) - import requests # local: keeps the module importable offline without network deps - self._www2_wait() - resp = requests.get(url, headers={"User-Agent": USER_AGENT}, timeout=120) + resp = polite_get(url, user_agent=USER_AGENT, error=SourceError, timeout=120) body = resp.content or b"" - if b"Request Rejected" in body[:2000]: - raise SourceError(f"census.gov rejected the request for {url} (firewall page); retry later.") if resp.status_code != 200: raise SourceError(f"HTTP {resp.status_code} for {url}") if self.record: @@ -707,21 +706,19 @@ def summary_file_rows(self, year: int, table: str, geo_ids: Sequence[str]) -> Di text = gzip.decompress(self.read_fixture(name)).decode("utf-8") meta = self.fixture_meta(name) return _parse_sf(text.splitlines(), geo_ids, meta.get("byte_range", "")) - import requests # local: keeps the module importable offline without network deps - fetched = b"" total = None while True: self._www2_wait() start = len(fetched) - resp = requests.get( + resp = polite_get( url, - headers={"Range": f"bytes={start}-{start + SUMMARY_FILE_CHUNK - 1}", "User-Agent": USER_AGENT}, + user_agent=USER_AGENT, + error=SourceError, + headers={"Range": f"bytes={start}-{start + SUMMARY_FILE_CHUNK - 1}"}, timeout=120, ) body = resp.content or b"" - if b"Request Rejected" in body[:2000]: - raise SourceError(f"census.gov rejected the request for {url} (firewall page); retry later.") if resp.status_code == 404: raise SourceError(f"Not published (HTTP 404): {url}") if resp.status_code not in (200, 206): diff --git a/census/workflows_geo.py b/census/workflows_geo.py index 293f02b..80b2e3c 100644 --- a/census/workflows_geo.py +++ b/census/workflows_geo.py @@ -188,8 +188,8 @@ def spatial_autocorrelation( local: Also return local clusters with multiple-comparison correction (two-sided p_value = 2 x p_sim from 9999 permutations, BH, alpha 0.10). Simulation: rates at nominal under randomness, - little power on ~150 areas, and spatially patterned sampling - error creates spurious clusters (see local_notes). + little power on ~150 areas, and spatially varying SEs create + spurious significant areas (see local_notes). replicates_path: Census replicates (build_analysis_frame) for a design-based SE of I. That SE is conservative (validated coverage 0.987 for nominal 0.90). @@ -224,9 +224,14 @@ def spatial_regression( weights: str = "queen", replicates_path: Optional[str] = None, method: Optional[str] = None, + correct_outcome_error: bool = False, + se_col: Optional[str] = None, + controlled_total: Optional[bool] = None, ) -> Dict[str, Any]: """Spatial lag or spatial error regression (PySAL spreg) with Lagrange multiplier diagnostics; replicate SEs when replicates are given. + Lag model: report ``effects`` (direct/indirect/total, with SEs and + intervals) first; its beta is not the marginal effect. Args: method: "gm" or "ml". Default: "gm" (GMM) for the lag model, "ml" @@ -235,6 +240,15 @@ def spatial_regression( 0.3, 0.6) for nominal 0.90, ML lag only 0.48 and 0.003. Error model: lambda is attenuated (either method); beta intervals hold. See the result notes. + correct_outcome_error: Error model only (default False). Corrects + lambda for the outcome's sampling error, using se_col (the + outcome's SE column) as known variances; assumes independent + errors across areas. method is then "reml" (default) or "ml". + Validated: lambda intervals covered 0.86-0.93. Warns for + population/household totals (negatively correlated errors). + At most 2,000 areas (dense O(n^3) fit); larger frames fail: + use counties or a regional subset. SEs of 0 are refused. + controlled_total: true flags the outcome as a controlled total. Example: spatial_regression("frame.parquet", "poverty_rate ~ median_rent", "tract", 2023, model="error") """ @@ -242,12 +256,26 @@ def spatial_regression( try: gdf, w = _weights_for(_with_boundaries(_load_frame(input_path), level, year), weights) - res = spatial.spatial_regression(formula, gdf, w, model=model, method=method, replicates=_release_replicates(_read_replicates(replicates_path), year)) + res = spatial.spatial_regression(formula, gdf, w, model=model, method=method, + replicates=_release_replicates(_read_replicates(replicates_path), year), + correct_outcome_error=correct_outcome_error, se_col=se_col, + controlled_total=controlled_total) except _GEO_ERRORS as exc: - return _fail(errors.INVALID_INPUT, str(exc)) + step = "" + if correct_outcome_error and f"limited to {spatial.CORRECTION_MAX_N} areas" in str(exc): + step = (f"refit at county level, or subset input_path to a region of at most " + f"{spatial.CORRECTION_MAX_N} areas (one or a few states or metro areas) and rerun; or drop " + "correct_outcome_error and report the attenuation caveat from the notes") + return _fail(errors.INVALID_INPUT, str(exc), step) out = res.to_dict() - return _ok(result_summary(res), "make_table or report coefficients with the LM diagnostics", - **{k: v for k, v in out.items() if k != "summary"}) + step = ("report the effects table first (direct, indirect and total effects with their intervals: the " + "marginal effects; beta is not one), then rho and the LM diagnostics" + if res.effects is not None else + "report the corrected lambda with its interval and the independence assumption from the notes " + "(heed any controlled-total warning); compare with diagnostics.outcome_error_correction." + "lambda_uncorrected" if correct_outcome_error else + "make_table or report coefficients with the LM diagnostics") + return _ok(result_summary(res), step, **{k: v for k, v in out.items() if k != "summary"}) def spatial_small_area_estimates( diff --git a/docs/agent_protocol.md b/docs/agent_protocol.md index d9ca733..c92a001 100644 --- a/docs/agent_protocol.md +++ b/docs/agent_protocol.md @@ -180,7 +180,7 @@ Validated in `docs/audits/inference_validation.md` by simulation with known trut - **Maps:** `make_map(input_path, column, level, year, se_col=...)`. Pass an SE or MOE column so unreliable areas are hatched. Report the class separability printed in the caption; when it is low, adjacent colors are not statistically distinguishable. Use `kind="significance"` with a `batch_significance` output to map change, never raw differences. - **Boundaries** are matched to the ACS release automatically (2010-based tracts through the 2019 release, 2020-based from 2020; Connecticut planning regions from 2022). -- **Spatial statistics:** `spatial_autocorrelation` (Moran's I, optional local clusters with correction), `spatial_regression` (lag or error), `spatial_small_area_estimates` (spatial Fay–Herriot). Noisy small-area estimates can create or hide clusters: pass replicates, or rerun on Fay–Herriot estimates as a sensitivity check. +- **Spatial statistics:** `spatial_autocorrelation` (Moran's I, optional local clusters with correction), `spatial_regression` (lag or error; for the error model pass `correct_outcome_error=true` with `se_col` = the outcome's SE column to correct lambda for sampling error, which assumes independent errors across areas: fine for rates, shares and medians, warned for population/household totals), `spatial_small_area_estimates` (spatial Fay–Herriot). Noisy small-area estimates can create or hide clusters: pass replicates, or rerun on Fay–Herriot estimates as a sensitivity check. - **Tract harmonization** uses decennial block population or housing-unit weights by default (`weight="auto"`); the first national run downloads about 3,200 county block-count requests, then caches them. - **Custom areas:** `custom_area_mapping(areas_path, ...)` then `aggregate_geographies`. Partial overlaps are reported with their allocation assumption. - **Errata:** every extraction plan lists Census errata and user notes for the table and releases; mention them in write-ups. diff --git a/docs/audits/CHECK_MAP.md b/docs/audits/CHECK_MAP.md index c9d3f0e..59b2773 100644 --- a/docs/audits/CHECK_MAP.md +++ b/docs/audits/CHECK_MAP.md @@ -19,7 +19,10 @@ All commands use `.venv/bin/python` from the repository root. | Linear trend test (`census.trends`) | `-m pytest -q tests/test_trends.py` | `linear_trend` (gated) | | Moran's I replicate SE (`census.spatial`) | `-m pytest -q tests/test_spatial.py` | `morans_i_replicate_se` (info) | | Spatial lag / error regression, LM diagnostics (`census.spatial.spatial_regression`) | `-m pytest -q tests/test_spatial.py tests/test_workflows_geo.py` | `spatial_regression_coverage` (info, D15-1) | -| Local Moran's I / LISA (`census.spatial.local_morans`) | `-m pytest -q tests/test_spatial.py tests/test_workflows_geo.py` | `local_morans_size` (info, D15-1) | +| Error model corrected for outcome sampling error (`spatial_regression(model='error', correct_outcome_error=True, se_col=...)`; spatial Fay-Herriot likelihood via `census.sae_spatial`) | `-m pytest -q tests/test_spatial_lambda_correction.py tests/test_workflows_geo.py` (the A1 study's diagonal-Sigma rows call the tool's own function `census.spatial.error_model_known_variances`, checked in `tests/test_review_fixes_r16.py`; it also equals the study-only `me_sem_fit` on the seeded draws, REML and ML; 20-draw coverage smoke check; warnings, boundary, non-convergence, SE = 0 and the `CORRECTION_MAX_N` size limit) | `spatial_error_attenuation` rows "spatial Fay-Herriot REML rho" (the tool's default; ML rows for `method='ml'`) and `spatial_error_attenuation_correlated` (info, on demand) | +| Lag-model effects (direct/indirect/total; `census.spatial.lag_effects`, `report_tables.effects_table`) and the Round 16 study estimators (`census.simulation_spreg`) | `-m pytest -q tests/test_spatial_regression_validation.py` (effects vs S(W) algebra and spreg's multipliers; study likelihoods vs spreg and `census.sae_spatial`) | `spatial_dgp_recovery`, `spatial_error_attenuation`, `spatial_error_attenuation_correlated`, `common_factor_lr`, `common_factor_lr_acs` (info, on demand) | +| Local Moran's I / LISA (`census.spatial.local_morans`) | `-m pytest -q tests/test_spatial.py tests/test_workflows_geo.py` | `local_morans_size` (info, D15-1); `lisa_validation_null`, `lisa_validation_clusters` (info, on demand, Round 16) | +| LISA validation prototypes (`census.lisa_validation`: correlated-error scenarios, replicate-stability filter, LISA on spatial FH EBLUPs, Wong 2021 heuristic) | `-m pytest -q tests/test_lisa_validation.py` | `lisa_validation_null`, `lisa_validation_clusters` (info, on demand) | | Multilevel, known variance (`census.multilevel`) | `-m pytest -q tests/test_multilevel.py` (includes the dense fixed-variance reference `TestKnownVarianceDenseReference`) | `multilevel_known_variance`, `multilevel_random_slope`, `multilevel_replicate` | | Difference-in-differences (`census.did`) | `-m pytest -q tests/test_did.py` (includes the hand-built differenced reference `TestTwoGroupDenseReference`) | `did_two_group`, `did_event`, `did_replicate`; serial correlation (info only): `did_ar1_sensitivity`, `did_serial` | | Release gate itself (`scripts/inference_validation.py`) | `-m pytest -q tests/test_validation_gate.py` | n/a | @@ -94,10 +97,51 @@ Release studies: 9999 permutations with BH at alpha 0.10: per-area rate 0.099 (previous default 0.196) and family-wise rate 0.088 (MC SE 0.009) under randomness; power for the hot spot 0.011; spatially patterned SEs still - give spurious clusters in 38% of maps (previous default 26%). The + give at least one significant area in 38% of maps (previous default + 26%); those SEs are heterogeneous but independent, so this is not + evidence about spatially correlated errors (Round 16 + `lisa_validation_*`). The previous behavior stays available (`method="ml"`; `permutations=999, sided="folded"`). LM tests' size 0.046-0.056 at 0.05. Results and caveat text: `census.spatial`. +- LISA under sampling error (PLAN_ROUND_16 workstream B): + `scripts/inference_validation.py --studies lisa_validation_null + lisa_validation_clusters --jobs 2` (group `lisa`, own configuration). Run + on demand only: not in the default report (each study takes about 20-30 + min single-process, above the ~5 min budget for default-report studies), + not in `REQUIRED_STUDIES`, every row `info`. Five scenarios (no true + clustering with independent homogeneous, heterogeneous independent and + spatially correlated errors; a hot spot with independent and correlated + errors), the correlated errors parameterized by the mean rook-neighbour + error correlation r1 in -0.2 .. 0.5 (SAR structure, a stated assumption); + four methods on the same maps (tool-default LISA, LISA filtered by + `replicate_quadrant_share`, LISA on spatial Fay-Herriot EBLUPs, Wong 2021 + heuristic hot/cold spots, the latter scored against its own target). + The prototypes live in `census.lisa_validation`; no tool default changed. + Design, metric definitions, results with MC SEs and proposals: + `docs/audits/lisa_validation.md`. +- Spatial regression validation (PLAN_ROUND_16 workstream A): five + info-only studies in `census.simulation_spreg` (group + `spreg_validation`), run only when named, e.g. + `scripts/inference_validation.py --studies spatial_dgp_recovery --out + /tmp/r16a.md` (do not overwrite `docs/audits/inference_validation.md`). + They are not in the default full report and not in `REQUIRED_STUDIES` + (each takes more than 5 min single-process). + `spatial_error_attenuation` / `_correlated`: lambda of the tool's ML + error model vs the spatial Fay-Herriot rho (REML and ML) under + independent and SAR-correlated outcome error, rook/queen/kNN. The + spatial Fay-Herriot REML rows are the evidence for the owner-approved + tool option `spatial_regression(model='error', + correct_outcome_error=True)`: those rows call the tool's own + estimator, `census.spatial.error_model_known_variances` + (`tests/test_review_fixes_r16.py`, `tests/test_spatial_lambda_correction.py`). + `spatial_dgp_recovery`: SEM, SAR, SDM, omitted-covariate and Juhl DGPs + fitted by all four estimators; effects coverage (delta-method and + simulated intervals) and OLS RS test rejection rates. + `common_factor_lr`: LR common-factor test in Juhl's (2021) design + (queen 7x7-20x20) with chi-square and residual-bootstrap critical values; + `common_factor_lr_acs`: the same under ACS-like outcome error. Designs, + results and conclusions: `docs/audits/spatial_regression_validation.md`. - Runtime (PLAN_ROUND_13 C3): the script pins BLAS/OpenMP to one thread per process, and `--jobs N` (0 = one per CPU) runs studies in parallel processes with output identical to a serial run (except the `seconds` diff --git a/docs/audits/acs_sampling_covariance.md b/docs/audits/acs_sampling_covariance.md new file mode 100644 index 0000000..e371317 --- /dev/null +++ b/docs/audits/acs_sampling_covariance.md @@ -0,0 +1,1236 @@ +# ACS sampling-error covariance between areas (VRE replicates) + +Generated 2026-11-09 06:54 UTC by `scripts/acs_error_covariance.py` (commit v0.14.0-29-g1847007-dirty, 10.0 min). PLAN_ROUND_16 workstream C (C1-C2). Status: **info** (gates nothing). Per-class numbers: `docs/audits/acs_sampling_covariance.csv` (written by the script; git-ignored like the other audit CSVs). + +## Question + +Do the sampling errors of ACS 5-year estimates covary between areas (neighbouring tracts or counties, tracts in the same county), and can 80 replicates measure that covariance well enough to build it into a measurement-error correction (workstream A1) or a LISA null model (workstream B)? + +## Methods + +* **Covariance.** For estimates X (area i) and Y (area j) of one release, Cov = 4/80 sum_r (X_r - X)(Y_r - Y) over the 80 successive-difference replicates of the Census Variance Replicate Estimate (VRE) tables, the cross-product form of the VRE variance formula (census.replicate.VRE_DOCUMENTATION, formula 1). The 4/80 factor cancels in the correlation r, which is the cosine of the two replicate-deviation vectors. Implementation: `census/replicate_covariance.py`. +* **Variables.** Counts: total population B01003_001 (tracts only; county populations are controlled to the population estimates and have no sampling error), occupied housing units B25003_001, persons below poverty B17001_002. Proportions: poverty rate B17001_002/B17001_001 and owner share B25003_002/B25003_001, recomputed on every replicate. Median: household income linearly interpolated from the B19001 replicate distributions (census.medians; no published median table has VRE files - checked against the 2019 and 2024 VRE table lists); areas whose median is top- or bottom-coded are excluded. Areas with a zero estimate denominator, a zero denominator in any replicate (the replicate ratio is undefined; the SE convention of VRE documentation Example 7, ratio 0, would fabricate a deviation of minus the estimate), controlled estimates or zero replicate variance are excluded (their correlation is undefined). +* **Pairs.** Contiguity from queen weights (libpysal via census.spatial.spatial_weights) on the Census cartographic boundary files of the release's vintage; orders 1-3 are graph distances in the queen contiguity graph (order 1 split into rook = shared edge vs queen-only = corner). Tracts: all pairs of order 1-3; tract pairs in the same county beyond order 3 (random sample of at most 150,000); and a random sample of 100,000 draws of tract pairs in different counties beyond order 3. Counties: all pairs in the state. Distance = centroid distance in an equal-area projection (EPSG:5070). +* **Structured summaries, not pairwise matrices.** The replicate covariance of n areas has rank <= 80, and one pairwise r from 80 replicates has null SD 1/sqrt(80). So only class summaries are reported: mean and median r per contiguity order, within vs across county, distance band. Their uncertainty: a bootstrap over the 80 replicate columns (200 draws; replicate noise; approximate because SDR deviations are not exactly iid across replicates) and a delete-one-county jackknife (drops every pair touching the county; area heterogeneity plus replicate noise). Intervals are mean +/- 1.96 x the larger of the two SEs. +* **Noise floor.** Analytic: if one deviation vector is spherically symmetric in R^80 and independent of the other, (1 + r)/2 ~ Beta(39.5, 39.5): SD 0.112, two-sided 5% critical |r| 0.219 (40 replicates: SD 0.158, critical 0.308). Empirical: (a) replicate-index derangements applied to the second area of each far pair; (b) Michigan x New York area pairs, whose samples are independent, so their correlations show the real null spread of SDR replicate correlations. +* **Reliability.** Split-half agreement: per-pair r computed on two disjoint halves of the replicates (random, odd/even, 1-40 vs 41-80); the correlation across pairs of the two halves' r is the reliability of a 40-replicate pairwise r, stepped up to 80 replicates by Spearman-Brown. Aggregation ratio (counts, tracts): Var(sum of a county's tracts) / sum of the tracts' variances = 1 + sum of within-county covariances / sum of variances; < 1 means negatively correlated errors (the signature of weighting controls), and it pools every within-county pair in one number. +* **Replicate design spectrum.** The eigenvalues of the replicate-space Gram matrix of standardized deviations (80 x 80, over thousands of tracts) are compared with the Marchenko-Pastur lower edge for independent replicates (`replicate_spectrum`). Directions far below it are near-null for every area: a property of the replicate-factor design, not of geography. They lower the effective number of replicates, hence widen the null spread of r. +* **Structured model and shrinkage, scored out of sample.** Replicates are split in half (random, seeded); on one half fit (i) one correlation per contiguity order 1-3 (0 beyond), (ii) for tracts one per order x within/across county (0 beyond order 3), (iii) as (ii) plus one for far pairs within a county, and (iv) Ledoit-Wolf shrinkage of the replicate correlation toward the identity; score the Gaussian log-likelihood of the other half's standardized replicate deviations (scaled so E[x x^T] = Sigma) relative to independence; average both directions. Each half is first projected orthogonally to the design near-null directions and the constant vector: otherwise the held-out half's component along those directions is minus the training half's for every area at once, a cross-half dependence that Ledoit-Wolf exploits (it then 'gains' 30-90 nats per replicate even on variables with no neighbour correlation; with the projection it loses, as it does on independent synthetic data). A gain > 0 means the fitted structure predicts held-out replicates better than independent errors. A fitted structure that is not positive definite is projected to the nearest correlation matrix with eigenvalues >= 1e-3 and flagged. + +## Data + +Releases 2020-2024, 2015-2019; states Michigan (26) and New York (36). Files (cached, not committed): + +| kind | year | level | file | SHA-256 | +| :--- | :--- | :--- | :--- | :--- | +| VRE | 2024 | tract | `B01003_26.csv.zip` | `235fda51b0cc2f2cef29f0525b30cdef5fe2addd829a00125eb86102d4b0c4de` | +| VRE | 2024 | tract | `B25003_26.csv.zip` | `beb257fde07b8885af75aa5233c90fa96a092931b9952a2b3a53711a44c503bc` | +| VRE | 2024 | tract | `B17001_26.csv.zip` | `06aac06540352fbddb516fb3dd31eb652eb5ae969db3450b9fcbfda8c5cfa58a` | +| VRE | 2024 | tract | `B19001_26.csv.zip` | `a6df2bd01e65a71191d76bfb9428ea79fd0f958e1525cae11b39a1a84682e846` | +| boundary | 2024 | tract | `cb_2024_26_tract_500k.zip` | `9bd8aaf99a903bffff16efc59bad8f0ebea65dcf086ef315cea89663f4cc3b6e` | +| VRE | 2024 | tract | `B01003_36.csv.zip` | `87b39e2b153bb8ea64b0283624687e61429afa89f1cca3437cd3b7a46752a970` | +| VRE | 2024 | tract | `B25003_36.csv.zip` | `05462d99d4039ca4264e4dcfa133b5b4f2e5589d9577051536f54a017e7d3584` | +| VRE | 2024 | tract | `B17001_36.csv.zip` | `6b1e85f8c8be2c25edf462fd8a40cd50f85f30748cf4b0d9f2df809de230da83` | +| VRE | 2024 | tract | `B19001_36.csv.zip` | `6a68190ea29b73389399e397ac5aeef97faa60db5dbfa964fcfb014fcce3094a` | +| boundary | 2024 | tract | `cb_2024_36_tract_500k.zip` | `ab7f72cb4a66079fd5b3a99302af0a48bdf87d53aec3b49e5ee4d18a7280cd02` | +| VRE | 2024 | county | `B01003.csv.zip` | `65e008a179c90712205a77b7a9bb0cdc8af535ae759e5b13f7faba970928f62f` | +| VRE | 2024 | county | `B25003.csv.zip` | `9aceebd33bd3a840d95341b3c09f2536a15e56c3fa2c4d08d09058aee13b0c35` | +| VRE | 2024 | county | `B17001.csv.zip` | `3d24fa18e9f005b3c9455d3bda4a4ab843eb8f443d423b56b00496b58ad0c605` | +| VRE | 2024 | county | `B19001.csv.zip` | `95cb48eabcaef5c44fc68377f8b596f31cbb079d846bbea9c6bf7e83ba896d0f` | +| boundary | 2024 | county | `cb_2024_us_county_500k.zip` | `da4051717caec55c75e3748c3608c2a3dbde8d1ff401bbaf4f952e3c3fb63ef1` | +| VRE | 2019 | tract | `B25003_26.csv.zip` | `0b6a3f85d252700385d92a435396f24750d659b89f114e3d6006e525231317d4` | +| VRE | 2019 | tract | `B17001_26.csv.zip` | `8f6e7fb8d21f80e605d03ce4fab11fc3a372184cb1e550d64b84cf779569d729` | +| VRE | 2019 | tract | `B19001_26.csv.zip` | `0cd0858b255228c403627e3bd08094950a56e3d15d414446fd77a719ecf99168` | +| boundary | 2019 | tract | `cb_2019_26_tract_500k.zip` | `a0a60fdb67c9d99c99a4c9a58c38e7f05b7b38e5e43802a02016a5514a3f3e2d` | +| VRE | 2019 | tract | `B01003_36.csv.zip` | `ffb652e753dc225a7be6e0936e30fdd6ce1048d31f1009b037abe5dc0315569a` | +| VRE | 2019 | tract | `B25003_36.csv.zip` | `3abde86fb9e93ab998d31c2521a17cf8d53e71233c4e763536ef4fe7d37be5a5` | +| VRE | 2019 | tract | `B17001_36.csv.zip` | `8dbe92bf59958da80c5e12cea3c76d762b6e0d94e63dd72cf1537906efc30724` | +| VRE | 2019 | tract | `B19001_36.csv.zip` | `461ebbd9fab3f9f051c443a63842d3217f0f7ca1ddf21746cda8058f6493ca41` | +| boundary | 2019 | tract | `cb_2019_36_tract_500k.zip` | `e4ed39cd98bc03365d96b6588a8ff5e2e2c71134d9afa1ff9fd093286c43b553` | +| VRE | 2019 | county | `B01003.csv.zip` | `61cb0a5357aafcbba1d498872165d0402c28f460910592e7e2d9f24bfe9aeaf7` | +| VRE | 2019 | county | `B25003.csv.zip` | `b0cbdab927b427839e5d6a9bda4afa5fc36dd8bd243f38d7101a39effe6170c6` | +| VRE | 2019 | county | `B17001.csv.zip` | `05bc088f51d2c31fbb0aff6fe328c08bab7a94c6de9b531ffb382918bb87de15` | +| VRE | 2019 | county | `B19001.csv.zip` | `56374faae955dd44c9eec5f45c7b2d145781b6271028d714eae6ebafc2f260d3` | +| boundary | 2019 | county | `cb_2019_us_county_500k.zip` | `1c4b703cd34c3f5e5e6368f91f1bd80de9281ef81ad91e005844848df9976c4b` | + +**Unavailable files** (skipped; the variables that need them are not analysed for that state and release, everything else is): + +| year | level | state | table | URL | reason | +| :--- | :--- | :--- | :--- | :--- | :--- | +| 2019 | tract | Michigan | B01003 | https://www2.census.gov/programs-surveys/acs/replicate_estimates/2019/data/5-year/140/B01003_26.csv.zip | census.gov firewall 'Request Rejected' page (support ID 13979921158196382640), also on the single delayed retry | + +A rejection of one specific URL can persist while every other file in the same directory downloads normally, so it is not retried further. To include such a file, download it in a browser and save it as `$CENSUS_VRE_DIR/2019/140/B01003_26.csv.zip` (default `$CENSUS_VRE_DIR` = `.cache/vre`); census.replicate then records its SHA-256 with source 'manual download' and the report marks it. + +Versions: census-acs-toolkit 0.14.0, python 3.14.4, numpy 2.5.4, scipy 1.18.1, pandas 3.1.1, geopandas 1.2.0, libpysal 4.15.1 + +Areas analysed (after exclusions): + +| year | state | geography | variable | type | areas | analysed | zero variance | note | +| :--- | :--- | :--- | :--- | :--- | :--- | :--- | :--- | :--- | +| 2024 | Michigan | tract | population | count | 2971 | 2914 | 57 | | +| 2024 | Michigan | tract | households | count | 2971 | 2897 | 74 | | +| 2024 | Michigan | tract | poverty_count | count | 2971 | 2893 | 78 | | +| 2024 | Michigan | tract | poverty_rate | proportion | 2971 | 2887 | 13 | | +| 2024 | Michigan | tract | owner_share | proportion | 2971 | 2865 | 31 | 1 area(s) with a zero replicate denominator excluded | +| 2024 | Michigan | tract | median_hh_income | median | 2971 | 2870 | 6 | 20 top/bottom-coded medians excluded | +| 2024 | New York | tract | population | count | 5394 | 5289 | 91 | | +| 2024 | New York | tract | households | count | 5394 | 5251 | 129 | | +| 2024 | New York | tract | poverty_count | count | 5394 | 5242 | 138 | | +| 2024 | New York | tract | poverty_rate | proportion | 5394 | 5238 | 22 | 2 area(s) with a zero replicate denominator excluded | +| 2024 | New York | tract | owner_share | proportion | 5394 | 5145 | 105 | 1 area(s) with a zero replicate denominator excluded | +| 2024 | New York | tract | median_hh_income | median | 5394 | 5053 | 11 | 186 top/bottom-coded medians excluded | +| 2024 | Michigan | county | households | count | 83 | 83 | 0 | | +| 2024 | Michigan | county | poverty_count | count | 83 | 83 | 0 | | +| 2024 | Michigan | county | poverty_rate | proportion | 83 | 83 | 0 | | +| 2024 | Michigan | county | owner_share | proportion | 83 | 83 | 0 | | +| 2024 | Michigan | county | median_hh_income | median | 83 | 83 | 0 | 0 top/bottom-coded medians excluded | +| 2024 | New York | county | households | count | 62 | 62 | 0 | | +| 2024 | New York | county | poverty_count | count | 62 | 62 | 0 | | +| 2024 | New York | county | poverty_rate | proportion | 62 | 62 | 0 | | +| 2024 | New York | county | owner_share | proportion | 62 | 62 | 0 | | +| 2024 | New York | county | median_hh_income | median | 62 | 62 | 0 | 0 top/bottom-coded medians excluded | +| 2019 | Michigan | tract | households | count | 2772 | 2738 | 34 | | +| 2019 | Michigan | tract | poverty_count | count | 2772 | 2736 | 36 | | +| 2019 | Michigan | tract | poverty_rate | proportion | 2772 | 2734 | 8 | | +| 2019 | Michigan | tract | owner_share | proportion | 2772 | 2722 | 16 | | +| 2019 | Michigan | tract | median_hh_income | median | 2772 | 2730 | 2 | 6 top/bottom-coded medians excluded | +| 2019 | New York | tract | population | count | 4906 | 4854 | 52 | | +| 2019 | New York | tract | households | count | 4906 | 4828 | 78 | | +| 2019 | New York | tract | poverty_count | count | 4906 | 4811 | 95 | | +| 2019 | New York | tract | poverty_rate | proportion | 4906 | 4806 | 28 | 2 area(s) with a zero replicate denominator excluded | +| 2019 | New York | tract | owner_share | proportion | 4906 | 4747 | 80 | 1 area(s) with a zero replicate denominator excluded | +| 2019 | New York | tract | median_hh_income | median | 4906 | 4745 | 16 | 66 top/bottom-coded medians excluded | +| 2019 | Michigan | county | households | count | 83 | 83 | 0 | | +| 2019 | Michigan | county | poverty_count | count | 83 | 83 | 0 | | +| 2019 | Michigan | county | poverty_rate | proportion | 83 | 83 | 0 | | +| 2019 | Michigan | county | owner_share | proportion | 83 | 83 | 0 | | +| 2019 | Michigan | county | median_hh_income | median | 83 | 83 | 0 | 0 top/bottom-coded medians excluded | +| 2019 | New York | county | households | count | 62 | 62 | 0 | | +| 2019 | New York | county | poverty_count | count | 62 | 62 | 0 | | +| 2019 | New York | county | poverty_rate | proportion | 62 | 62 | 0 | | +| 2019 | New York | county | owner_share | proportion | 62 | 62 | 0 | | +| 2019 | New York | county | median_hh_income | median | 62 | 62 | 0 | 0 top/bottom-coded medians excluded | + +Skipped: 2024 Michigan county population: 83 controlled (no sampling error); 2024 New York county population: 62 controlled (no sampling error); 2019 Michigan tract population: VRE file B01003 unavailable (see Data: unavailable files); 2019 Michigan county population: 83 controlled (no sampling error); 2019 New York county population: 62 controlled (no sampling error). + +## Results 2020-2024 + +Mean pairwise r [95% interval] (number of pairs) by queen contiguity order. + +### Tracts: contiguity order + +| state | variable | order 1 | order 2 | order 3 | beyond order 3 | +| :--- | :--- | :--- | :--- | :--- | :--- | +| Michigan | population | -0.071 [-0.090, -0.052] (n=9099) | -0.014 [-0.017, -0.011] (n=21283) | -0.004 [-0.005, -0.003] (n=36300) | -0.001 [-0.001, 0.000] (n=225050) | +| Michigan | households | -0.025 [-0.033, -0.017] (n=9033) | -0.008 [-0.011, -0.004] (n=21067) | -0.003 [-0.005, -0.002] (n=35923) | -0.001 [-0.001, 0.000] (n=223197) | +| Michigan | poverty_count | -0.004 [-0.007, -0.002] (n=9024) | -0.003 [-0.005, -0.002] (n=21028) | -0.000 [-0.002, 0.001] (n=35876) | 0.000 [-0.001, 0.002] (n=222204) | +| Michigan | poverty_rate | -0.000 [-0.003, 0.002] (n=8988) | -0.002 [-0.004, -0.001] (n=20944) | 0.000 [-0.002, 0.002] (n=35733) | 0.000 [-0.001, 0.001] (n=221162) | +| Michigan | owner_share | -0.002 [-0.005, 0.000] (n=8875) | 0.000 [-0.001, 0.002] (n=20622) | -0.001 [-0.002, 0.001] (n=35161) | 0.000 [-0.000, 0.001] (n=217458) | +| Michigan | median_hh_income | 0.000 [-0.002, 0.003] (n=8892) | 0.000 [-0.002, 0.003] (n=20659) | 0.001 [-0.000, 0.002] (n=35172) | 0.001 [0.000, 0.002] (n=217954) | +| New York | population | -0.040 [-0.060, -0.020] (n=15821) | -0.009 [-0.013, -0.004] (n=37762) | -0.003 [-0.004, -0.002] (n=65290) | -0.001 [-0.002, -0.001] (n=230255) | +| New York | households | -0.015 [-0.022, -0.008] (n=15668) | -0.003 [-0.006, -0.001] (n=37281) | -0.002 [-0.003, -0.001] (n=64393) | -0.001 [-0.001, -0.000] (n=227670) | +| New York | poverty_count | -0.002 [-0.003, 0.000] (n=15646) | -0.001 [-0.002, 0.001] (n=37221) | 0.000 [-0.001, 0.001] (n=64288) | -0.000 [-0.001, 0.001] (n=226711) | +| New York | poverty_rate | -0.000 [-0.002, 0.002] (n=15627) | -0.000 [-0.001, 0.001] (n=37151) | 0.000 [-0.001, 0.001] (n=64145) | 0.000 [-0.001, 0.001] (n=226417) | +| New York | owner_share | -0.001 [-0.002, 0.001] (n=15121) | 0.000 [-0.001, 0.001] (n=35819) | 0.001 [-0.000, 0.002] (n=61842) | 0.000 [-0.000, 0.001] (n=217364) | +| New York | median_hh_income | 0.000 [-0.002, 0.002] (n=14672) | 0.001 [-0.000, 0.002] (n=34551) | -0.000 [-0.001, 0.001] (n=59637) | 0.001 [0.000, 0.001] (n=210794) | + +### Counties: contiguity order + +| state | variable | order 1 | order 2 | order 3 | beyond order 3 | +| :--- | :--- | :--- | :--- | :--- | :--- | +| Michigan | households | 0.026 [0.001, 0.051] (n=208) | 0.031 [0.009, 0.052] (n=339) | 0.027 [0.005, 0.049] (n=404) | 0.033 [0.015, 0.051] (n=2452) | +| Michigan | poverty_count | -0.005 [-0.027, 0.017] (n=208) | 0.009 [-0.007, 0.026] (n=339) | 0.005 [-0.014, 0.024] (n=404) | 0.002 [-0.007, 0.010] (n=2452) | +| Michigan | poverty_rate | -0.005 [-0.027, 0.017] (n=208) | 0.009 [-0.007, 0.026] (n=339) | 0.005 [-0.014, 0.024] (n=404) | 0.002 [-0.007, 0.010] (n=2452) | +| Michigan | owner_share | -0.003 [-0.025, 0.018] (n=208) | 0.018 [-0.001, 0.036] (n=339) | 0.022 [0.006, 0.039] (n=404) | 0.001 [-0.006, 0.009] (n=2452) | +| Michigan | median_hh_income | 0.003 [-0.016, 0.022] (n=208) | 0.002 [-0.014, 0.018] (n=339) | 0.002 [-0.015, 0.020] (n=404) | -0.000 [-0.007, 0.007] (n=2452) | +| New York | households | 0.021 [-0.005, 0.048] (n=142) | 0.024 [-0.001, 0.049] (n=231) | 0.025 [0.001, 0.050] (n=252) | 0.031 [0.015, 0.047] (n=1266) | +| New York | poverty_count | 0.003 [-0.025, 0.031] (n=142) | 0.006 [-0.012, 0.024] (n=231) | 0.019 [-0.006, 0.045] (n=252) | 0.008 [-0.003, 0.019] (n=1266) | +| New York | poverty_rate | 0.003 [-0.025, 0.031] (n=142) | 0.006 [-0.012, 0.024] (n=231) | 0.019 [-0.006, 0.045] (n=252) | 0.008 [-0.003, 0.019] (n=1266) | +| New York | owner_share | -0.002 [-0.034, 0.030] (n=142) | 0.010 [-0.015, 0.035] (n=231) | -0.002 [-0.021, 0.017] (n=252) | 0.012 [0.002, 0.022] (n=1266) | +| New York | median_hh_income | 0.006 [-0.019, 0.031] (n=142) | -0.008 [-0.031, 0.015] (n=231) | 0.004 [-0.015, 0.024] (n=252) | 0.012 [0.002, 0.021] (n=1266) | + +### Tracts: within vs across county (weighting controls) + +| state | variable | order 1 / within county | order 1 / across county | order 2 / within county | order 2 / across county | beyond order 3 / within county | beyond order 3 / across county | +| :--- | :--- | :--- | :--- | :--- | :--- | :--- | :--- | +| Michigan | population | -0.081 [-0.106, -0.056] (n=8000) | 0.003 [-0.007, 0.014] (n=1099) | -0.018 [-0.024, -0.012] (n=15774) | -0.001 [-0.005, 0.002] (n=5509) | -0.001 [-0.001, -0.000] (n=137288) | 0.000 [-0.001, 0.001] (n=87762) | +| Michigan | households | -0.028 [-0.039, -0.018] (n=7936) | -0.003 [-0.012, 0.006] (n=1097) | -0.009 [-0.015, -0.004] (n=15569) | -0.002 [-0.006, 0.002] (n=5498) | -0.001 [-0.003, -0.000] (n=136491) | 0.001 [-0.001, 0.002] (n=86706) | +| Michigan | poverty_count | -0.005 [-0.007, -0.002] (n=7926) | 0.001 [-0.008, 0.011] (n=1098) | -0.005 [-0.006, -0.003] (n=15537) | 0.000 [-0.003, 0.003] (n=5491) | 0.000 [-0.001, 0.002] (n=135706) | -0.000 [-0.002, 0.001] (n=86498) | +| Michigan | poverty_rate | -0.001 [-0.003, 0.002] (n=7892) | 0.002 [-0.007, 0.012] (n=1096) | -0.003 [-0.005, -0.002] (n=15457) | 0.001 [-0.003, 0.004] (n=5487) | 0.001 [-0.001, 0.002] (n=135029) | -0.000 [-0.002, 0.001] (n=86133) | +| Michigan | owner_share | -0.002 [-0.005, -0.000] (n=7781) | 0.000 [-0.010, 0.011] (n=1094) | 0.001 [-0.001, 0.003] (n=15164) | -0.001 [-0.006, 0.003] (n=5458) | 0.000 [-0.000, 0.001] (n=132634) | 0.000 [-0.001, 0.001] (n=84824) | +| Michigan | median_hh_income | 0.000 [-0.002, 0.003] (n=7804) | -0.001 [-0.011, 0.009] (n=1088) | -0.000 [-0.002, 0.002] (n=15220) | 0.002 [-0.002, 0.006] (n=5439) | 0.001 [0.001, 0.002] (n=132753) | 0.000 [-0.001, 0.001] (n=85201) | +| New York | population | -0.043 [-0.065, -0.020] (n=14769) | 0.002 [-0.006, 0.009] (n=1052) | -0.010 [-0.016, -0.005] (n=31951) | 0.001 [-0.003, 0.005] (n=5811) | -0.001 [-0.002, -0.001] (n=140773) | -0.001 [-0.002, 0.000] (n=89482) | +| New York | households | -0.016 [-0.024, -0.008] (n=14624) | -0.000 [-0.011, 0.011] (n=1044) | -0.004 [-0.008, -0.001] (n=31575) | 0.002 [-0.002, 0.006] (n=5706) | -0.001 [-0.002, -0.001] (n=139403) | 0.000 [-0.001, 0.001] (n=88267) | +| New York | poverty_count | -0.002 [-0.004, -0.000] (n=14606) | 0.004 [-0.005, 0.013] (n=1040) | -0.000 [-0.002, 0.001] (n=31514) | -0.002 [-0.006, 0.002] (n=5707) | -0.000 [-0.002, 0.002] (n=138746) | 0.000 [-0.001, 0.001] (n=87965) | +| New York | poverty_rate | -0.000 [-0.002, 0.002] (n=14588) | 0.003 [-0.007, 0.013] (n=1039) | 0.000 [-0.001, 0.001] (n=31453) | -0.002 [-0.006, 0.003] (n=5698) | -0.000 [-0.002, 0.002] (n=138557) | 0.000 [-0.000, 0.001] (n=87860) | +| New York | owner_share | -0.001 [-0.003, 0.001] (n=14105) | 0.004 [-0.007, 0.015] (n=1016) | 0.000 [-0.001, 0.001] (n=30292) | -0.000 [-0.005, 0.004] (n=5527) | 0.000 [-0.000, 0.001] (n=132456) | 0.001 [-0.000, 0.001] (n=84908) | +| New York | median_hh_income | 0.000 [-0.002, 0.003] (n=13668) | 0.000 [-0.009, 0.010] (n=1004) | 0.001 [0.000, 0.003] (n=29082) | -0.001 [-0.006, 0.003] (n=5469) | 0.001 [0.000, 0.001] (n=129020) | 0.001 [-0.001, 0.002] (n=81774) | + +### Rook (shared edge) vs queen-only (corner) first-order neighbours + +Tracts: + +| state | variable | order 1 rook (shared edge) | order 1 queen-only (corner) | +| :--- | :--- | :--- | :--- | +| Michigan | population | -0.080 [-0.099, -0.060] (n=7433) | -0.029 [-0.039, -0.020] (n=1666) | +| Michigan | households | -0.028 [-0.035, -0.021] (n=7373) | -0.013 [-0.024, -0.003] (n=1660) | +| Michigan | poverty_count | -0.005 [-0.008, -0.002] (n=7363) | 0.000 [-0.004, 0.005] (n=1661) | +| Michigan | poverty_rate | -0.001 [-0.003, 0.002] (n=7333) | 0.003 [-0.003, 0.008] (n=1655) | +| Michigan | owner_share | -0.003 [-0.005, 0.000] (n=7244) | 0.000 [-0.010, 0.011] (n=1631) | +| Michigan | median_hh_income | 0.001 [-0.002, 0.003] (n=7257) | -0.002 [-0.008, 0.005] (n=1635) | +| New York | population | -0.043 [-0.065, -0.022] (n=13919) | -0.014 [-0.024, -0.003] (n=1902) | +| New York | households | -0.016 [-0.023, -0.008] (n=13778) | -0.010 [-0.017, -0.002] (n=1890) | +| New York | poverty_count | -0.002 [-0.004, -0.000] (n=13753) | 0.003 [-0.003, 0.008] (n=1893) | +| New York | poverty_rate | -0.000 [-0.002, 0.002] (n=13736) | 0.002 [-0.003, 0.007] (n=1891) | +| New York | owner_share | -0.001 [-0.003, 0.001] (n=13308) | 0.001 [-0.004, 0.006] (n=1813) | +| New York | median_hh_income | -0.000 [-0.003, 0.002] (n=12911) | 0.004 [-0.003, 0.011] (n=1761) | + +Counties: + +| state | variable | order 1 rook (shared edge) | order 1 queen-only (corner) | +| :--- | :--- | :--- | :--- | +| Michigan | households | 0.030 [0.003, 0.057] (n=180) | 0.001 [-0.068, 0.071] (n=28) | +| Michigan | poverty_count | -0.004 [-0.026, 0.019] (n=180) | -0.016 [-0.082, 0.051] (n=28) | +| Michigan | poverty_rate | -0.004 [-0.026, 0.019] (n=180) | -0.015 [-0.082, 0.053] (n=28) | +| Michigan | owner_share | -0.007 [-0.030, 0.016] (n=180) | 0.019 [-0.052, 0.089] (n=28) | +| Michigan | median_hh_income | 0.002 [-0.018, 0.022] (n=180) | 0.008 [-0.041, 0.057] (n=28) | +| New York | households | 0.023 [-0.004, 0.049] (n=140) | -0.075 [-0.226, 0.076] (n=2) | +| New York | poverty_count | 0.002 [-0.027, 0.031] (n=140) | 0.102 [-0.033, 0.237] (n=2) | +| New York | poverty_rate | 0.002 [-0.027, 0.031] (n=140) | 0.097 [-0.038, 0.233] (n=2) | +| New York | owner_share | -0.002 [-0.034, 0.030] (n=140) | -0.014 [-0.345, 0.316] (n=2) | +| New York | median_hh_income | 0.008 [-0.018, 0.033] (n=140) | -0.077 [-0.411, 0.258] (n=2) | + +### By centroid distance + +Tracts: + +| state | variable | 0-2 km | 2-5 km | 5-10 km | 10-20 km | 20-50 km | 50-100 km | 100-inf km | +| :--- | :--- | :--- | :--- | :--- | :--- | :--- | :--- | :--- | +| Michigan | population | -0.045 [-0.070, -0.020] (n=3779) | -0.024 [-0.052, 0.003] (n=19037) | -0.009 [-0.019, 0.002] (n=34754) | -0.003 [-0.006, 0.000] (n=66527) | -0.001 [-0.003, 0.001] (n=89209) | -0.001 [-0.004, 0.001] (n=19601) | 0.001 [-0.001, 0.002] (n=58825) | +| Michigan | households | -0.016 [-0.033, 0.000] (n=3735) | -0.009 [-0.020, 0.002] (n=18826) | -0.005 [-0.011, 0.001] (n=34360) | -0.003 [-0.005, -0.000] (n=66049) | -0.002 [-0.004, 0.000] (n=88762) | 0.000 [-0.002, 0.002] (n=19364) | 0.002 [0.000, 0.003] (n=58124) | +| Michigan | poverty_count | -0.005 [-0.008, -0.002] (n=3741) | -0.003 [-0.004, -0.001] (n=18819) | -0.000 [-0.003, 0.002] (n=34298) | -0.000 [-0.002, 0.001] (n=65783) | 0.000 [-0.001, 0.001] (n=88201) | 0.001 [-0.001, 0.003] (n=19331) | -0.000 [-0.002, 0.001] (n=57959) | +| Michigan | poverty_rate | 0.000 [-0.003, 0.004] (n=3719) | -0.001 [-0.003, 0.001] (n=18714) | 0.000 [-0.002, 0.003] (n=34131) | 0.000 [-0.002, 0.002] (n=65534) | 0.000 [-0.000, 0.001] (n=87741) | 0.001 [-0.001, 0.003] (n=19254) | -0.001 [-0.002, 0.001] (n=57734) | +| Michigan | owner_share | -0.004 [-0.010, 0.002] (n=3652) | -0.001 [-0.003, 0.001] (n=18367) | 0.001 [-0.001, 0.003] (n=33447) | 0.000 [-0.001, 0.002] (n=64393) | -0.000 [-0.001, 0.001] (n=86317) | 0.001 [-0.001, 0.003] (n=18934) | -0.000 [-0.001, 0.001] (n=57006) | +| Michigan | median_hh_income | 0.002 [-0.002, 0.006] (n=3697) | 0.001 [-0.003, 0.005] (n=18471) | 0.002 [-0.001, 0.004] (n=33510) | 0.001 [0.000, 0.002] (n=64441) | 0.001 [-0.000, 0.002] (n=86237) | 0.001 [-0.002, 0.004] (n=19017) | 0.000 [-0.002, 0.002] (n=57304) | +| New York | population | -0.006 [-0.013, 0.001] (n=45076) | -0.008 [-0.015, -0.001] (n=59683) | -0.005 [-0.010, 0.000] (n=74890) | -0.002 [-0.004, -0.000] (n=58714) | -0.002 [-0.004, -0.000] (n=40629) | 0.000 [-0.002, 0.002] (n=14459) | -0.001 [-0.002, -0.000] (n=55677) | +| New York | households | -0.002 [-0.005, 0.001] (n=44716) | -0.003 [-0.006, 0.000] (n=58849) | -0.003 [-0.005, -0.001] (n=74152) | -0.002 [-0.003, -0.001] (n=58039) | -0.002 [-0.004, 0.000] (n=40143) | 0.001 [-0.002, 0.004] (n=14253) | -0.000 [-0.001, 0.001] (n=54860) | +| New York | poverty_count | -0.000 [-0.001, 0.001] (n=44676) | -0.001 [-0.002, 0.001] (n=58855) | 0.000 [-0.001, 0.002] (n=73813) | -0.000 [-0.003, 0.002] (n=57668) | -0.001 [-0.002, 0.001] (n=39943) | 0.001 [-0.002, 0.004] (n=14232) | 0.000 [-0.002, 0.002] (n=54679) | +| New York | poverty_rate | 0.000 [-0.001, 0.001] (n=44632) | -0.001 [-0.002, 0.001] (n=58704) | 0.001 [-0.001, 0.002] (n=73705) | -0.000 [-0.002, 0.001] (n=57560) | -0.000 [-0.002, 0.001] (n=39911) | 0.002 [-0.001, 0.004] (n=14225) | 0.001 [-0.001, 0.002] (n=54603) | +| New York | owner_share | 0.000 [-0.001, 0.001] (n=42025) | 0.001 [-0.000, 0.002] (n=56058) | 0.001 [-0.001, 0.002] (n=70722) | -0.000 [-0.001, 0.001] (n=55345) | 0.000 [-0.001, 0.002] (n=38968) | -0.000 [-0.003, 0.003] (n=13816) | 0.000 [-0.001, 0.002] (n=53212) | +| New York | median_hh_income | 0.001 [-0.001, 0.003] (n=41976) | 0.001 [-0.000, 0.002] (n=54594) | 0.001 [-0.000, 0.001] (n=68456) | 0.000 [-0.001, 0.002] (n=53399) | 0.000 [-0.001, 0.002] (n=36338) | 0.001 [-0.001, 0.004] (n=12913) | 0.000 [-0.001, 0.002] (n=51978) | + +Counties: + +| state | variable | 0-50 km | 50-100 km | 100-200 km | 200-400 km | 400-inf km | +| :--- | :--- | :--- | :--- | :--- | :--- | :--- | +| Michigan | households | 0.031 [0.003, 0.059] (n=139) | 0.026 [0.004, 0.049] (n=374) | 0.027 [0.010, 0.045] (n=1023) | 0.034 [0.014, 0.054] (n=1309) | 0.038 [0.005, 0.071] (n=558) | +| Michigan | poverty_count | -0.007 [-0.033, 0.020] (n=139) | 0.008 [-0.007, 0.024] (n=374) | 0.003 [-0.008, 0.013] (n=1023) | 0.002 [-0.009, 0.012] (n=1309) | 0.001 [-0.018, 0.021] (n=558) | +| Michigan | poverty_rate | -0.007 [-0.033, 0.020] (n=139) | 0.008 [-0.007, 0.024] (n=374) | 0.003 [-0.008, 0.013] (n=1023) | 0.002 [-0.009, 0.013] (n=1309) | 0.001 [-0.018, 0.021] (n=558) | +| Michigan | owner_share | -0.007 [-0.035, 0.022] (n=139) | 0.018 [0.001, 0.035] (n=374) | 0.010 [0.000, 0.021] (n=1023) | 0.001 [-0.009, 0.011] (n=1309) | -0.001 [-0.014, 0.011] (n=558) | +| Michigan | median_hh_income | 0.004 [-0.021, 0.028] (n=139) | 0.004 [-0.011, 0.019] (n=374) | -0.002 [-0.012, 0.008] (n=1023) | 0.002 [-0.007, 0.010] (n=1309) | -0.000 [-0.014, 0.014] (n=558) | +| New York | households | -0.001 [-0.037, 0.036] (n=91) | 0.038 [0.011, 0.065] (n=235) | 0.027 [0.008, 0.045] (n=537) | 0.032 [0.017, 0.046] (n=867) | 0.021 [-0.035, 0.077] (n=161) | +| New York | poverty_count | 0.003 [-0.028, 0.035] (n=91) | 0.009 [-0.010, 0.028] (n=235) | 0.011 [-0.006, 0.028] (n=537) | 0.009 [-0.003, 0.020] (n=867) | 0.006 [-0.021, 0.033] (n=161) | +| New York | poverty_rate | 0.004 [-0.028, 0.035] (n=91) | 0.009 [-0.011, 0.029] (n=235) | 0.011 [-0.006, 0.028] (n=537) | 0.009 [-0.003, 0.021] (n=867) | 0.007 [-0.021, 0.034] (n=161) | +| New York | owner_share | 0.017 [-0.030, 0.064] (n=91) | 0.012 [-0.011, 0.035] (n=235) | 0.002 [-0.013, 0.017] (n=537) | 0.012 [0.000, 0.023] (n=867) | 0.007 [-0.017, 0.031] (n=161) | +| New York | median_hh_income | 0.005 [-0.030, 0.040] (n=91) | 0.011 [-0.011, 0.034] (n=235) | 0.004 [-0.013, 0.022] (n=537) | 0.007 [-0.005, 0.018] (n=867) | 0.023 [-0.000, 0.047] (n=161) | + +### Aggregation ratio: Var(county sum of tracts) / sum of tract variances + +| state | variable | counties | tracts | pooled ratio (SE) | median county ratio | implied mean within-county r (SE) | +| :--- | :--- | :--- | :--- | :--- | :--- | :--- | +| Michigan | population | 83 | 2914 | 0.000 (0.000) | 0.000 | -0.0043 (0.0000) | +| Michigan | households | 83 | 2897 | 0.218 (0.008) | 0.474 | -0.0039 (0.0000) | +| Michigan | poverty_count | 82 | 2892 | 0.915 (0.051) | 0.896 | -0.0004 (0.0003) | +| New York | population | 62 | 5289 | 0.005 (0.001) | 0.000 | -0.0030 (0.0000) | +| New York | households | 62 | 5251 | 0.259 (0.018) | 0.331 | -0.0026 (0.0001) | +| New York | poverty_count | 62 | 5242 | 0.879 (0.044) | 0.936 | -0.0005 (0.0002) | + +### Noise floor and spread + +Analytic null (80 replicates): SD 0.112, 95% |r| 0.219. + +Michigan x New York pairs (independent samples): + +| geography | variable | pairs | mean r (SE) | SD of r | 95% |r| | share |r| > analytic critical | +| :--- | :--- | :--- | :--- | :--- | :--- | :--- | +| tract | population | 50000 | -0.0009 (0.0004) | 0.115 | 0.229 | 0.060 | +| tract | households | 50000 | 0.0006 (0.0004) | 0.116 | 0.230 | 0.060 | +| tract | poverty_count | 50000 | 0.0008 (0.0005) | 0.116 | 0.236 | 0.064 | +| tract | poverty_rate | 50000 | -0.0003 (0.0005) | 0.115 | 0.231 | 0.062 | +| tract | owner_share | 50000 | 0.0009 (0.0005) | 0.116 | 0.231 | 0.062 | +| tract | median_hh_income | 50000 | 0.0011 (0.0004) | 0.115 | 0.226 | 0.058 | +| county | households | 5146 | 0.0320 (0.0032) | 0.121 | 0.243 | 0.077 | +| county | poverty_count | 5146 | 0.0017 (0.0017) | 0.116 | 0.223 | 0.055 | +| county | poverty_rate | 5146 | 0.0018 (0.0017) | 0.116 | 0.223 | 0.055 | +| county | owner_share | 5146 | 0.0066 (0.0019) | 0.116 | 0.226 | 0.058 | +| county | median_hh_income | 5146 | 0.0010 (0.0014) | 0.115 | 0.226 | 0.058 | + +Derangement null on far pairs (same state, beyond order 3): + +| state | geography | variable | pairs | mean | SD | 95% |r| | +| :--- | :--- | :--- | :--- | :--- | :--- | :--- | +| Michigan | tract | population | 225050 | 0.0000 | 0.112 | 0.218 | +| Michigan | tract | households | 223197 | 0.0003 | 0.112 | 0.219 | +| Michigan | tract | poverty_count | 222204 | -0.0001 | 0.112 | 0.219 | +| Michigan | tract | poverty_rate | 221162 | -0.0000 | 0.112 | 0.218 | +| Michigan | tract | owner_share | 217458 | 0.0003 | 0.112 | 0.219 | +| Michigan | tract | median_hh_income | 217954 | 0.0006 | 0.113 | 0.220 | +| New York | tract | population | 230255 | 0.0000 | 0.112 | 0.219 | +| New York | tract | households | 227670 | 0.0003 | 0.112 | 0.219 | +| New York | tract | poverty_count | 226711 | 0.0001 | 0.112 | 0.219 | +| New York | tract | poverty_rate | 226417 | 0.0001 | 0.112 | 0.219 | +| New York | tract | owner_share | 217364 | 0.0005 | 0.112 | 0.219 | +| New York | tract | median_hh_income | 210794 | 0.0008 | 0.112 | 0.220 | +| Michigan | county | households | 2452 | 0.0311 | 0.123 | 0.248 | +| Michigan | county | poverty_count | 2452 | 0.0010 | 0.112 | 0.219 | +| Michigan | county | poverty_rate | 2452 | 0.0009 | 0.112 | 0.219 | +| Michigan | county | owner_share | 2452 | 0.0023 | 0.112 | 0.220 | +| Michigan | county | median_hh_income | 2452 | -0.0002 | 0.112 | 0.219 | +| New York | county | households | 1266 | 0.0281 | 0.115 | 0.231 | +| New York | county | poverty_count | 1266 | 0.0114 | 0.112 | 0.220 | +| New York | county | poverty_rate | 1266 | 0.0116 | 0.112 | 0.221 | +| New York | county | owner_share | 1266 | 0.0096 | 0.112 | 0.220 | +| New York | county | median_hh_income | 1266 | 0.0091 | 0.112 | 0.219 | + +Replicate design spectrum (lowest eigenvalues relative to the mean; Marchenko-Pastur lower edge for independent replicates in brackets): + +| state | geography | variable | lowest 4 eigenvalues (MP edge) | near-null | effective replicates | null SD 1/sqrt(eff.) | +| :--- | :--- | :--- | :--- | :--- | :--- | :--- | +| Michigan | tract | population | 0.040 0.046 0.073 0.391 (0.696) | 3 | 77 | 0.114 | +| Michigan | tract | households | 0.028 0.030 0.054 0.399 (0.695) | 3 | 77 | 0.114 | +| Michigan | tract | poverty_count | 0.028 0.030 0.064 0.403 (0.695) | 3 | 77 | 0.114 | +| Michigan | tract | poverty_rate | 0.028 0.031 0.064 0.402 (0.695) | 3 | 77 | 0.114 | +| Michigan | tract | owner_share | 0.029 0.032 0.063 0.391 (0.694) | 3 | 77 | 0.114 | +| Michigan | tract | median_hh_income | 0.068 0.073 0.108 0.421 (0.694) | 3 | 77 | 0.114 | +| New York | tract | population | 0.038 0.043 0.073 0.417 (0.769) | 3 | 77 | 0.114 | +| New York | tract | households | 0.024 0.026 0.050 0.404 (0.768) | 3 | 77 | 0.114 | +| New York | tract | poverty_count | 0.028 0.032 0.061 0.408 (0.768) | 3 | 77 | 0.114 | +| New York | tract | poverty_rate | 0.030 0.034 0.062 0.410 (0.768) | 3 | 77 | 0.114 | +| New York | tract | owner_share | 0.028 0.031 0.062 0.407 (0.766) | 3 | 77 | 0.114 | +| New York | tract | median_hh_income | 0.081 0.085 0.114 0.422 (0.764) | 3 | 77 | 0.114 | +| Michigan | county | households | 0.000 0.001 0.003 0.004 (-) | not tested (too few areas) | - | - | +| Michigan | county | poverty_count | 0.001 0.002 0.002 0.004 (-) | not tested (too few areas) | - | - | +| Michigan | county | poverty_rate | 0.001 0.002 0.003 0.004 (-) | not tested (too few areas) | - | - | +| Michigan | county | owner_share | 0.000 0.001 0.001 0.004 (-) | not tested (too few areas) | - | - | +| Michigan | county | median_hh_income | 0.000 0.001 0.003 0.004 (-) | not tested (too few areas) | - | - | +| New York | county | households | 0.008 0.016 0.025 0.030 (-) | not tested (too few areas) | - | - | +| New York | county | poverty_count | 0.019 0.034 0.042 0.054 (-) | not tested (too few areas) | - | - | +| New York | county | poverty_rate | 0.019 0.033 0.043 0.055 (-) | not tested (too few areas) | - | - | +| New York | county | owner_share | 0.012 0.019 0.031 0.038 (-) | not tested (too few areas) | - | - | +| New York | county | median_hh_income | 0.018 0.030 0.032 0.059 (-) | not tested (too few areas) | - | - | + +Spread of pairwise r per class: SD, excess SD over the analytic null (sqrt(SD^2 - 1/80)) and over the empirical Michigan x New York null (sqrt(SD^2 - SD_cross^2)), and the share of pairs with |r| above the analytic 5% critical value (0.05 expected under no correlation; about 0.06 in the independent cross-state pairs): + +| state | geography | variable | class | SD of r | excess SD (analytic) | excess SD (cross-state) | share |r| > crit | +| :--- | :--- | :--- | :--- | :--- | :--- | :--- | :--- | +| Michigan | tract | population | order 1 | 0.192 | 0.156 | 0.153 | 0.168 | +| Michigan | tract | population | order 2 | 0.126 | 0.058 | 0.050 | 0.076 | +| Michigan | tract | population | order 3 | 0.118 | 0.039 | 0.026 | 0.063 | +| Michigan | tract | population | beyond order 3 | 0.115 | 0.027 | 0.000 | 0.060 | +| Michigan | tract | households | order 1 | 0.136 | 0.078 | 0.072 | 0.105 | +| Michigan | tract | households | order 2 | 0.120 | 0.042 | 0.030 | 0.069 | +| Michigan | tract | households | order 3 | 0.116 | 0.032 | 0.011 | 0.062 | +| Michigan | tract | households | beyond order 3 | 0.115 | 0.028 | 0.000 | 0.061 | +| Michigan | tract | poverty_count | order 1 | 0.115 | 0.027 | 0.000 | 0.063 | +| Michigan | tract | poverty_count | order 2 | 0.116 | 0.032 | 0.011 | 0.066 | +| Michigan | tract | poverty_count | order 3 | 0.116 | 0.031 | 0.008 | 0.064 | +| Michigan | tract | poverty_count | beyond order 3 | 0.115 | 0.027 | 0.000 | 0.064 | +| Michigan | tract | poverty_rate | order 1 | 0.115 | 0.027 | 0.006 | 0.065 | +| Michigan | tract | poverty_rate | order 2 | 0.116 | 0.032 | 0.017 | 0.065 | +| Michigan | tract | poverty_rate | order 3 | 0.116 | 0.032 | 0.017 | 0.063 | +| Michigan | tract | poverty_rate | beyond order 3 | 0.115 | 0.027 | 0.006 | 0.062 | +| Michigan | tract | owner_share | order 1 | 0.115 | 0.028 | 0.000 | 0.061 | +| Michigan | tract | owner_share | order 2 | 0.117 | 0.033 | 0.012 | 0.062 | +| Michigan | tract | owner_share | order 3 | 0.115 | 0.029 | 0.000 | 0.060 | +| Michigan | tract | owner_share | beyond order 3 | 0.115 | 0.027 | 0.000 | 0.060 | +| Michigan | tract | median_hh_income | order 1 | 0.115 | 0.028 | 0.000 | 0.059 | +| Michigan | tract | median_hh_income | order 2 | 0.115 | 0.027 | 0.000 | 0.057 | +| Michigan | tract | median_hh_income | order 3 | 0.116 | 0.029 | 0.005 | 0.058 | +| Michigan | tract | median_hh_income | beyond order 3 | 0.116 | 0.030 | 0.010 | 0.059 | +| New York | tract | population | order 2 | 0.122 | 0.048 | 0.038 | 0.071 | +| New York | tract | population | order 1 | 0.175 | 0.134 | 0.131 | 0.118 | +| New York | tract | population | order 3 | 0.117 | 0.035 | 0.020 | 0.062 | +| New York | tract | population | beyond order 3 | 0.115 | 0.029 | 0.000 | 0.061 | +| New York | tract | households | order 2 | 0.117 | 0.036 | 0.019 | 0.065 | +| New York | tract | households | order 1 | 0.128 | 0.062 | 0.055 | 0.083 | +| New York | tract | households | order 3 | 0.116 | 0.031 | 0.008 | 0.064 | +| New York | tract | households | beyond order 3 | 0.115 | 0.028 | 0.000 | 0.061 | +| New York | tract | poverty_count | order 2 | 0.116 | 0.029 | 0.000 | 0.066 | +| New York | tract | poverty_count | order 1 | 0.115 | 0.028 | 0.000 | 0.064 | +| New York | tract | poverty_count | order 3 | 0.115 | 0.028 | 0.000 | 0.066 | +| New York | tract | poverty_count | beyond order 3 | 0.115 | 0.029 | 0.000 | 0.065 | +| New York | tract | poverty_rate | order 2 | 0.115 | 0.028 | 0.007 | 0.063 | +| New York | tract | poverty_rate | order 1 | 0.115 | 0.027 | 0.002 | 0.065 | +| New York | tract | poverty_rate | order 3 | 0.115 | 0.028 | 0.008 | 0.063 | +| New York | tract | poverty_rate | beyond order 3 | 0.116 | 0.030 | 0.013 | 0.064 | +| New York | tract | owner_share | order 2 | 0.115 | 0.028 | 0.000 | 0.060 | +| New York | tract | owner_share | order 1 | 0.115 | 0.028 | 0.000 | 0.060 | +| New York | tract | owner_share | order 3 | 0.115 | 0.029 | 0.000 | 0.061 | +| New York | tract | owner_share | beyond order 3 | 0.115 | 0.029 | 0.000 | 0.061 | +| New York | tract | median_hh_income | order 2 | 0.115 | 0.028 | 0.000 | 0.057 | +| New York | tract | median_hh_income | order 1 | 0.115 | 0.027 | 0.000 | 0.054 | +| New York | tract | median_hh_income | order 3 | 0.116 | 0.032 | 0.013 | 0.060 | +| New York | tract | median_hh_income | beyond order 3 | 0.116 | 0.029 | 0.005 | 0.059 | +| Michigan | county | households | order 1 | 0.117 | 0.034 | 0.000 | 0.082 | +| Michigan | county | households | order 3 | 0.120 | 0.043 | 0.000 | 0.079 | +| Michigan | county | households | order 2 | 0.121 | 0.045 | 0.000 | 0.065 | +| Michigan | county | households | beyond order 3 | 0.126 | 0.059 | 0.037 | 0.093 | +| Michigan | county | poverty_count | order 1 | 0.114 | 0.023 | 0.000 | 0.058 | +| Michigan | county | poverty_count | order 3 | 0.116 | 0.029 | 0.000 | 0.059 | +| Michigan | county | poverty_count | order 2 | 0.113 | 0.018 | 0.000 | 0.044 | +| Michigan | county | poverty_count | beyond order 3 | 0.116 | 0.030 | 0.000 | 0.059 | +| Michigan | county | poverty_rate | order 1 | 0.114 | 0.022 | 0.000 | 0.058 | +| Michigan | county | poverty_rate | order 3 | 0.116 | 0.030 | 0.000 | 0.057 | +| Michigan | county | poverty_rate | order 2 | 0.113 | 0.018 | 0.000 | 0.041 | +| Michigan | county | poverty_rate | beyond order 3 | 0.116 | 0.030 | 0.004 | 0.058 | +| Michigan | county | owner_share | order 1 | 0.116 | 0.031 | 0.006 | 0.053 | +| Michigan | county | owner_share | order 3 | 0.117 | 0.034 | 0.016 | 0.064 | +| Michigan | county | owner_share | order 2 | 0.115 | 0.027 | 0.000 | 0.044 | +| Michigan | county | owner_share | beyond order 3 | 0.115 | 0.026 | 0.000 | 0.056 | +| Michigan | county | median_hh_income | order 1 | 0.112 | 0.010 | 0.000 | 0.062 | +| Michigan | county | median_hh_income | order 3 | 0.114 | 0.024 | 0.000 | 0.057 | +| Michigan | county | median_hh_income | order 2 | 0.115 | 0.028 | 0.011 | 0.062 | +| Michigan | county | median_hh_income | beyond order 3 | 0.111 | 0.000 | 0.000 | 0.048 | +| New York | county | households | order 3 | 0.119 | 0.041 | 0.000 | 0.063 | +| New York | county | households | order 1 | 0.112 | 0.000 | 0.000 | 0.049 | +| New York | county | households | order 2 | 0.126 | 0.058 | 0.035 | 0.091 | +| New York | county | households | beyond order 3 | 0.118 | 0.036 | 0.000 | 0.070 | +| New York | county | poverty_count | order 3 | 0.123 | 0.050 | 0.040 | 0.079 | +| New York | county | poverty_count | order 1 | 0.118 | 0.038 | 0.024 | 0.056 | +| New York | county | poverty_count | order 2 | 0.115 | 0.027 | 0.000 | 0.061 | +| New York | county | poverty_count | beyond order 3 | 0.114 | 0.023 | 0.000 | 0.064 | +| New York | county | poverty_rate | order 3 | 0.122 | 0.050 | 0.040 | 0.079 | +| New York | county | poverty_rate | order 1 | 0.119 | 0.041 | 0.028 | 0.049 | +| New York | county | poverty_rate | order 2 | 0.115 | 0.027 | 0.000 | 0.065 | +| New York | county | poverty_rate | beyond order 3 | 0.114 | 0.023 | 0.000 | 0.062 | +| New York | county | owner_share | order 3 | 0.109 | 0.000 | 0.000 | 0.048 | +| New York | county | owner_share | order 1 | 0.115 | 0.028 | 0.000 | 0.063 | +| New York | county | owner_share | order 2 | 0.115 | 0.029 | 0.000 | 0.056 | +| New York | county | owner_share | beyond order 3 | 0.115 | 0.025 | 0.000 | 0.063 | +| New York | county | median_hh_income | order 3 | 0.111 | 0.000 | 0.000 | 0.048 | +| New York | county | median_hh_income | order 1 | 0.117 | 0.034 | 0.023 | 0.063 | +| New York | county | median_hh_income | order 2 | 0.119 | 0.041 | 0.031 | 0.074 | +| New York | county | median_hh_income | beyond order 3 | 0.114 | 0.020 | 0.000 | 0.051 | + +### Split-half reliability of individual pairwise correlations + +Per class: half-A and half-B class means (random split), and the reliability of a single pair's r (correlation across pairs between halves; stepped up to 80 replicates), for each split. + +| state | geography | variable | class | pairs | mean (half A) | mean (half B) | reliability (random / odd-even / first-last) | +| :--- | :--- | :--- | :--- | :--- | :--- | :--- | :--- | +| Michigan | tract | population | order 1 | 9099 | -0.070 | -0.070 | 0.67 / 0.67 / 0.66 | +| Michigan | tract | population | order 2 | 21283 | -0.014 | -0.014 | 0.22 / 0.21 / 0.18 | +| Michigan | tract | population | order 3 | 36300 | -0.003 | -0.005 | 0.11 / 0.11 / 0.09 | +| Michigan | tract | population | beyond order 3 | 225050 | -0.000 | -0.001 | 0.06 / 0.05 / 0.02 | +| Michigan | tract | households | order 1 | 9033 | -0.024 | -0.026 | 0.34 / 0.32 / 0.30 | +| Michigan | tract | households | order 2 | 21067 | -0.007 | -0.007 | 0.14 / 0.11 / 0.08 | +| Michigan | tract | households | order 3 | 35923 | -0.003 | -0.004 | 0.08 / 0.07 / 0.03 | +| Michigan | tract | households | beyond order 3 | 223197 | -0.001 | -0.001 | 0.07 / 0.05 / 0.01 | +| Michigan | tract | poverty_count | order 1 | 9024 | -0.003 | -0.004 | 0.04 / 0.07 / -0.01 | +| Michigan | tract | poverty_count | order 2 | 21028 | -0.003 | -0.003 | 0.07 / 0.08 / 0.02 | +| Michigan | tract | poverty_count | order 3 | 35876 | -0.001 | 0.000 | 0.07 / 0.08 / 0.04 | +| Michigan | tract | poverty_count | beyond order 3 | 222204 | 0.001 | -0.000 | 0.06 / 0.05 / -0.00 | +| Michigan | tract | poverty_rate | order 1 | 8988 | 0.000 | -0.000 | 0.05 / 0.07 / -0.01 | +| Michigan | tract | poverty_rate | order 2 | 20944 | -0.002 | -0.002 | 0.07 / 0.08 / 0.03 | +| Michigan | tract | poverty_rate | order 3 | 35733 | -0.000 | 0.000 | 0.07 / 0.08 / 0.04 | +| Michigan | tract | poverty_rate | beyond order 3 | 221162 | 0.001 | -0.000 | 0.06 / 0.05 / -0.00 | +| Michigan | tract | owner_share | order 1 | 8875 | -0.003 | -0.001 | 0.06 / 0.06 / -0.02 | +| Michigan | tract | owner_share | order 2 | 20622 | 0.000 | 0.001 | 0.07 / 0.07 / 0.05 | +| Michigan | tract | owner_share | order 3 | 35161 | -0.001 | -0.000 | 0.06 / 0.05 / 0.01 | +| Michigan | tract | owner_share | beyond order 3 | 217458 | 0.000 | -0.000 | 0.06 / 0.05 / -0.00 | +| Michigan | tract | median_hh_income | order 1 | 8892 | -0.001 | 0.001 | 0.06 / 0.06 / 0.03 | +| Michigan | tract | median_hh_income | order 2 | 20659 | 0.000 | 0.001 | 0.06 / 0.06 / 0.03 | +| Michigan | tract | median_hh_income | order 3 | 35172 | 0.001 | 0.001 | 0.06 / 0.06 / 0.04 | +| Michigan | tract | median_hh_income | beyond order 3 | 217954 | 0.001 | 0.001 | 0.07 / 0.07 / 0.05 | +| New York | tract | population | order 2 | 37762 | -0.009 | -0.008 | 0.16 / 0.16 / 0.12 | +| New York | tract | population | order 1 | 15821 | -0.039 | -0.041 | 0.61 / 0.60 / 0.58 | +| New York | tract | population | order 3 | 65290 | -0.003 | -0.003 | 0.10 / 0.09 / 0.05 | +| New York | tract | population | beyond order 3 | 230255 | -0.001 | -0.001 | 0.07 / 0.06 / 0.01 | +| New York | tract | households | order 2 | 37281 | -0.004 | -0.003 | 0.09 / 0.09 / 0.05 | +| New York | tract | households | order 1 | 15668 | -0.014 | -0.016 | 0.25 / 0.21 / 0.20 | +| New York | tract | households | order 3 | 64393 | -0.003 | -0.002 | 0.09 / 0.06 / 0.00 | +| New York | tract | households | beyond order 3 | 227670 | -0.001 | -0.001 | 0.06 / 0.05 / -0.00 | +| New York | tract | poverty_count | order 2 | 37221 | -0.000 | -0.001 | 0.06 / 0.05 / 0.02 | +| New York | tract | poverty_count | order 1 | 15646 | -0.001 | -0.002 | 0.05 / 0.05 / -0.02 | +| New York | tract | poverty_count | order 3 | 64288 | 0.001 | -0.000 | 0.06 / 0.05 / -0.00 | +| New York | tract | poverty_count | beyond order 3 | 226711 | -0.000 | 0.000 | 0.06 / 0.06 / -0.00 | +| New York | tract | poverty_rate | order 2 | 37151 | 0.001 | -0.001 | 0.06 / 0.04 / 0.01 | +| New York | tract | poverty_rate | order 1 | 15627 | 0.001 | -0.001 | 0.04 / 0.03 / -0.02 | +| New York | tract | poverty_rate | order 3 | 64145 | 0.000 | -0.000 | 0.05 / 0.05 / 0.00 | +| New York | tract | poverty_rate | beyond order 3 | 226417 | 0.000 | 0.000 | 0.07 / 0.06 / 0.00 | +| New York | tract | owner_share | order 2 | 35819 | -0.001 | 0.001 | 0.05 / 0.04 / -0.00 | +| New York | tract | owner_share | order 1 | 15121 | -0.001 | -0.000 | 0.06 / 0.06 / -0.02 | +| New York | tract | owner_share | order 3 | 61842 | 0.001 | 0.001 | 0.06 / 0.06 / -0.01 | +| New York | tract | owner_share | beyond order 3 | 217364 | 0.000 | 0.001 | 0.06 / 0.05 / 0.01 | +| New York | tract | median_hh_income | order 2 | 34551 | 0.000 | 0.002 | 0.07 / 0.06 / 0.04 | +| New York | tract | median_hh_income | order 1 | 14672 | -0.000 | 0.000 | 0.02 / 0.04 / 0.03 | +| New York | tract | median_hh_income | order 3 | 59637 | -0.000 | 0.000 | 0.07 / 0.08 / 0.04 | +| New York | tract | median_hh_income | beyond order 3 | 210794 | 0.000 | 0.001 | 0.07 / 0.07 / 0.04 | +| Michigan | county | households | order 1 | 208 | 0.003 | 0.052 | 0.09 / -0.04 / 0.14 | +| Michigan | county | households | order 3 | 404 | 0.019 | 0.035 | 0.10 / 0.17 / 0.15 | +| Michigan | county | households | order 2 | 339 | 0.028 | 0.035 | 0.18 / 0.24 / 0.18 | +| Michigan | county | households | beyond order 3 | 2452 | 0.029 | 0.036 | 0.29 / 0.26 / 0.25 | +| Michigan | county | poverty_count | order 1 | 208 | -0.011 | 0.001 | -0.03 / 0.11 / -0.09 | +| Michigan | county | poverty_count | order 3 | 404 | -0.005 | 0.014 | 0.11 / 0.06 / 0.03 | +| Michigan | county | poverty_count | order 2 | 339 | 0.007 | 0.012 | 0.14 / 0.01 / -0.10 | +| Michigan | county | poverty_count | beyond order 3 | 2452 | 0.002 | 0.001 | 0.04 / 0.02 / 0.04 | +| Michigan | county | poverty_rate | order 1 | 208 | -0.012 | 0.001 | -0.03 / 0.11 / -0.10 | +| Michigan | county | poverty_rate | order 3 | 404 | -0.005 | 0.014 | 0.11 / 0.06 / 0.04 | +| Michigan | county | poverty_rate | order 2 | 339 | 0.008 | 0.012 | 0.14 / 0.00 / -0.11 | +| Michigan | county | poverty_rate | beyond order 3 | 2452 | 0.002 | 0.000 | 0.04 / 0.02 / 0.04 | +| Michigan | county | owner_share | order 1 | 208 | -0.001 | -0.011 | -0.03 / 0.12 / 0.03 | +| Michigan | county | owner_share | order 3 | 404 | 0.014 | 0.031 | 0.13 / 0.05 / 0.17 | +| Michigan | county | owner_share | order 2 | 339 | 0.025 | 0.009 | -0.01 / 0.07 / 0.01 | +| Michigan | county | owner_share | beyond order 3 | 2452 | -0.000 | 0.004 | 0.04 / 0.01 / 0.01 | +| Michigan | county | median_hh_income | order 1 | 208 | 0.013 | -0.008 | 0.14 / 0.05 / -0.06 | +| Michigan | county | median_hh_income | order 3 | 404 | 0.002 | 0.003 | 0.17 / -0.03 / -0.02 | +| Michigan | county | median_hh_income | order 2 | 339 | -0.004 | 0.009 | 0.08 / 0.13 / 0.14 | +| Michigan | county | median_hh_income | beyond order 3 | 2452 | 0.001 | -0.001 | 0.01 / -0.06 / -0.06 | +| New York | county | households | order 3 | 252 | 0.025 | 0.026 | 0.23 / 0.26 / 0.25 | +| New York | county | households | order 1 | 142 | 0.029 | 0.013 | 0.11 / 0.24 / 0.00 | +| New York | county | households | order 2 | 231 | 0.041 | 0.006 | 0.18 / 0.34 / 0.27 | +| New York | county | households | beyond order 3 | 1266 | 0.036 | 0.026 | 0.17 / 0.10 / 0.10 | +| New York | county | poverty_count | order 3 | 252 | 0.019 | 0.017 | 0.05 / 0.12 / 0.15 | +| New York | county | poverty_count | order 1 | 142 | 0.011 | -0.004 | 0.36 / -0.08 / 0.26 | +| New York | county | poverty_count | order 2 | 231 | 0.011 | -0.000 | 0.08 / 0.13 / -0.02 | +| New York | county | poverty_count | beyond order 3 | 1266 | 0.002 | 0.014 | 0.07 / 0.05 / -0.04 | +| New York | county | poverty_rate | order 3 | 252 | 0.019 | 0.017 | 0.05 / 0.12 / 0.14 | +| New York | county | poverty_rate | order 1 | 142 | 0.012 | -0.005 | 0.37 / -0.06 / 0.27 | +| New York | county | poverty_rate | order 2 | 231 | 0.012 | -0.000 | 0.06 / 0.13 / -0.03 | +| New York | county | poverty_rate | beyond order 3 | 1266 | 0.003 | 0.014 | 0.07 / 0.05 / -0.04 | +| New York | county | owner_share | order 3 | 252 | -0.002 | 0.000 | -0.10 / 0.01 / -0.09 | +| New York | county | owner_share | order 1 | 142 | 0.003 | -0.006 | 0.15 / 0.10 / 0.13 | +| New York | county | owner_share | order 2 | 231 | -0.005 | 0.024 | 0.05 / 0.15 / 0.06 | +| New York | county | owner_share | beyond order 3 | 1266 | 0.019 | 0.006 | 0.02 / 0.09 / 0.08 | +| New York | county | median_hh_income | order 3 | 252 | 0.001 | 0.007 | 0.05 / -0.05 / -0.10 | +| New York | county | median_hh_income | order 1 | 142 | 0.021 | -0.007 | 0.04 / 0.20 / 0.17 | +| New York | county | median_hh_income | order 2 | 231 | -0.016 | 0.001 | 0.10 / 0.02 / -0.01 | +| New York | county | median_hh_income | beyond order 3 | 1266 | 0.009 | 0.015 | 0.03 / 0.05 / -0.06 | + +### Held-out replicate log-likelihood gain over independence (nats per replicate) + +| state | geography | variable | areas | order (1-3) | order x county | order x county (+ far within county) | ledoit_wolf | fitted order-1 r (halves) | LW intensity (40 reps) | LW intensity (80 reps) | rank / areas | +| :--- | :--- | :--- | :--- | :--- | :--- | :--- | :--- | :--- | :--- | :--- | :--- | +| Michigan | tract | population | 2914 | 32.87 | -164.87 (projected) | -551.70 (projected) | -25.00 | -0.070 | 0.870 | 0.924 | 80 / 2914 | +| Michigan | tract | households | 2897 | 4.41 | 5.07 | -165.60 (projected) | -27.34 | -0.026 | 0.871 | 0.931 | 80 / 2897 | +| Michigan | tract | poverty_count | 2893 | 0.18 | 0.16 | 0.15 | -28.70 | -0.004 | 0.870 | 0.930 | 80 / 2893 | +| Michigan | tract | poverty_rate | 2887 | 0.05 | -0.04 | -0.05 | -28.20 | 0.000 | 0.870 | 0.929 | 80 / 2887 | +| Michigan | tract | owner_share | 2865 | -0.00 | -0.00 | -0.03 | -28.50 | -0.002 | 0.870 | 0.931 | 80 / 2865 | +| Michigan | tract | median_hh_income | 2870 | -0.02 | -0.10 | -0.14 | -36.92 | -0.001 | 0.872 | 0.930 | 80 / 2870 | +| New York | tract | population | 5289 | 16.85 | 17.15 | -244.21 (projected) | -36.87 | -0.040 | 0.869 | 0.928 | 80 / 5289 | +| New York | tract | households | 5251 | 2.20 | 2.27 | -373.56 (projected) | -37.33 | -0.015 | 0.870 | 0.929 | 80 / 5251 | +| New York | tract | poverty_count | 5242 | -0.04 | -0.10 | -0.10 | -37.16 | -0.002 | 0.870 | 0.927 | 80 / 5242 | +| New York | tract | poverty_rate | 5238 | -0.09 | -0.13 | -0.13 | -38.19 | 0.000 | 0.870 | 0.927 | 80 / 5238 | +| New York | tract | owner_share | 5145 | -0.02 | -0.06 | -0.06 | -40.09 | -0.001 | 0.869 | 0.928 | 80 / 5145 | +| New York | tract | median_hh_income | 5053 | -0.04 | -0.08 | -0.09 | -49.58 | -0.001 | 0.872 | 0.927 | 80 / 5053 | +| Michigan | county | households | 83 | -0.10 | - | - | -0.17 | -0.000 | 0.910 | 0.764 | 80 / 83 | +| Michigan | county | poverty_count | 83 | -0.03 | - | - | -0.35 | -0.008 | 0.896 | 0.952 | 80 / 83 | +| Michigan | county | poverty_rate | 83 | -0.03 | - | - | -0.36 | -0.008 | 0.894 | 0.950 | 80 / 83 | +| Michigan | county | owner_share | 83 | -0.04 | - | - | -0.54 | -0.011 | 0.881 | 0.950 | 80 / 83 | +| Michigan | county | median_hh_income | 83 | -0.06 | - | - | -0.22 | 0.003 | 0.931 | 1.000 | 80 / 83 | +| New York | county | households | 62 | -0.08 | - | - | -0.12 | -0.007 | 0.893 | 0.852 | 62 / 62 | +| New York | county | poverty_count | 62 | -0.05 | - | - | -0.07 | -0.005 | 0.911 | 0.953 | 62 / 62 | +| New York | county | poverty_rate | 62 | -0.05 | - | - | -0.08 | -0.005 | 0.910 | 0.953 | 62 / 62 | +| New York | county | owner_share | 62 | -0.08 | - | - | -0.52 | -0.014 | 0.883 | 0.971 | 62 / 62 | +| New York | county | median_hh_income | 62 | -0.06 | - | - | -0.18 | -0.001 | 0.901 | 0.973 | 62 / 62 | + +## Results 2015-2019 + +Mean pairwise r [95% interval] (number of pairs) by queen contiguity order. + +### Tracts: contiguity order + +| state | variable | order 1 | order 2 | order 3 | beyond order 3 | +| :--- | :--- | :--- | :--- | :--- | :--- | +| Michigan | households | 0.025 [0.016, 0.033] (n=8614) | 0.006 [0.000, 0.013] (n=20000) | 0.001 [-0.002, 0.005] (n=33974) | -0.000 [-0.001, 0.001] (n=234537) | +| Michigan | poverty_count | -0.004 [-0.007, -0.001] (n=8601) | -0.002 [-0.004, 0.000] (n=19957) | 0.000 [-0.002, 0.002] (n=33911) | -0.000 [-0.000, 0.000] (n=234178) | +| Michigan | poverty_rate | -0.000 [-0.003, 0.003] (n=8581) | -0.001 [-0.003, 0.001] (n=19916) | 0.000 [-0.002, 0.002] (n=33847) | 0.000 [-0.001, 0.001] (n=233863) | +| Michigan | owner_share | -0.001 [-0.004, 0.001] (n=8503) | -0.001 [-0.004, 0.002] (n=19730) | 0.001 [-0.000, 0.002] (n=33514) | 0.000 [-0.001, 0.001] (n=231151) | +| Michigan | median_hh_income | 0.002 [-0.001, 0.004] (n=8563) | -0.000 [-0.003, 0.003] (n=19869) | 0.001 [-0.001, 0.002] (n=33766) | 0.001 [-0.000, 0.002] (n=232936) | +| New York | population | -0.039 [-0.060, -0.018] (n=14629) | -0.008 [-0.013, -0.003] (n=35303) | -0.002 [-0.004, -0.001] (n=61223) | -0.001 [-0.002, -0.000] (n=236096) | +| New York | households | 0.012 [0.003, 0.020] (n=14502) | 0.005 [0.002, 0.008] (n=34948) | 0.002 [0.001, 0.004] (n=60587) | -0.000 [-0.001, 0.001] (n=234210) | +| New York | poverty_count | -0.003 [-0.005, -0.001] (n=14431) | -0.000 [-0.002, 0.001] (n=34710) | -0.001 [-0.002, 0.000] (n=60162) | 0.000 [-0.000, 0.001] (n=232610) | +| New York | poverty_rate | -0.001 [-0.002, 0.001] (n=14378) | -0.000 [-0.001, 0.001] (n=34613) | -0.001 [-0.002, 0.001] (n=60035) | 0.000 [-0.000, 0.001] (n=232123) | +| New York | owner_share | 0.000 [-0.002, 0.002] (n=14033) | 0.000 [-0.001, 0.002] (n=33723) | 0.000 [-0.001, 0.002] (n=58430) | 0.000 [-0.000, 0.001] (n=225550) | +| New York | median_hh_income | 0.002 [-0.001, 0.004] (n=14048) | 0.000 [-0.001, 0.002] (n=33653) | 0.001 [-0.001, 0.002] (n=58491) | 0.000 [-0.000, 0.001] (n=227384) | + +### Counties: contiguity order + +| state | variable | order 1 | order 2 | order 3 | beyond order 3 | +| :--- | :--- | :--- | :--- | :--- | :--- | +| Michigan | households | 0.023 [-0.004, 0.050] (n=208) | 0.027 [0.008, 0.047] (n=339) | 0.016 [-0.002, 0.035] (n=404) | 0.023 [0.009, 0.038] (n=2452) | +| Michigan | poverty_count | -0.007 [-0.031, 0.016] (n=208) | 0.011 [-0.006, 0.028] (n=339) | 0.008 [-0.005, 0.022] (n=404) | 0.003 [-0.005, 0.010] (n=2452) | +| Michigan | poverty_rate | -0.007 [-0.030, 0.016] (n=208) | 0.012 [-0.005, 0.028] (n=339) | 0.008 [-0.005, 0.022] (n=404) | 0.003 [-0.005, 0.010] (n=2452) | +| Michigan | owner_share | -0.001 [-0.023, 0.021] (n=208) | 0.016 [-0.003, 0.036] (n=339) | 0.005 [-0.010, 0.021] (n=404) | 0.006 [-0.002, 0.014] (n=2452) | +| Michigan | median_hh_income | 0.005 [-0.016, 0.026] (n=208) | 0.008 [-0.008, 0.024] (n=339) | 0.003 [-0.012, 0.018] (n=404) | 0.001 [-0.008, 0.009] (n=2452) | +| New York | households | 0.015 [-0.017, 0.047] (n=142) | 0.005 [-0.023, 0.032] (n=231) | 0.028 [0.008, 0.049] (n=252) | 0.031 [0.013, 0.050] (n=1266) | +| New York | poverty_count | 0.014 [-0.014, 0.043] (n=142) | 0.001 [-0.019, 0.021] (n=231) | -0.001 [-0.021, 0.019] (n=252) | 0.007 [-0.001, 0.016] (n=1266) | +| New York | poverty_rate | 0.016 [-0.013, 0.044] (n=142) | 0.001 [-0.019, 0.020] (n=231) | -0.002 [-0.022, 0.019] (n=252) | 0.007 [-0.001, 0.016] (n=1266) | +| New York | owner_share | 0.028 [-0.007, 0.063] (n=142) | 0.025 [0.002, 0.047] (n=231) | 0.018 [-0.008, 0.043] (n=252) | 0.036 [0.018, 0.055] (n=1266) | +| New York | median_hh_income | 0.024 [-0.003, 0.052] (n=142) | 0.006 [-0.019, 0.030] (n=231) | 0.004 [-0.019, 0.028] (n=252) | 0.011 [-0.001, 0.022] (n=1266) | + +### Tracts: within vs across county (weighting controls) + +| state | variable | order 1 / within county | order 1 / across county | order 2 / within county | order 2 / across county | beyond order 3 / within county | beyond order 3 / across county | +| :--- | :--- | :--- | :--- | :--- | :--- | :--- | :--- | +| Michigan | households | 0.028 [0.020, 0.035] (n=7563) | 0.001 [-0.012, 0.014] (n=1051) | 0.008 [0.001, 0.016] (n=14828) | 0.001 [-0.003, 0.005] (n=5172) | -0.001 [-0.003, 0.002] (n=146625) | 0.000 [-0.001, 0.001] (n=87912) | +| Michigan | poverty_count | -0.005 [-0.008, -0.002] (n=7549) | 0.001 [-0.007, 0.010] (n=1052) | -0.001 [-0.003, 0.001] (n=14793) | -0.003 [-0.008, 0.001] (n=5164) | 0.000 [-0.000, 0.001] (n=146388) | -0.001 [-0.002, 0.000] (n=87790) | +| Michigan | poverty_rate | -0.000 [-0.003, 0.002] (n=7531) | 0.001 [-0.007, 0.009] (n=1050) | 0.000 [-0.002, 0.003] (n=14758) | -0.003 [-0.007, 0.002] (n=5158) | 0.001 [0.000, 0.001] (n=146203) | -0.001 [-0.002, 0.000] (n=87660) | +| Michigan | owner_share | -0.001 [-0.004, 0.002] (n=7458) | -0.005 [-0.015, 0.005] (n=1045) | -0.000 [-0.005, 0.004] (n=14587) | -0.002 [-0.006, 0.003] (n=5143) | 0.000 [-0.000, 0.001] (n=144184) | -0.000 [-0.001, 0.001] (n=86967) | +| Michigan | median_hh_income | 0.001 [-0.001, 0.004] (n=7512) | 0.005 [-0.006, 0.016] (n=1051) | -0.000 [-0.004, 0.004] (n=14702) | -0.000 [-0.004, 0.004] (n=5167) | 0.001 [-0.001, 0.003] (n=145491) | 0.001 [-0.001, 0.002] (n=87445) | +| New York | population | -0.042 [-0.065, -0.018] (n=13624) | 0.000 [-0.008, 0.009] (n=1005) | -0.010 [-0.016, -0.004] (n=29760) | -0.000 [-0.004, 0.004] (n=5543) | -0.002 [-0.002, -0.001] (n=145264) | 0.000 [-0.001, 0.001] (n=90832) | +| New York | households | 0.013 [0.003, 0.022] (n=13506) | -0.004 [-0.013, 0.006] (n=996) | 0.007 [0.004, 0.009] (n=29494) | -0.003 [-0.010, 0.003] (n=5454) | -0.001 [-0.002, 0.000] (n=144341) | 0.001 [-0.001, 0.003] (n=89869) | +| New York | poverty_count | -0.003 [-0.005, -0.001] (n=13441) | 0.004 [-0.006, 0.013] (n=990) | -0.000 [-0.002, 0.001] (n=29280) | 0.000 [-0.004, 0.004] (n=5430) | -0.000 [-0.001, 0.001] (n=143413) | 0.000 [-0.001, 0.001] (n=89197) | +| New York | poverty_rate | -0.001 [-0.003, 0.001] (n=13388) | 0.003 [-0.008, 0.013] (n=990) | -0.000 [-0.001, 0.001] (n=29185) | 0.000 [-0.003, 0.004] (n=5428) | 0.000 [-0.001, 0.001] (n=143103) | 0.000 [-0.001, 0.001] (n=89020) | +| New York | owner_share | 0.000 [-0.002, 0.002] (n=13062) | 0.001 [-0.009, 0.011] (n=971) | 0.000 [-0.001, 0.002] (n=28439) | 0.002 [-0.003, 0.006] (n=5284) | 0.001 [-0.000, 0.001] (n=138644) | 0.000 [-0.001, 0.002] (n=86906) | +| New York | median_hh_income | 0.001 [-0.001, 0.003] (n=13070) | 0.009 [-0.003, 0.021] (n=978) | 0.000 [-0.001, 0.002] (n=28295) | 0.000 [-0.004, 0.005] (n=5358) | 0.000 [-0.001, 0.001] (n=140550) | 0.000 [-0.001, 0.001] (n=86834) | + +### Rook (shared edge) vs queen-only (corner) first-order neighbours + +Tracts: + +| state | variable | order 1 rook (shared edge) | order 1 queen-only (corner) | +| :--- | :--- | :--- | :--- | +| Michigan | households | 0.025 [0.017, 0.034] (n=7033) | 0.021 [0.009, 0.033] (n=1581) | +| Michigan | poverty_count | -0.005 [-0.008, -0.001] (n=7018) | -0.002 [-0.008, 0.004] (n=1583) | +| Michigan | poverty_rate | -0.000 [-0.003, 0.003] (n=7001) | 0.001 [-0.006, 0.007] (n=1580) | +| Michigan | owner_share | -0.001 [-0.004, 0.002] (n=6938) | -0.005 [-0.010, 0.001] (n=1565) | +| Michigan | median_hh_income | 0.001 [-0.002, 0.004] (n=6986) | 0.004 [-0.003, 0.010] (n=1577) | +| New York | population | -0.043 [-0.065, -0.021] (n=12901) | -0.009 [-0.018, 0.001] (n=1728) | +| New York | households | 0.013 [0.003, 0.022] (n=12783) | 0.004 [-0.004, 0.012] (n=1719) | +| New York | poverty_count | -0.003 [-0.005, -0.000] (n=12720) | -0.005 [-0.010, 0.001] (n=1711) | +| New York | poverty_rate | -0.000 [-0.002, 0.002] (n=12672) | -0.004 [-0.010, 0.002] (n=1706) | +| New York | owner_share | -0.001 [-0.003, 0.002] (n=12368) | 0.008 [0.000, 0.015] (n=1665) | +| New York | median_hh_income | 0.002 [-0.001, 0.004] (n=12379) | 0.001 [-0.008, 0.009] (n=1669) | + +Counties: + +| state | variable | order 1 rook (shared edge) | order 1 queen-only (corner) | +| :--- | :--- | :--- | :--- | +| Michigan | households | 0.019 [-0.012, 0.050] (n=180) | 0.049 [0.003, 0.095] (n=28) | +| Michigan | poverty_count | -0.005 [-0.031, 0.020] (n=180) | -0.020 [-0.074, 0.033] (n=28) | +| Michigan | poverty_rate | -0.005 [-0.030, 0.021] (n=180) | -0.020 [-0.073, 0.032] (n=28) | +| Michigan | owner_share | 0.007 [-0.016, 0.029] (n=180) | -0.051 [-0.098, -0.004] (n=28) | +| Michigan | median_hh_income | 0.002 [-0.021, 0.025] (n=180) | 0.025 [-0.025, 0.075] (n=28) | +| New York | households | 0.015 [-0.017, 0.048] (n=140) | -0.013 [-0.213, 0.187] (n=2) | +| New York | poverty_count | 0.015 [-0.013, 0.044] (n=140) | -0.049 [-0.640, 0.541] (n=2) | +| New York | poverty_rate | 0.017 [-0.012, 0.045] (n=140) | -0.053 [-0.688, 0.582] (n=2) | +| New York | owner_share | 0.027 [-0.008, 0.063] (n=140) | 0.071 [-0.123, 0.264] (n=2) | +| New York | median_hh_income | 0.023 [-0.004, 0.050] (n=140) | 0.135 [-0.144, 0.414] (n=2) | + +### By centroid distance + +Tracts: + +| state | variable | 0-2 km | 2-5 km | 5-10 km | 10-20 km | 20-50 km | 50-100 km | 100-inf km | +| :--- | :--- | :--- | :--- | :--- | :--- | :--- | :--- | :--- | +| Michigan | households | 0.032 [0.006, 0.058] (n=3786) | 0.014 [0.005, 0.023] (n=18947) | 0.003 [0.001, 0.006] (n=35548) | -0.002 [-0.005, 0.002] (n=69531) | -0.001 [-0.003, 0.000] (n=91378) | -0.001 [-0.004, 0.002] (n=20313) | 0.001 [-0.001, 0.002] (n=57622) | +| Michigan | poverty_count | -0.003 [-0.010, 0.004] (n=3789) | -0.002 [-0.005, 0.001] (n=18945) | -0.001 [-0.002, -0.000] (n=35516) | 0.000 [-0.001, 0.001] (n=69418) | 0.000 [-0.000, 0.001] (n=91157) | -0.000 [-0.003, 0.002] (n=20216) | -0.000 [-0.002, 0.001] (n=57606) | +| Michigan | poverty_rate | 0.001 [-0.006, 0.008] (n=3778) | -0.001 [-0.002, 0.001] (n=18895) | -0.001 [-0.002, 0.000] (n=35416) | 0.001 [-0.000, 0.001] (n=69298) | 0.000 [-0.000, 0.001] (n=91079) | -0.001 [-0.003, 0.002] (n=20196) | -0.000 [-0.002, 0.001] (n=57545) | +| Michigan | owner_share | -0.001 [-0.006, 0.004] (n=3726) | -0.000 [-0.005, 0.005] (n=18628) | 0.001 [-0.001, 0.003] (n=34906) | 0.000 [-0.001, 0.001] (n=68456) | 0.000 [-0.000, 0.001] (n=89982) | 0.001 [-0.002, 0.004] (n=20075) | -0.001 [-0.002, 0.000] (n=57125) | +| Michigan | median_hh_income | 0.001 [-0.003, 0.004] (n=3753) | 0.000 [-0.004, 0.004] (n=18798) | 0.001 [-0.001, 0.002] (n=35254) | 0.002 [-0.000, 0.003] (n=69047) | 0.001 [-0.000, 0.002] (n=90750) | 0.001 [-0.002, 0.004] (n=20151) | 0.001 [-0.000, 0.002] (n=57381) | +| New York | population | -0.005 [-0.013, 0.003] (n=41793) | -0.008 [-0.015, -0.001] (n=60659) | -0.004 [-0.008, 0.000] (n=77159) | -0.002 [-0.005, -0.000] (n=58451) | -0.002 [-0.004, -0.000] (n=39904) | -0.001 [-0.003, 0.001] (n=14162) | -0.000 [-0.002, 0.002] (n=55123) | +| New York | households | 0.006 [-0.001, 0.013] (n=41584) | 0.005 [0.000, 0.009] (n=60046) | 0.000 [-0.002, 0.003] (n=76635) | -0.002 [-0.003, -0.000] (n=57971) | -0.001 [-0.002, 0.001] (n=39492) | 0.000 [-0.004, 0.004] (n=13996) | 0.001 [-0.001, 0.003] (n=54523) | +| New York | poverty_count | -0.001 [-0.002, 0.000] (n=41371) | -0.001 [-0.002, 0.000] (n=59777) | -0.000 [-0.001, 0.001] (n=76082) | -0.001 [-0.002, 0.000] (n=57471) | -0.000 [-0.001, 0.001] (n=39165) | 0.000 [-0.004, 0.005] (n=13940) | 0.001 [-0.000, 0.002] (n=54107) | +| New York | poverty_rate | -0.001 [-0.002, 0.000] (n=41258) | 0.000 [-0.001, 0.001] (n=59610) | 0.000 [-0.001, 0.001] (n=75907) | -0.001 [-0.002, 0.001] (n=57343) | 0.000 [-0.001, 0.001] (n=39104) | 0.000 [-0.005, 0.005] (n=13925) | 0.001 [-0.000, 0.002] (n=54002) | +| New York | owner_share | 0.001 [-0.001, 0.002] (n=39501) | 0.001 [-0.001, 0.002] (n=57517) | 0.000 [-0.001, 0.001] (n=73645) | 0.000 [-0.001, 0.002] (n=55901) | 0.001 [-0.001, 0.002] (n=38467) | -0.001 [-0.003, 0.002] (n=13691) | 0.000 [-0.001, 0.002] (n=53014) | +| New York | median_hh_income | 0.001 [-0.000, 0.003] (n=40628) | 0.001 [-0.000, 0.001] (n=58252) | 0.000 [-0.001, 0.001] (n=74411) | -0.000 [-0.001, 0.001] (n=55926) | 0.001 [-0.001, 0.002] (n=37726) | -0.001 [-0.004, 0.002] (n=13563) | 0.000 [-0.001, 0.002] (n=53070) | + +Counties: + +| state | variable | 0-50 km | 50-100 km | 100-200 km | 200-400 km | 400-inf km | +| :--- | :--- | :--- | :--- | :--- | :--- | :--- | +| Michigan | households | 0.018 [-0.015, 0.051] (n=139) | 0.026 [0.008, 0.045] (n=375) | 0.024 [0.007, 0.040] (n=1022) | 0.015 [0.000, 0.029] (n=1309) | 0.040 [0.013, 0.066] (n=558) | +| Michigan | poverty_count | -0.000 [-0.030, 0.029] (n=139) | 0.002 [-0.016, 0.019] (n=375) | 0.010 [0.000, 0.020] (n=1022) | 0.001 [-0.009, 0.012] (n=1309) | -0.001 [-0.018, 0.016] (n=558) | +| Michigan | poverty_rate | 0.000 [-0.029, 0.030] (n=139) | 0.002 [-0.015, 0.020] (n=375) | 0.010 [0.000, 0.020] (n=1022) | 0.001 [-0.009, 0.012] (n=1309) | -0.001 [-0.018, 0.015] (n=558) | +| Michigan | owner_share | 0.008 [-0.018, 0.034] (n=139) | 0.011 [-0.007, 0.028] (n=375) | 0.010 [0.000, 0.020] (n=1022) | 0.006 [-0.004, 0.016] (n=1309) | -0.001 [-0.017, 0.015] (n=558) | +| Michigan | median_hh_income | -0.003 [-0.029, 0.024] (n=139) | 0.008 [-0.007, 0.023] (n=375) | 0.008 [-0.004, 0.019] (n=1022) | 0.001 [-0.010, 0.011] (n=1309) | -0.009 [-0.026, 0.009] (n=558) | +| New York | households | 0.034 [-0.022, 0.089] (n=91) | 0.026 [-0.004, 0.056] (n=235) | 0.017 [-0.001, 0.036] (n=537) | 0.027 [0.007, 0.046] (n=867) | 0.052 [0.010, 0.093] (n=161) | +| New York | poverty_count | 0.002 [-0.033, 0.037] (n=91) | 0.007 [-0.009, 0.023] (n=235) | 0.005 [-0.009, 0.019] (n=537) | 0.005 [-0.005, 0.015] (n=867) | 0.015 [-0.009, 0.040] (n=161) | +| New York | poverty_rate | 0.003 [-0.033, 0.038] (n=91) | 0.007 [-0.009, 0.023] (n=235) | 0.005 [-0.008, 0.019] (n=537) | 0.005 [-0.005, 0.015] (n=867) | 0.014 [-0.011, 0.040] (n=161) | +| New York | owner_share | 0.046 [-0.006, 0.099] (n=91) | 0.033 [0.003, 0.063] (n=235) | 0.023 [0.003, 0.043] (n=537) | 0.031 [0.013, 0.050] (n=867) | 0.054 [0.013, 0.095] (n=161) | +| New York | median_hh_income | 0.020 [-0.011, 0.051] (n=91) | 0.026 [0.006, 0.046] (n=235) | 0.012 [-0.007, 0.030] (n=537) | 0.007 [-0.006, 0.020] (n=867) | -0.004 [-0.030, 0.023] (n=161) | + +### Aggregation ratio: Var(county sum of tracts) / sum of tract variances + +| state | variable | counties | tracts | pooled ratio (SE) | median county ratio | implied mean within-county r (SE) | +| :--- | :--- | :--- | :--- | :--- | :--- | :--- | +| Michigan | households | 82 | 2737 | 0.902 (0.053) | 0.798 | -0.0006 (0.0003) | +| Michigan | poverty_count | 83 | 2736 | 0.886 (0.043) | 0.939 | -0.0006 (0.0002) | +| New York | population | 62 | 4854 | 0.000 (0.000) | 0.000 | -0.0032 (0.0000) | +| New York | households | 62 | 4828 | 0.997 (0.056) | 0.806 | -0.0000 (0.0002) | +| New York | poverty_count | 62 | 4811 | 0.944 (0.041) | 0.893 | -0.0002 (0.0002) | + +### Noise floor and spread + +Analytic null (80 replicates): SD 0.112, 95% |r| 0.219. + +Michigan x New York pairs (independent samples): + +| geography | variable | pairs | mean r (SE) | SD of r | 95% |r| | share |r| > analytic critical | +| :--- | :--- | :--- | :--- | :--- | :--- | :--- | +| tract | households | 50000 | 0.0003 (0.0005) | 0.116 | 0.229 | 0.060 | +| tract | poverty_count | 50000 | -0.0005 (0.0004) | 0.115 | 0.231 | 0.062 | +| tract | poverty_rate | 50000 | -0.0007 (0.0004) | 0.116 | 0.234 | 0.063 | +| tract | owner_share | 50000 | -0.0002 (0.0004) | 0.115 | 0.226 | 0.058 | +| tract | median_hh_income | 50000 | 0.0014 (0.0005) | 0.116 | 0.227 | 0.059 | +| county | households | 5146 | 0.0285 (0.0028) | 0.120 | 0.240 | 0.078 | +| county | poverty_count | 5146 | 0.0055 (0.0020) | 0.116 | 0.228 | 0.061 | +| county | poverty_rate | 5146 | 0.0055 (0.0020) | 0.116 | 0.229 | 0.063 | +| county | owner_share | 5146 | 0.0184 (0.0027) | 0.115 | 0.228 | 0.060 | +| county | median_hh_income | 5146 | 0.0074 (0.0022) | 0.117 | 0.231 | 0.064 | + +Derangement null on far pairs (same state, beyond order 3): + +| state | geography | variable | pairs | mean | SD | 95% |r| | +| :--- | :--- | :--- | :--- | :--- | :--- | :--- | +| Michigan | tract | households | 234537 | 0.0004 | 0.113 | 0.222 | +| Michigan | tract | poverty_count | 234178 | 0.0002 | 0.112 | 0.219 | +| Michigan | tract | poverty_rate | 233863 | 0.0003 | 0.112 | 0.219 | +| Michigan | tract | owner_share | 231151 | 0.0004 | 0.112 | 0.218 | +| Michigan | tract | median_hh_income | 232936 | 0.0007 | 0.112 | 0.219 | +| New York | tract | population | 236096 | -0.0001 | 0.112 | 0.219 | +| New York | tract | households | 234210 | 0.0004 | 0.113 | 0.221 | +| New York | tract | poverty_count | 232610 | 0.0001 | 0.112 | 0.219 | +| New York | tract | poverty_rate | 232123 | 0.0001 | 0.112 | 0.219 | +| New York | tract | owner_share | 225550 | 0.0009 | 0.112 | 0.219 | +| New York | tract | median_hh_income | 227384 | 0.0008 | 0.112 | 0.219 | +| Michigan | county | households | 2452 | 0.0265 | 0.118 | 0.236 | +| Michigan | county | poverty_count | 2452 | 0.0023 | 0.112 | 0.218 | +| Michigan | county | poverty_rate | 2452 | 0.0024 | 0.112 | 0.218 | +| Michigan | county | owner_share | 2452 | 0.0074 | 0.112 | 0.221 | +| Michigan | county | median_hh_income | 2452 | 0.0016 | 0.112 | 0.219 | +| New York | county | households | 1266 | 0.0328 | 0.120 | 0.242 | +| New York | county | poverty_count | 1266 | 0.0103 | 0.111 | 0.216 | +| New York | county | poverty_rate | 1266 | 0.0103 | 0.111 | 0.216 | +| New York | county | owner_share | 1266 | 0.0384 | 0.117 | 0.244 | +| New York | county | median_hh_income | 1266 | 0.0150 | 0.114 | 0.221 | + +Replicate design spectrum (lowest eigenvalues relative to the mean; Marchenko-Pastur lower edge for independent replicates in brackets): + +| state | geography | variable | lowest 4 eigenvalues (MP edge) | near-null | effective replicates | null SD 1/sqrt(eff.) | +| :--- | :--- | :--- | :--- | :--- | :--- | :--- | +| Michigan | tract | households | 0.052 0.054 0.078 0.400 (0.687) | 3 | 77 | 0.114 | +| Michigan | tract | poverty_count | 0.025 0.025 0.052 0.410 (0.687) | 3 | 77 | 0.114 | +| Michigan | tract | poverty_rate | 0.026 0.026 0.054 0.409 (0.687) | 3 | 77 | 0.114 | +| Michigan | tract | owner_share | 0.027 0.031 0.064 0.402 (0.687) | 3 | 77 | 0.114 | +| Michigan | tract | median_hh_income | 0.063 0.066 0.089 0.404 (0.687) | 3 | 77 | 0.114 | +| New York | tract | population | 0.047 0.054 0.086 0.421 (0.760) | 3 | 77 | 0.114 | +| New York | tract | households | 0.043 0.045 0.072 0.414 (0.759) | 3 | 77 | 0.114 | +| New York | tract | poverty_count | 0.024 0.025 0.052 0.421 (0.759) | 3 | 77 | 0.114 | +| New York | tract | poverty_rate | 0.025 0.027 0.053 0.418 (0.759) | 3 | 77 | 0.114 | +| New York | tract | owner_share | 0.028 0.029 0.062 0.410 (0.757) | 3 | 77 | 0.114 | +| New York | tract | median_hh_income | 0.062 0.069 0.089 0.409 (0.757) | 3 | 77 | 0.114 | +| Michigan | county | households | 0.001 0.001 0.001 0.004 (-) | not tested (too few areas) | - | - | +| Michigan | county | poverty_count | 0.000 0.001 0.002 0.003 (-) | not tested (too few areas) | - | - | +| Michigan | county | poverty_rate | 0.001 0.001 0.002 0.004 (-) | not tested (too few areas) | - | - | +| Michigan | county | owner_share | 0.001 0.001 0.002 0.005 (-) | not tested (too few areas) | - | - | +| Michigan | county | median_hh_income | 0.001 0.002 0.002 0.003 (-) | not tested (too few areas) | - | - | +| New York | county | households | 0.019 0.025 0.038 0.046 (-) | not tested (too few areas) | - | - | +| New York | county | poverty_count | 0.019 0.027 0.029 0.035 (-) | not tested (too few areas) | - | - | +| New York | county | poverty_rate | 0.019 0.026 0.033 0.036 (-) | not tested (too few areas) | - | - | +| New York | county | owner_share | 0.011 0.020 0.028 0.038 (-) | not tested (too few areas) | - | - | +| New York | county | median_hh_income | 0.022 0.027 0.037 0.052 (-) | not tested (too few areas) | - | - | + +Spread of pairwise r per class: SD, excess SD over the analytic null (sqrt(SD^2 - 1/80)) and over the empirical Michigan x New York null (sqrt(SD^2 - SD_cross^2)), and the share of pairs with |r| above the analytic 5% critical value (0.05 expected under no correlation; about 0.06 in the independent cross-state pairs): + +| state | geography | variable | class | SD of r | excess SD (analytic) | excess SD (cross-state) | share |r| > crit | +| :--- | :--- | :--- | :--- | :--- | :--- | :--- | :--- | +| Michigan | tract | households | order 1 | 0.140 | 0.085 | 0.079 | 0.115 | +| Michigan | tract | households | order 2 | 0.124 | 0.054 | 0.044 | 0.076 | +| Michigan | tract | households | order 3 | 0.119 | 0.041 | 0.026 | 0.066 | +| Michigan | tract | households | beyond order 3 | 0.117 | 0.034 | 0.011 | 0.061 | +| Michigan | tract | poverty_count | order 1 | 0.114 | 0.024 | 0.000 | 0.061 | +| Michigan | tract | poverty_count | order 2 | 0.116 | 0.030 | 0.010 | 0.064 | +| Michigan | tract | poverty_count | order 3 | 0.115 | 0.029 | 0.007 | 0.064 | +| Michigan | tract | poverty_count | beyond order 3 | 0.116 | 0.031 | 0.013 | 0.064 | +| Michigan | tract | poverty_rate | order 1 | 0.114 | 0.023 | 0.000 | 0.060 | +| Michigan | tract | poverty_rate | order 2 | 0.116 | 0.030 | 0.000 | 0.062 | +| Michigan | tract | poverty_rate | order 3 | 0.116 | 0.030 | 0.000 | 0.062 | +| Michigan | tract | poverty_rate | beyond order 3 | 0.116 | 0.030 | 0.005 | 0.063 | +| Michigan | tract | owner_share | order 1 | 0.115 | 0.027 | 0.000 | 0.058 | +| Michigan | tract | owner_share | order 2 | 0.115 | 0.025 | 0.000 | 0.058 | +| Michigan | tract | owner_share | order 3 | 0.115 | 0.028 | 0.006 | 0.059 | +| Michigan | tract | owner_share | beyond order 3 | 0.115 | 0.029 | 0.009 | 0.059 | +| Michigan | tract | median_hh_income | order 1 | 0.113 | 0.019 | 0.000 | 0.055 | +| Michigan | tract | median_hh_income | order 2 | 0.116 | 0.032 | 0.009 | 0.060 | +| Michigan | tract | median_hh_income | order 3 | 0.116 | 0.029 | 0.000 | 0.058 | +| Michigan | tract | median_hh_income | beyond order 3 | 0.115 | 0.027 | 0.000 | 0.057 | +| New York | tract | population | order 1 | 0.173 | 0.133 | - | 0.122 | +| New York | tract | population | order 3 | 0.118 | 0.038 | - | 0.060 | +| New York | tract | population | order 2 | 0.124 | 0.053 | - | 0.069 | +| New York | tract | population | beyond order 3 | 0.115 | 0.029 | - | 0.058 | +| New York | tract | households | order 1 | 0.131 | 0.067 | 0.059 | 0.090 | +| New York | tract | households | order 3 | 0.118 | 0.038 | 0.021 | 0.065 | +| New York | tract | households | order 2 | 0.120 | 0.044 | 0.030 | 0.069 | +| New York | tract | households | beyond order 3 | 0.117 | 0.033 | 0.009 | 0.061 | +| New York | tract | poverty_count | order 1 | 0.115 | 0.028 | 0.005 | 0.063 | +| New York | tract | poverty_count | order 3 | 0.115 | 0.027 | 0.000 | 0.062 | +| New York | tract | poverty_count | order 2 | 0.116 | 0.029 | 0.008 | 0.064 | +| New York | tract | poverty_count | beyond order 3 | 0.115 | 0.029 | 0.007 | 0.064 | +| New York | tract | poverty_rate | order 1 | 0.115 | 0.027 | 0.000 | 0.066 | +| New York | tract | poverty_rate | order 3 | 0.115 | 0.027 | 0.000 | 0.062 | +| New York | tract | poverty_rate | order 2 | 0.115 | 0.029 | 0.000 | 0.062 | +| New York | tract | poverty_rate | beyond order 3 | 0.115 | 0.029 | 0.000 | 0.063 | +| New York | tract | owner_share | order 1 | 0.115 | 0.025 | 0.000 | 0.059 | +| New York | tract | owner_share | order 3 | 0.116 | 0.029 | 0.009 | 0.059 | +| New York | tract | owner_share | order 2 | 0.115 | 0.028 | 0.003 | 0.059 | +| New York | tract | owner_share | beyond order 3 | 0.115 | 0.029 | 0.008 | 0.059 | +| New York | tract | median_hh_income | order 1 | 0.115 | 0.029 | 0.000 | 0.059 | +| New York | tract | median_hh_income | order 3 | 0.115 | 0.028 | 0.000 | 0.057 | +| New York | tract | median_hh_income | order 2 | 0.115 | 0.027 | 0.000 | 0.057 | +| New York | tract | median_hh_income | beyond order 3 | 0.115 | 0.027 | 0.000 | 0.058 | +| Michigan | county | households | order 1 | 0.124 | 0.054 | 0.032 | 0.096 | +| Michigan | county | households | order 3 | 0.115 | 0.027 | 0.000 | 0.064 | +| Michigan | county | households | order 2 | 0.117 | 0.033 | 0.000 | 0.062 | +| Michigan | county | households | beyond order 3 | 0.123 | 0.052 | 0.028 | 0.076 | +| Michigan | county | poverty_count | order 1 | 0.115 | 0.027 | 0.000 | 0.043 | +| Michigan | county | poverty_count | order 3 | 0.107 | 0.000 | 0.000 | 0.045 | +| Michigan | county | poverty_count | order 2 | 0.113 | 0.014 | 0.000 | 0.053 | +| Michigan | county | poverty_count | beyond order 3 | 0.116 | 0.032 | 0.010 | 0.058 | +| Michigan | county | poverty_rate | order 1 | 0.114 | 0.024 | 0.000 | 0.043 | +| Michigan | county | poverty_rate | order 3 | 0.108 | 0.000 | 0.000 | 0.052 | +| Michigan | county | poverty_rate | order 2 | 0.112 | 0.009 | 0.000 | 0.056 | +| Michigan | county | poverty_rate | beyond order 3 | 0.116 | 0.031 | 0.006 | 0.059 | +| Michigan | county | owner_share | order 1 | 0.113 | 0.013 | 0.000 | 0.043 | +| Michigan | county | owner_share | order 3 | 0.109 | 0.000 | 0.000 | 0.040 | +| Michigan | county | owner_share | order 2 | 0.109 | 0.000 | 0.000 | 0.041 | +| Michigan | county | owner_share | beyond order 3 | 0.115 | 0.028 | 0.009 | 0.057 | +| Michigan | county | median_hh_income | order 1 | 0.115 | 0.026 | 0.000 | 0.053 | +| Michigan | county | median_hh_income | order 3 | 0.114 | 0.020 | 0.000 | 0.050 | +| Michigan | county | median_hh_income | order 2 | 0.109 | 0.000 | 0.000 | 0.032 | +| Michigan | county | median_hh_income | beyond order 3 | 0.117 | 0.035 | 0.011 | 0.062 | +| New York | county | households | order 3 | 0.111 | 0.000 | 0.000 | 0.071 | +| New York | county | households | order 1 | 0.122 | 0.048 | 0.020 | 0.063 | +| New York | county | households | order 2 | 0.128 | 0.062 | 0.044 | 0.091 | +| New York | county | households | beyond order 3 | 0.124 | 0.054 | 0.031 | 0.084 | +| New York | county | poverty_count | order 3 | 0.112 | 0.000 | 0.000 | 0.060 | +| New York | county | poverty_count | order 1 | 0.120 | 0.043 | 0.030 | 0.049 | +| New York | county | poverty_count | order 2 | 0.112 | 0.000 | 0.000 | 0.039 | +| New York | county | poverty_count | beyond order 3 | 0.113 | 0.018 | 0.000 | 0.050 | +| New York | county | poverty_rate | order 3 | 0.111 | 0.000 | 0.000 | 0.060 | +| New York | county | poverty_rate | order 1 | 0.120 | 0.043 | 0.030 | 0.056 | +| New York | county | poverty_rate | order 2 | 0.112 | 0.009 | 0.000 | 0.043 | +| New York | county | poverty_rate | beyond order 3 | 0.113 | 0.018 | 0.000 | 0.053 | +| New York | county | owner_share | order 3 | 0.128 | 0.062 | 0.056 | 0.087 | +| New York | county | owner_share | order 1 | 0.120 | 0.044 | 0.035 | 0.056 | +| New York | county | owner_share | order 2 | 0.116 | 0.030 | 0.013 | 0.061 | +| New York | county | owner_share | beyond order 3 | 0.119 | 0.041 | 0.031 | 0.078 | +| New York | county | median_hh_income | order 3 | 0.109 | 0.000 | 0.000 | 0.032 | +| New York | county | median_hh_income | order 1 | 0.116 | 0.029 | 0.000 | 0.056 | +| New York | county | median_hh_income | order 2 | 0.111 | 0.000 | 0.000 | 0.056 | +| New York | county | median_hh_income | beyond order 3 | 0.117 | 0.035 | 0.010 | 0.051 | + +### Split-half reliability of individual pairwise correlations + +Per class: half-A and half-B class means (random split), and the reliability of a single pair's r (correlation across pairs between halves; stepped up to 80 replicates), for each split. + +| state | geography | variable | class | pairs | mean (half A) | mean (half B) | reliability (random / odd-even / first-last) | +| :--- | :--- | :--- | :--- | :--- | :--- | :--- | :--- | +| Michigan | tract | households | order 1 | 8614 | 0.022 | 0.027 | 0.37 / 0.37 / 0.35 | +| Michigan | tract | households | order 2 | 20000 | 0.005 | 0.007 | 0.19 / 0.19 / 0.20 | +| Michigan | tract | households | order 3 | 33974 | 0.001 | 0.002 | 0.11 / 0.12 / 0.13 | +| Michigan | tract | households | beyond order 3 | 234537 | -0.000 | -0.000 | 0.08 / 0.09 / 0.07 | +| Michigan | tract | poverty_count | order 1 | 8601 | -0.007 | -0.001 | 0.03 / 0.05 / -0.04 | +| Michigan | tract | poverty_count | order 2 | 19957 | -0.002 | -0.001 | 0.08 / 0.06 / 0.01 | +| Michigan | tract | poverty_count | order 3 | 33911 | -0.001 | 0.000 | 0.07 / 0.05 / -0.01 | +| Michigan | tract | poverty_count | beyond order 3 | 234178 | -0.000 | 0.000 | 0.07 / 0.06 / 0.02 | +| Michigan | tract | poverty_rate | order 1 | 8581 | -0.003 | 0.003 | 0.02 / 0.04 / -0.05 | +| Michigan | tract | poverty_rate | order 2 | 19916 | -0.000 | -0.000 | 0.07 / 0.07 / 0.01 | +| Michigan | tract | poverty_rate | order 3 | 33847 | -0.001 | 0.001 | 0.07 / 0.05 / 0.00 | +| Michigan | tract | poverty_rate | beyond order 3 | 233863 | 0.000 | 0.000 | 0.07 / 0.06 / 0.01 | +| Michigan | tract | owner_share | order 1 | 8503 | -0.000 | -0.002 | 0.04 / 0.05 / 0.02 | +| Michigan | tract | owner_share | order 2 | 19730 | -0.001 | -0.001 | 0.06 / 0.05 / -0.02 | +| Michigan | tract | owner_share | order 3 | 33514 | 0.001 | 0.001 | 0.06 / 0.06 / -0.00 | +| Michigan | tract | owner_share | beyond order 3 | 231151 | 0.000 | 0.000 | 0.06 / 0.06 / 0.01 | +| Michigan | tract | median_hh_income | order 1 | 8563 | 0.000 | 0.003 | 0.02 / 0.03 / 0.02 | +| Michigan | tract | median_hh_income | order 2 | 19869 | -0.000 | 0.000 | 0.08 / 0.08 / 0.04 | +| Michigan | tract | median_hh_income | order 3 | 33766 | 0.001 | 0.000 | 0.06 / 0.06 / 0.03 | +| Michigan | tract | median_hh_income | beyond order 3 | 232936 | 0.001 | 0.001 | 0.05 / 0.06 / 0.03 | +| New York | tract | population | order 1 | 14629 | -0.038 | -0.039 | 0.60 / 0.60 / 0.59 | +| New York | tract | population | order 3 | 61223 | -0.002 | -0.002 | 0.09 / 0.10 / 0.08 | +| New York | tract | population | order 2 | 35303 | -0.008 | -0.008 | 0.19 / 0.19 / 0.15 | +| New York | tract | population | beyond order 3 | 236096 | -0.001 | -0.001 | 0.07 / 0.06 / 0.02 | +| New York | tract | households | order 1 | 14502 | 0.011 | 0.012 | 0.28 / 0.27 / 0.28 | +| New York | tract | households | order 3 | 60587 | 0.002 | 0.003 | 0.11 / 0.12 / 0.10 | +| New York | tract | households | order 2 | 34948 | 0.005 | 0.005 | 0.13 / 0.12 / 0.12 | +| New York | tract | households | beyond order 3 | 234210 | -0.000 | -0.000 | 0.09 / 0.08 / 0.07 | +| New York | tract | poverty_count | order 1 | 14431 | -0.003 | -0.003 | 0.06 / 0.08 / 0.01 | +| New York | tract | poverty_count | order 3 | 60162 | 0.000 | -0.002 | 0.06 / 0.05 / -0.01 | +| New York | tract | poverty_count | order 2 | 34710 | 0.000 | -0.001 | 0.06 / 0.05 / 0.01 | +| New York | tract | poverty_count | beyond order 3 | 232610 | 0.000 | -0.000 | 0.07 / 0.06 / 0.01 | +| New York | tract | poverty_rate | order 1 | 14378 | -0.001 | -0.000 | 0.06 / 0.07 / 0.01 | +| New York | tract | poverty_rate | order 3 | 60035 | 0.000 | -0.001 | 0.06 / 0.05 / -0.01 | +| New York | tract | poverty_rate | order 2 | 34613 | 0.000 | -0.000 | 0.06 / 0.05 / -0.00 | +| New York | tract | poverty_rate | beyond order 3 | 232123 | 0.000 | -0.000 | 0.07 / 0.06 / 0.01 | +| New York | tract | owner_share | order 1 | 14033 | 0.001 | -0.000 | 0.05 / 0.05 / -0.02 | +| New York | tract | owner_share | order 3 | 58430 | 0.000 | 0.001 | 0.06 / 0.06 / 0.02 | +| New York | tract | owner_share | order 2 | 33723 | 0.000 | 0.000 | 0.04 / 0.07 / 0.01 | +| New York | tract | owner_share | beyond order 3 | 225550 | 0.001 | 0.000 | 0.06 / 0.05 / 0.02 | +| New York | tract | median_hh_income | order 1 | 14048 | 0.002 | 0.001 | 0.05 / 0.07 / 0.05 | +| New York | tract | median_hh_income | order 3 | 58491 | 0.000 | 0.001 | 0.06 / 0.05 / 0.03 | +| New York | tract | median_hh_income | order 2 | 33653 | 0.001 | 0.000 | 0.06 / 0.07 / 0.01 | +| New York | tract | median_hh_income | beyond order 3 | 227384 | -0.000 | 0.000 | 0.06 / 0.05 / 0.02 | +| Michigan | county | households | order 1 | 208 | 0.014 | 0.033 | 0.17 / 0.19 / 0.30 | +| Michigan | county | households | order 3 | 404 | 0.025 | 0.008 | 0.15 / 0.16 / 0.06 | +| Michigan | county | households | order 2 | 339 | 0.031 | 0.025 | 0.19 / 0.05 / 0.10 | +| Michigan | county | households | beyond order 3 | 2452 | 0.023 | 0.023 | 0.20 / 0.15 / 0.19 | +| Michigan | county | poverty_count | order 1 | 208 | -0.002 | -0.010 | -0.06 / 0.11 / 0.12 | +| Michigan | county | poverty_count | order 3 | 404 | 0.008 | 0.007 | -0.37 / -0.04 / -0.05 | +| Michigan | county | poverty_count | order 2 | 339 | 0.009 | 0.013 | 0.02 / -0.10 / -0.06 | +| Michigan | county | poverty_count | beyond order 3 | 2452 | -0.001 | 0.005 | 0.07 / 0.09 / 0.04 | +| Michigan | county | poverty_rate | order 1 | 208 | -0.002 | -0.008 | -0.06 / 0.11 / 0.10 | +| Michigan | county | poverty_rate | order 3 | 404 | 0.007 | 0.008 | -0.37 / -0.02 / -0.05 | +| Michigan | county | poverty_rate | order 2 | 339 | 0.008 | 0.014 | 0.02 / -0.09 / -0.07 | +| Michigan | county | poverty_rate | beyond order 3 | 2452 | -0.001 | 0.005 | 0.06 / 0.08 / 0.03 | +| Michigan | county | owner_share | order 1 | 208 | -0.009 | 0.007 | -0.09 / 0.01 / -0.18 | +| Michigan | county | owner_share | order 3 | 404 | 0.004 | 0.007 | 0.01 / -0.14 / 0.01 | +| Michigan | county | owner_share | order 2 | 339 | 0.019 | 0.014 | -0.04 / -0.07 / -0.05 | +| Michigan | county | owner_share | beyond order 3 | 2452 | 0.002 | 0.012 | 0.06 / 0.09 / 0.03 | +| Michigan | county | median_hh_income | order 1 | 208 | -0.006 | 0.014 | -0.03 / -0.07 / 0.15 | +| Michigan | county | median_hh_income | order 3 | 404 | 0.019 | -0.012 | -0.07 / 0.07 / -0.07 | +| Michigan | county | median_hh_income | order 2 | 339 | 0.004 | 0.013 | -0.14 / 0.02 / 0.10 | +| Michigan | county | median_hh_income | beyond order 3 | 2452 | -0.003 | 0.004 | 0.12 / 0.07 / 0.09 | +| New York | county | households | order 3 | 252 | 0.016 | 0.039 | 0.04 / 0.03 / 0.05 | +| New York | county | households | order 1 | 142 | 0.012 | 0.015 | 0.04 / 0.15 / 0.12 | +| New York | county | households | order 2 | 231 | -0.009 | 0.019 | 0.13 / 0.30 / 0.23 | +| New York | county | households | beyond order 3 | 1266 | 0.025 | 0.037 | 0.19 / 0.21 / 0.28 | +| New York | county | poverty_count | order 3 | 252 | -0.007 | 0.005 | -0.05 / -0.05 / 0.12 | +| New York | county | poverty_count | order 1 | 142 | 0.025 | 0.005 | 0.06 / 0.18 / 0.02 | +| New York | county | poverty_count | order 2 | 231 | 0.004 | -0.001 | -0.11 / 0.06 / -0.24 | +| New York | county | poverty_count | beyond order 3 | 1266 | 0.007 | 0.007 | 0.08 / 0.06 / -0.01 | +| New York | county | poverty_rate | order 3 | 252 | -0.008 | 0.004 | -0.05 / -0.05 / 0.10 | +| New York | county | poverty_rate | order 1 | 142 | 0.025 | 0.008 | 0.07 / 0.20 / 0.02 | +| New York | county | poverty_rate | order 2 | 231 | 0.004 | -0.002 | -0.09 / 0.07 / -0.24 | +| New York | county | poverty_rate | beyond order 3 | 1266 | 0.007 | 0.007 | 0.08 / 0.06 / -0.01 | +| New York | county | owner_share | order 3 | 252 | 0.025 | 0.010 | 0.17 / 0.30 / 0.22 | +| New York | county | owner_share | order 1 | 142 | 0.041 | 0.012 | 0.13 / 0.09 / 0.10 | +| New York | county | owner_share | order 2 | 231 | 0.019 | 0.031 | 0.20 / 0.04 / -0.07 | +| New York | county | owner_share | beyond order 3 | 1266 | 0.043 | 0.029 | 0.14 / 0.13 / 0.19 | +| New York | county | median_hh_income | order 3 | 252 | -0.001 | 0.010 | -0.06 / -0.07 / -0.15 | +| New York | county | median_hh_income | order 1 | 142 | 0.018 | 0.031 | 0.01 / 0.17 / 0.10 | +| New York | county | median_hh_income | order 2 | 231 | 0.009 | 0.002 | 0.00 / -0.02 / 0.00 | +| New York | county | median_hh_income | beyond order 3 | 1266 | 0.010 | 0.010 | 0.10 / 0.04 / 0.11 | + +## Stability across releases (2015-2019 vs 2020-2024) + +Mean r per class in each release (different tract vintages: 2010 tracts in 2015-2019, 2020 tracts in 2020-2024; so pair sets differ). Difference with a combined SE. + +| state | geography | variable | class | 2019 | 2024 | difference (SE) | +| :--- | :--- | :--- | :--- | :--- | :--- | :--- | +| Michigan | tract | households | order 1 | 0.025 [0.016, 0.033] | -0.025 [-0.033, -0.017] | -0.050 (0.006) | +| Michigan | tract | households | order 2 | 0.006 [0.000, 0.013] | -0.008 [-0.011, -0.004] | -0.014 (0.004) | +| Michigan | tract | households | order 3 | 0.001 [-0.002, 0.005] | -0.003 [-0.005, -0.002] | -0.005 (0.002) | +| Michigan | tract | households | beyond order 3 | -0.000 [-0.001, 0.001] | -0.001 [-0.001, 0.000] | -0.000 (0.001) | +| Michigan | tract | households | order 1 / within county | 0.028 [0.020, 0.035] | -0.028 [-0.039, -0.018] | -0.056 (0.006) | +| Michigan | tract | households | order 2 / within county | 0.008 [0.001, 0.016] | -0.009 [-0.015, -0.004] | -0.018 (0.005) | +| Michigan | tract | households | order 2 / across county | 0.001 [-0.003, 0.005] | -0.002 [-0.006, 0.002] | -0.003 (0.003) | +| Michigan | tract | households | order 1 / across county | 0.001 [-0.012, 0.014] | -0.003 [-0.012, 0.006] | -0.004 (0.008) | +| Michigan | tract | households | order 3 / across county | 0.001 [-0.002, 0.004] | 0.000 [-0.002, 0.003] | -0.000 (0.002) | +| Michigan | tract | households | order 3 / within county | 0.002 [-0.003, 0.007] | -0.006 [-0.009, -0.003] | -0.008 (0.003) | +| Michigan | tract | households | beyond order 3 / within county | -0.001 [-0.003, 0.002] | -0.001 [-0.003, -0.000] | -0.001 (0.001) | +| Michigan | tract | households | beyond order 3 / across county | 0.000 [-0.001, 0.001] | 0.001 [-0.001, 0.002] | 0.001 (0.001) | +| Michigan | tract | poverty_count | order 1 | -0.004 [-0.007, -0.001] | -0.004 [-0.007, -0.002] | 0.000 (0.002) | +| Michigan | tract | poverty_count | order 2 | -0.002 [-0.004, 0.000] | -0.003 [-0.005, -0.002] | -0.002 (0.001) | +| Michigan | tract | poverty_count | order 3 | 0.000 [-0.002, 0.002] | -0.000 [-0.002, 0.001] | -0.000 (0.001) | +| Michigan | tract | poverty_count | beyond order 3 | -0.000 [-0.000, 0.000] | 0.000 [-0.001, 0.002] | 0.000 (0.001) | +| Michigan | tract | poverty_count | order 1 / within county | -0.005 [-0.008, -0.002] | -0.005 [-0.007, -0.002] | 0.000 (0.002) | +| Michigan | tract | poverty_count | order 2 / within county | -0.001 [-0.003, 0.001] | -0.005 [-0.006, -0.003] | -0.003 (0.001) | +| Michigan | tract | poverty_count | order 2 / across county | -0.003 [-0.008, 0.001] | 0.000 [-0.003, 0.003] | 0.003 (0.003) | +| Michigan | tract | poverty_count | order 1 / across county | 0.001 [-0.007, 0.010] | 0.001 [-0.008, 0.011] | 0.000 (0.007) | +| Michigan | tract | poverty_count | order 3 / across county | 0.001 [-0.002, 0.003] | 0.001 [-0.002, 0.003] | -0.000 (0.002) | +| Michigan | tract | poverty_count | order 3 / within county | -0.000 [-0.002, 0.001] | -0.001 [-0.003, 0.001] | -0.001 (0.001) | +| Michigan | tract | poverty_count | beyond order 3 / within county | 0.000 [-0.000, 0.001] | 0.000 [-0.001, 0.002] | 0.000 (0.001) | +| Michigan | tract | poverty_count | beyond order 3 / across county | -0.001 [-0.002, 0.000] | -0.000 [-0.002, 0.001] | 0.000 (0.001) | +| Michigan | tract | poverty_rate | order 1 | -0.000 [-0.003, 0.003] | -0.000 [-0.003, 0.002] | -0.000 (0.002) | +| Michigan | tract | poverty_rate | order 2 | -0.001 [-0.003, 0.001] | -0.002 [-0.004, -0.001] | -0.002 (0.001) | +| Michigan | tract | poverty_rate | order 3 | 0.000 [-0.002, 0.002] | 0.000 [-0.002, 0.002] | -0.000 (0.001) | +| Michigan | tract | poverty_rate | beyond order 3 | 0.000 [-0.001, 0.001] | 0.000 [-0.001, 0.001] | 0.000 (0.001) | +| Michigan | tract | poverty_rate | order 1 / within county | -0.000 [-0.003, 0.002] | -0.001 [-0.003, 0.002] | -0.000 (0.002) | +| Michigan | tract | poverty_rate | order 2 / within county | 0.000 [-0.002, 0.003] | -0.003 [-0.005, -0.002] | -0.003 (0.002) | +| Michigan | tract | poverty_rate | order 2 / across county | -0.003 [-0.007, 0.002] | 0.001 [-0.003, 0.004] | 0.003 (0.003) | +| Michigan | tract | poverty_rate | order 1 / across county | 0.001 [-0.007, 0.009] | 0.002 [-0.007, 0.012] | 0.001 (0.006) | +| Michigan | tract | poverty_rate | order 3 / across county | 0.000 [-0.002, 0.003] | 0.000 [-0.002, 0.003] | 0.000 (0.002) | +| Michigan | tract | poverty_rate | order 3 / within county | 0.000 [-0.002, 0.002] | -0.000 [-0.003, 0.002] | -0.000 (0.002) | +| Michigan | tract | poverty_rate | beyond order 3 / within county | 0.001 [0.000, 0.001] | 0.001 [-0.001, 0.002] | 0.000 (0.001) | +| Michigan | tract | poverty_rate | beyond order 3 / across county | -0.001 [-0.002, 0.000] | -0.000 [-0.002, 0.001] | 0.000 (0.001) | +| Michigan | tract | owner_share | order 1 | -0.001 [-0.004, 0.001] | -0.002 [-0.005, 0.000] | -0.001 (0.002) | +| Michigan | tract | owner_share | order 2 | -0.001 [-0.004, 0.002] | 0.000 [-0.001, 0.002] | 0.001 (0.002) | +| Michigan | tract | owner_share | order 3 | 0.001 [-0.000, 0.002] | -0.001 [-0.002, 0.001] | -0.002 (0.001) | +| Michigan | tract | owner_share | beyond order 3 | 0.000 [-0.001, 0.001] | 0.000 [-0.000, 0.001] | 0.000 (0.000) | +| Michigan | tract | owner_share | order 1 / within county | -0.001 [-0.004, 0.002] | -0.002 [-0.005, -0.000] | -0.002 (0.002) | +| Michigan | tract | owner_share | order 2 / within county | -0.000 [-0.005, 0.004] | 0.001 [-0.001, 0.003] | 0.001 (0.002) | +| Michigan | tract | owner_share | order 2 / across county | -0.002 [-0.006, 0.003] | -0.001 [-0.006, 0.003] | 0.001 (0.003) | +| Michigan | tract | owner_share | order 1 / across county | -0.005 [-0.015, 0.005] | 0.000 [-0.010, 0.011] | 0.006 (0.007) | +| Michigan | tract | owner_share | order 3 / across county | 0.000 [-0.003, 0.004] | -0.000 [-0.003, 0.003] | -0.000 (0.002) | +| Michigan | tract | owner_share | order 3 / within county | 0.002 [0.000, 0.003] | -0.001 [-0.003, 0.001] | -0.003 (0.001) | +| Michigan | tract | owner_share | beyond order 3 / within county | 0.000 [-0.000, 0.001] | 0.000 [-0.000, 0.001] | -0.000 (0.001) | +| Michigan | tract | owner_share | beyond order 3 / across county | -0.000 [-0.001, 0.001] | 0.000 [-0.001, 0.001] | 0.001 (0.001) | +| Michigan | tract | median_hh_income | order 1 | 0.002 [-0.001, 0.004] | 0.000 [-0.002, 0.003] | -0.001 (0.002) | +| Michigan | tract | median_hh_income | order 2 | -0.000 [-0.003, 0.003] | 0.000 [-0.002, 0.003] | 0.001 (0.002) | +| Michigan | tract | median_hh_income | order 3 | 0.001 [-0.001, 0.002] | 0.001 [-0.000, 0.002] | 0.001 (0.001) | +| Michigan | tract | median_hh_income | beyond order 3 | 0.001 [-0.000, 0.002] | 0.001 [0.000, 0.002] | -0.000 (0.001) | +| Michigan | tract | median_hh_income | order 1 / within county | 0.001 [-0.001, 0.004] | 0.000 [-0.002, 0.003] | -0.001 (0.002) | +| Michigan | tract | median_hh_income | order 2 / within county | -0.000 [-0.004, 0.004] | -0.000 [-0.002, 0.002] | 0.000 (0.002) | +| Michigan | tract | median_hh_income | order 2 / across county | -0.000 [-0.004, 0.004] | 0.002 [-0.002, 0.006] | 0.002 (0.003) | +| Michigan | tract | median_hh_income | order 1 / across county | 0.005 [-0.006, 0.016] | -0.001 [-0.011, 0.009] | -0.006 (0.008) | +| Michigan | tract | median_hh_income | order 3 / across county | 0.001 [-0.002, 0.004] | 0.001 [-0.001, 0.004] | 0.000 (0.002) | +| Michigan | tract | median_hh_income | order 3 / within county | 0.000 [-0.001, 0.002] | 0.001 [-0.000, 0.002] | 0.001 (0.001) | +| Michigan | tract | median_hh_income | beyond order 3 / within county | 0.001 [-0.001, 0.003] | 0.001 [0.001, 0.002] | 0.000 (0.001) | +| Michigan | tract | median_hh_income | beyond order 3 / across county | 0.001 [-0.001, 0.002] | 0.000 [-0.001, 0.001] | -0.000 (0.001) | +| New York | tract | population | order 2 | -0.008 [-0.013, -0.003] | -0.009 [-0.013, -0.004] | -0.001 (0.003) | +| New York | tract | population | order 1 | -0.039 [-0.060, -0.018] | -0.040 [-0.060, -0.020] | -0.001 (0.015) | +| New York | tract | population | order 3 | -0.002 [-0.004, -0.001] | -0.003 [-0.004, -0.002] | -0.001 (0.001) | +| New York | tract | population | beyond order 3 | -0.001 [-0.002, -0.000] | -0.001 [-0.002, -0.001] | -0.000 (0.000) | +| New York | tract | population | order 2 / within county | -0.010 [-0.016, -0.004] | -0.010 [-0.016, -0.005] | -0.001 (0.004) | +| New York | tract | population | order 1 / within county | -0.042 [-0.065, -0.018] | -0.043 [-0.065, -0.020] | -0.001 (0.017) | +| New York | tract | population | order 3 / within county | -0.003 [-0.005, -0.001] | -0.004 [-0.006, -0.003] | -0.001 (0.001) | +| New York | tract | population | order 3 / across county | 0.001 [-0.002, 0.003] | -0.001 [-0.003, 0.002] | -0.001 (0.002) | +| New York | tract | population | order 2 / across county | -0.000 [-0.004, 0.004] | 0.001 [-0.003, 0.005] | 0.001 (0.003) | +| New York | tract | population | order 1 / across county | 0.000 [-0.008, 0.009] | 0.002 [-0.006, 0.009] | 0.001 (0.006) | +| New York | tract | population | beyond order 3 / within county | -0.002 [-0.002, -0.001] | -0.001 [-0.002, -0.001] | 0.000 (0.000) | +| New York | tract | population | beyond order 3 / across county | 0.000 [-0.001, 0.001] | -0.001 [-0.002, 0.000] | -0.001 (0.001) | +| New York | tract | households | order 2 | 0.005 [0.002, 0.008] | -0.003 [-0.006, -0.001] | -0.008 (0.002) | +| New York | tract | households | order 1 | 0.012 [0.003, 0.020] | -0.015 [-0.022, -0.008] | -0.027 (0.006) | +| New York | tract | households | order 3 | 0.002 [0.001, 0.004] | -0.002 [-0.003, -0.001] | -0.005 (0.001) | +| New York | tract | households | beyond order 3 | -0.000 [-0.001, 0.001] | -0.001 [-0.001, -0.000] | -0.001 (0.001) | +| New York | tract | households | order 2 / within county | 0.007 [0.004, 0.009] | -0.004 [-0.008, -0.001] | -0.011 (0.002) | +| New York | tract | households | order 1 / within county | 0.013 [0.003, 0.022] | -0.016 [-0.024, -0.008] | -0.029 (0.006) | +| New York | tract | households | order 3 / within county | 0.004 [0.002, 0.006] | -0.003 [-0.004, -0.002] | -0.007 (0.001) | +| New York | tract | households | order 3 / across county | -0.001 [-0.004, 0.002] | -0.000 [-0.003, 0.002] | 0.001 (0.002) | +| New York | tract | households | order 2 / across county | -0.003 [-0.010, 0.003] | 0.002 [-0.002, 0.006] | 0.005 (0.004) | +| New York | tract | households | order 1 / across county | -0.004 [-0.013, 0.006] | -0.000 [-0.011, 0.011] | 0.003 (0.007) | +| New York | tract | households | beyond order 3 / within county | -0.001 [-0.002, 0.000] | -0.001 [-0.002, -0.001] | -0.000 (0.001) | +| New York | tract | households | beyond order 3 / across county | 0.001 [-0.001, 0.003] | 0.000 [-0.001, 0.001] | -0.001 (0.001) | +| New York | tract | poverty_count | order 2 | -0.000 [-0.002, 0.001] | -0.001 [-0.002, 0.001] | -0.000 (0.001) | +| New York | tract | poverty_count | order 1 | -0.003 [-0.005, -0.001] | -0.002 [-0.003, 0.000] | 0.001 (0.001) | +| New York | tract | poverty_count | order 3 | -0.001 [-0.002, 0.000] | 0.000 [-0.001, 0.001] | 0.001 (0.001) | +| New York | tract | poverty_count | beyond order 3 | 0.000 [-0.000, 0.001] | -0.000 [-0.001, 0.001] | -0.000 (0.001) | +| New York | tract | poverty_count | order 2 / within county | -0.000 [-0.002, 0.001] | -0.000 [-0.002, 0.001] | -0.000 (0.001) | +| New York | tract | poverty_count | order 1 / within county | -0.003 [-0.005, -0.001] | -0.002 [-0.004, -0.000] | 0.001 (0.001) | +| New York | tract | poverty_count | order 3 / within county | -0.001 [-0.002, -0.000] | 0.000 [-0.001, 0.002] | 0.002 (0.001) | +| New York | tract | poverty_count | order 3 / across county | 0.000 [-0.003, 0.003] | -0.001 [-0.004, 0.002] | -0.001 (0.002) | +| New York | tract | poverty_count | order 2 / across county | 0.000 [-0.004, 0.004] | -0.002 [-0.006, 0.002] | -0.002 (0.003) | +| New York | tract | poverty_count | order 1 / across county | 0.004 [-0.006, 0.013] | 0.004 [-0.005, 0.013] | 0.000 (0.007) | +| New York | tract | poverty_count | beyond order 3 / within county | -0.000 [-0.001, 0.001] | -0.000 [-0.002, 0.002] | -0.000 (0.001) | +| New York | tract | poverty_count | beyond order 3 / across county | 0.000 [-0.001, 0.001] | 0.000 [-0.001, 0.001] | -0.000 (0.001) | +| New York | tract | poverty_rate | order 2 | -0.000 [-0.001, 0.001] | -0.000 [-0.001, 0.001] | 0.000 (0.001) | +| New York | tract | poverty_rate | order 1 | -0.001 [-0.002, 0.001] | -0.000 [-0.002, 0.002] | 0.001 (0.001) | +| New York | tract | poverty_rate | order 3 | -0.001 [-0.002, 0.001] | 0.000 [-0.001, 0.001] | 0.001 (0.001) | +| New York | tract | poverty_rate | beyond order 3 | 0.000 [-0.000, 0.001] | 0.000 [-0.001, 0.001] | -0.000 (0.001) | +| New York | tract | poverty_rate | order 2 / within county | -0.000 [-0.001, 0.001] | 0.000 [-0.001, 0.001] | 0.000 (0.001) | +| New York | tract | poverty_rate | order 1 / within county | -0.001 [-0.003, 0.001] | -0.000 [-0.002, 0.002] | 0.001 (0.001) | +| New York | tract | poverty_rate | order 3 / within county | -0.001 [-0.002, 0.000] | 0.000 [-0.001, 0.002] | 0.001 (0.001) | +| New York | tract | poverty_rate | order 3 / across county | -0.000 [-0.003, 0.003] | -0.001 [-0.004, 0.002] | -0.001 (0.002) | +| New York | tract | poverty_rate | order 2 / across county | 0.000 [-0.003, 0.004] | -0.002 [-0.006, 0.003] | -0.002 (0.003) | +| New York | tract | poverty_rate | order 1 / across county | 0.003 [-0.008, 0.013] | 0.003 [-0.007, 0.013] | 0.001 (0.007) | +| New York | tract | poverty_rate | beyond order 3 / within county | 0.000 [-0.001, 0.001] | -0.000 [-0.002, 0.002] | -0.000 (0.001) | +| New York | tract | poverty_rate | beyond order 3 / across county | 0.000 [-0.001, 0.001] | 0.000 [-0.000, 0.001] | 0.000 (0.001) | +| New York | tract | owner_share | order 2 | 0.000 [-0.001, 0.002] | 0.000 [-0.001, 0.001] | -0.000 (0.001) | +| New York | tract | owner_share | order 1 | 0.000 [-0.002, 0.002] | -0.001 [-0.002, 0.001] | -0.001 (0.001) | +| New York | tract | owner_share | order 3 | 0.000 [-0.001, 0.002] | 0.001 [-0.000, 0.002] | 0.000 (0.001) | +| New York | tract | owner_share | beyond order 3 | 0.000 [-0.000, 0.001] | 0.000 [-0.000, 0.001] | 0.000 (0.000) | +| New York | tract | owner_share | order 2 / within county | 0.000 [-0.001, 0.002] | 0.000 [-0.001, 0.001] | -0.000 (0.001) | +| New York | tract | owner_share | order 1 / within county | 0.000 [-0.002, 0.002] | -0.001 [-0.003, 0.001] | -0.001 (0.001) | +| New York | tract | owner_share | order 3 / within county | 0.000 [-0.001, 0.002] | 0.001 [-0.001, 0.002] | 0.000 (0.001) | +| New York | tract | owner_share | order 3 / across county | 0.000 [-0.003, 0.003] | 0.001 [-0.001, 0.004] | 0.001 (0.002) | +| New York | tract | owner_share | order 2 / across county | 0.002 [-0.003, 0.006] | -0.000 [-0.005, 0.004] | -0.002 (0.003) | +| New York | tract | owner_share | order 1 / across county | 0.001 [-0.009, 0.011] | 0.004 [-0.007, 0.015] | 0.003 (0.007) | +| New York | tract | owner_share | beyond order 3 / within county | 0.001 [-0.000, 0.001] | 0.000 [-0.000, 0.001] | -0.000 (0.001) | +| New York | tract | owner_share | beyond order 3 / across county | 0.000 [-0.001, 0.002] | 0.001 [-0.000, 0.001] | 0.000 (0.001) | +| New York | tract | median_hh_income | order 2 | 0.000 [-0.001, 0.002] | 0.001 [-0.000, 0.002] | 0.001 (0.001) | +| New York | tract | median_hh_income | order 1 | 0.002 [-0.001, 0.004] | 0.000 [-0.002, 0.002] | -0.001 (0.002) | +| New York | tract | median_hh_income | order 3 | 0.001 [-0.001, 0.002] | -0.000 [-0.001, 0.001] | -0.001 (0.001) | +| New York | tract | median_hh_income | beyond order 3 | 0.000 [-0.000, 0.001] | 0.001 [0.000, 0.001] | 0.001 (0.000) | +| New York | tract | median_hh_income | order 2 / within county | 0.000 [-0.001, 0.002] | 0.001 [0.000, 0.003] | 0.001 (0.001) | +| New York | tract | median_hh_income | order 1 / within county | 0.001 [-0.001, 0.003] | 0.000 [-0.002, 0.003] | -0.001 (0.002) | +| New York | tract | median_hh_income | order 3 / within county | 0.001 [-0.001, 0.003] | 0.000 [-0.001, 0.002] | -0.000 (0.001) | +| New York | tract | median_hh_income | order 3 / across county | 0.001 [-0.001, 0.004] | -0.001 [-0.005, 0.002] | -0.003 (0.002) | +| New York | tract | median_hh_income | order 2 / across county | 0.000 [-0.004, 0.005] | -0.001 [-0.006, 0.003] | -0.002 (0.003) | +| New York | tract | median_hh_income | order 1 / across county | 0.009 [-0.003, 0.021] | 0.000 [-0.009, 0.010] | -0.008 (0.008) | +| New York | tract | median_hh_income | beyond order 3 / within county | 0.000 [-0.001, 0.001] | 0.001 [0.000, 0.001] | 0.001 (0.000) | +| New York | tract | median_hh_income | beyond order 3 / across county | 0.000 [-0.001, 0.001] | 0.001 [-0.001, 0.002] | 0.001 (0.001) | +| Michigan | county | households | order 1 | 0.023 [-0.004, 0.050] | 0.026 [0.001, 0.051] | 0.003 (0.019) | +| Michigan | county | households | order 3 | 0.016 [-0.002, 0.035] | 0.027 [0.005, 0.049] | 0.011 (0.015) | +| Michigan | county | households | order 2 | 0.027 [0.008, 0.047] | 0.031 [0.009, 0.052] | 0.003 (0.015) | +| Michigan | county | households | beyond order 3 | 0.023 [0.009, 0.038] | 0.033 [0.015, 0.051] | 0.010 (0.012) | +| Michigan | county | poverty_count | order 1 | -0.007 [-0.031, 0.016] | -0.005 [-0.027, 0.017] | 0.002 (0.016) | +| Michigan | county | poverty_count | order 3 | 0.008 [-0.005, 0.022] | 0.005 [-0.014, 0.024] | -0.003 (0.012) | +| Michigan | county | poverty_count | order 2 | 0.011 [-0.006, 0.028] | 0.009 [-0.007, 0.026] | -0.002 (0.012) | +| Michigan | county | poverty_count | beyond order 3 | 0.003 [-0.005, 0.010] | 0.002 [-0.007, 0.010] | -0.001 (0.006) | +| Michigan | county | poverty_rate | order 1 | -0.007 [-0.030, 0.016] | -0.005 [-0.027, 0.017] | 0.002 (0.016) | +| Michigan | county | poverty_rate | order 3 | 0.008 [-0.005, 0.022] | 0.005 [-0.014, 0.024] | -0.003 (0.012) | +| Michigan | county | poverty_rate | order 2 | 0.012 [-0.005, 0.028] | 0.009 [-0.007, 0.026] | -0.002 (0.012) | +| Michigan | county | poverty_rate | beyond order 3 | 0.003 [-0.005, 0.010] | 0.002 [-0.007, 0.010] | -0.001 (0.006) | +| Michigan | county | owner_share | order 1 | -0.001 [-0.023, 0.021] | -0.003 [-0.025, 0.018] | -0.002 (0.016) | +| Michigan | county | owner_share | order 3 | 0.005 [-0.010, 0.021] | 0.022 [0.006, 0.039] | 0.017 (0.012) | +| Michigan | county | owner_share | order 2 | 0.016 [-0.003, 0.036] | 0.018 [-0.001, 0.036] | 0.001 (0.014) | +| Michigan | county | owner_share | beyond order 3 | 0.006 [-0.002, 0.014] | 0.001 [-0.006, 0.009] | -0.005 (0.006) | +| Michigan | county | median_hh_income | order 1 | 0.005 [-0.016, 0.026] | 0.003 [-0.016, 0.022] | -0.002 (0.014) | +| Michigan | county | median_hh_income | order 3 | 0.003 [-0.012, 0.018] | 0.002 [-0.015, 0.020] | -0.001 (0.012) | +| Michigan | county | median_hh_income | order 2 | 0.008 [-0.008, 0.024] | 0.002 [-0.014, 0.018] | -0.006 (0.011) | +| Michigan | county | median_hh_income | beyond order 3 | 0.001 [-0.008, 0.009] | -0.000 [-0.007, 0.007] | -0.001 (0.006) | +| New York | county | households | order 3 | 0.028 [0.008, 0.049] | 0.025 [0.001, 0.050] | -0.003 (0.016) | +| New York | county | households | order 1 | 0.015 [-0.017, 0.047] | 0.021 [-0.005, 0.048] | 0.006 (0.021) | +| New York | county | households | order 2 | 0.005 [-0.023, 0.032] | 0.024 [-0.001, 0.049] | 0.019 (0.019) | +| New York | county | households | beyond order 3 | 0.031 [0.013, 0.050] | 0.031 [0.015, 0.047] | -0.000 (0.012) | +| New York | county | poverty_count | order 3 | -0.001 [-0.021, 0.019] | 0.019 [-0.006, 0.045] | 0.021 (0.017) | +| New York | county | poverty_count | order 1 | 0.014 [-0.014, 0.043] | 0.003 [-0.025, 0.031] | -0.011 (0.020) | +| New York | county | poverty_count | order 2 | 0.001 [-0.019, 0.021] | 0.006 [-0.012, 0.024] | 0.006 (0.014) | +| New York | county | poverty_count | beyond order 3 | 0.007 [-0.001, 0.016] | 0.008 [-0.003, 0.019] | 0.000 (0.007) | +| New York | county | poverty_rate | order 3 | -0.002 [-0.022, 0.019] | 0.019 [-0.006, 0.045] | 0.021 (0.017) | +| New York | county | poverty_rate | order 1 | 0.016 [-0.013, 0.044] | 0.003 [-0.025, 0.031] | -0.012 (0.020) | +| New York | county | poverty_rate | order 2 | 0.001 [-0.019, 0.020] | 0.006 [-0.012, 0.024] | 0.006 (0.014) | +| New York | county | poverty_rate | beyond order 3 | 0.007 [-0.001, 0.016] | 0.008 [-0.003, 0.019] | 0.001 (0.007) | +| New York | county | owner_share | order 3 | 0.018 [-0.008, 0.043] | -0.002 [-0.021, 0.017] | -0.020 (0.016) | +| New York | county | owner_share | order 1 | 0.028 [-0.007, 0.063] | -0.002 [-0.034, 0.030] | -0.030 (0.024) | +| New York | county | owner_share | order 2 | 0.025 [0.002, 0.047] | 0.010 [-0.015, 0.035] | -0.015 (0.017) | +| New York | county | owner_share | beyond order 3 | 0.036 [0.018, 0.055] | 0.012 [0.002, 0.022] | -0.024 (0.011) | +| New York | county | median_hh_income | order 3 | 0.004 [-0.019, 0.028] | 0.004 [-0.015, 0.024] | 0.000 (0.015) | +| New York | county | median_hh_income | order 1 | 0.024 [-0.003, 0.052] | 0.006 [-0.019, 0.031] | -0.018 (0.019) | +| New York | county | median_hh_income | order 2 | 0.006 [-0.019, 0.030] | -0.008 [-0.031, 0.015] | -0.013 (0.017) | +| New York | county | median_hh_income | beyond order 3 | 0.011 [-0.001, 0.022] | 0.012 [0.002, 0.021] | 0.001 (0.008) | + +## Findings + +Numbers below refer to the tables in this report (2020-2024 release unless +stated). r is the replicate correlation of two areas' sampling errors. + +1. **Noise floor.** One pairwise r from 80 replicates is noise-dominated. Analytic null SD 0.112 (95% |r| + 0.219). Real independent pairs (Michigan x New York tracts) have mean r within 0.001 of 0 but SD + 0.115-0.116 and 95% |r| 0.226-0.236; 6% of them exceed the analytic critical value. The reason is the + replicate design: in every variable and both states the same 3 directions of replicate space are + near-null (eigenvalues 0.02-0.11 of the mean vs the Marchenko-Pastur edge 0.69-0.77), so a + correlation has about 77 effective replicates (null SD 1/sqrt(77) = 0.114). Derangements of replicate + indices destroy this structure and give the textbook 0.112, so they understate the real floor + slightly. Practical floor: |r| below ~0.23 is indistinguishable from 0 for a single pair. +2. **Uncontrolled characteristics: no detectable neighbour correlation.** Poverty rate, owner share and + interpolated median household income have mean tract r within +/-0.003 of 0 at every contiguity + order, within and across counties, and at every distance, with 95% intervals of about +/-0.002-0.003 + (e.g. order 1: Michigan poverty rate -0.000 [-0.003, 0.002], New York owner share -0.001 [-0.002, + 0.001], Michigan median income 0.000 [-0.002, 0.003]). The poverty count is the same with a slight + negative tilt among neighbours in the same county (Michigan order 1 -0.004 [-0.007, -0.002]). + Counties: the class means for these variables lie in -0.015..+0.025 with intervals of about + +/-0.02-0.03, none clearly different from the cross-state null (which itself has small positive means, + +0.006 to +0.018, from the offset described in 4). The per-pair spread beyond the cross-state null is ~0 and + the split-half reliability of an individual pair's r is 0.02-0.07, the same as for far pairs: the + individual pairwise correlations are replicate noise. +3. **Controlled totals: negative correlation among neighbours in the same county.** Total population + (B01003, whose county total is controlled to the population estimates) and occupied housing units + (B25003_001, tied to the housing-unit controls) show clearly negative first-order correlations inside + counties: population order 1 within county Michigan -0.081 [-0.106, -0.056], New York -0.043 + [-0.065, -0.020]; households -0.028 [-0.039, -0.018] and -0.016 [-0.024, -0.008]. They decay fast + with order (population order 2: -0.018 / -0.010; order 3: -0.007 / -0.004; far within county: -0.001) + and vanish across county lines at every order (order 1 across county: +0.003 [-0.007, 0.014] and + +0.002 [-0.006, 0.009]). Rook (shared-edge) pairs are more negative than corner-only pairs (Michigan + population -0.080 vs -0.029). The aggregation ratio confirms the control mechanism without any + pairwise noise: Var(sum of a county's tracts) / sum of the tract variances is 0.000 for population in + Michigan and 0.005 in New York (the tract errors sum to exactly zero within a county), 0.22 / 0.26 for + households and 0.92 / 0.88 for the poverty count. Here the structure is real at the pair level too: + split-half reliability of an individual neighbour pair's r is 0.6-0.67 for population and 0.2-0.34 for + households (excess SD 0.13-0.16 and 0.06-0.08), i.e. strongly heterogeneous, with some neighbour pairs + far more negative than the mean (a few tracts sharing a county's controlled total). These are + 2020-2024 numbers; in 2015-2019 population behaves the same but households do not (6). +4. **A design-wide positive offset, not spatial covariance, at the county level.** County households + (and New York county owner share in 2015-2019, +0.02-0.04) show mean r of about +0.03 at every order + and distance - and + equally between Michigan and New York counties (+0.029-0.032), whose samples are independent. The + replicate estimates of these counties sit systematically above the full-sample estimate (mean + deviation about 0.18 of its RMS), so the uncentred 4/80 cross-products of any two such counties are + positive (0.18^2 = 0.03). This is part of what the ACS covariance formula returns, but it is not a + neighbourhood effect, and with 80 replicates it cannot be separated from a true common component. +5. **Structured models out of sample.** One correlation per contiguity order, fitted on 40 replicates, + predicts the other 40 better than independence only where (3) found structure: population +33 (MI) and + +17 (NY) nats per replicate, households +4.4 / +2.2; for every uncontrolled variable and for counties + the gain is within +/-0.2 of 0 (no improvement). Splitting by county adds a little for households + (+5.1) and, for population in Michigan, cannot be represented as a positive-definite correlation + matrix (projected, -165): the within-county constraint is an exact linear restriction (the covariance + is singular), not a small correlation. Adding a single far-within-county correlation makes every + controlled-total fit non-positive-definite. Ledoit-Wolf shrinkage toward the identity has intensity + 0.87 (40 replicates) to 0.93 (80) for tracts - it keeps only 7-13% of the sample correlation - and, + scored on held-out replicates after removing the design directions, it is worse than independence + for every variable (-25 to -50 nats for tracts). Without that projection it appeared to gain 30-90 + nats, entirely from the cross-half dependence created by the near-null design directions. +6. **Stability across releases (tracts and counties, 2015-2019 vs 2020-2024).** One 2015-2019 file is + missing: Michigan tract total population (`2019/data/5-year/140/B01003_26.csv.zip`). census.gov's + firewall answered that one URL with its 'Request Rejected' page (support ID 13979921158196382640) on + the first request and on the single delayed retry, while every other 2019 tract file, including New + York's B01003_36 from the same directory, downloaded normally; the rejection is URL-specific, not + rate limiting (see Data: unavailable files). So the 2015-2019 tract comparison has no Michigan + population row; everything else was analysed. Results: (a) **Uncontrolled variables are stable.** + Poverty count, poverty rate, owner share and median income stay at 0 to within +/-0.005 at every order + in both states and releases; of 108 tract class comparisons for them, 3 differ by more than 2 combined + SEs (about 5 expected by chance; largest |z| 2.5, all differences <= 0.004). (b) **Population (New + York) is stable**: order 1 -0.039 [-0.060, -0.018] in 2015-2019 vs -0.040 [-0.060, -0.020] (within + county -0.042 vs -0.043), the same fast decay, a county aggregation ratio of 0.000 again (tract errors + sum to zero within a county), and split-half pair reliability 0.60 (2020-2024: 0.61). (c) + **Households change sign.** Occupied housing units (B25003_001) in 2015-2019 have *positive* mean r + between neighbouring tracts in the same county - Michigan order 1 +0.025 [0.016, 0.033] (within county + +0.028), New York +0.012 [0.003, 0.020] - against -0.025 and -0.015 in 2020-2024 (differences 8.4 and + 4.7 combined SEs), still zero across county lines (+0.001 / -0.004) and beyond order 3. Their + aggregation ratio is 0.90 (Michigan) and 1.00 (New York) instead of 0.22 / 0.26: in 2015-2019 the tract + household errors are not constrained to (nearly) cancel within a county. The pair reliability (0.37 / + 0.28) shows the pattern is real, not replicate noise. The study cannot say why from the replicates + alone; a change between the releases in how occupied-unit estimates relate to the weighting controls + is the obvious candidate, not verified here. (d) County class means agree within about 1.5 combined + SEs for every variable and order except New York owner share beyond order 3 (+0.036 vs +0.012, + difference -0.024, SE 0.011) - the county-wide offset in (4), not a neighbour effect. + + +## C2 conclusion + +**The covariance structure is estimable only as a few structured summaries, and for the variables most +analyses map it is indistinguishable from zero.** + +* For characteristics that are not weighting controls (rates, shares, medians and most counts), the mean + sampling-error correlation between neighbouring tracts is 0 to within about +/-0.003 (95%), at orders + 1-3, within and across counties, in Michigan and New York; between neighbouring counties it is 0 to + within about +/-0.03. Individual pairwise correlations are replicate noise (reliability ~0.05). There + is nothing to estimate beyond "approximately independent", and an unrestricted or shrunk replicate + covariance would only add noise (Ledoit-Wolf loses out of sample). +* For controlled totals (population, households, and to a lesser degree counts tied to them) the + structure is real and estimable: negative, confined to first- and second-order neighbours inside the + same county (order 1: -0.08 to -0.04 for population, -0.03 to -0.015 for households in 2020-2024; + population equally in 2015-2019, households there +0.01 to +0.03 instead, see Findings 6), and summarised + far more precisely by the within-county aggregation ratio (population: tract errors sum to zero within + a county). It is a linear constraint, not a smooth correlation function: a one-parameter-per-order + model captures it out of sample, a full within-county correlation model is not positive definite. +* Using it. In a measurement-error correction (A1) or LISA null model (B), sampling errors of rates, + shares and medians can be treated as independent across areas, with that assumption cited to this + study. For controlled counts the correct treatment is the replicate covariance itself (4/80 D D^T, + rank 80 - usable for sums and contrasts, as census.replicate already does), or a structured model with + one negative correlation per contiguity order within county; a generic positive spatial correlation + (e.g. SAR errors) has the wrong sign for them. +* **For the LISA parameter grid (-0.2..0.5):** the data place ACS sampling-error neighbour correlation at + about 0 (|rho| <= 0.003) for rates, shares and medians, and at -0.02 to -0.08 (95% bounds -0.11 to + -0.01) for controlled counts in tracts; the only positive neighbour correlations beyond the design-wide + county offset of about +0.03 are 2015-2019 tract households, +0.01 to +0.03 (95% bounds up to +0.033) + within county. The release matters for controlled counts. Grid values of +0.1..0.5 are therefore + stress tests outside the observed range, not plausible ACS conditions for these variables; the + realistic end of the grid is -0.1..0.05. Correlated-error scenarios with large positive rho should be + stated as hypothetical (for example for variables or survey designs not studied here). +* Limits: two states, two releases at tract level (2015-2019 without Michigan population: firewall + rejection of that one file), 80 replicates (about 77 effective), and counties as a proxy for weighting + areas. The 95% |r| of 0.23 for a single pair means no individual pair can be + flagged; only class means are evidence. + + +## Limitations + +* Two states, two releases. Michigan and New York contain dense urban tracts (Detroit, New York City) and + sparse rural ones; other states' weighting areas and sample sizes differ. +* Counties stand in for ACS weighting areas. Census controls weights to population and housing-unit + estimates within weighting areas that are counties or groups of small counties, so "within county" is an + approximation of "within weighting area"; for small Michigan counties grouped into one weighting area the + negative covariance would appear "across county". +* The replicate bootstrap treats the 80 SDR replicate deviations as exchangeable draws. They are + constructed from a Hadamard design and are close to, but not exactly, iid; the delete-one-county jackknife + and the split-half comparisons are reported alongside as checks, and every interval uses the larger SE. +* Tract-pair classes beyond order 3 are random samples (sizes in the CSV), so their means carry sampling + error from that draw as well (included in both SEs). +* The median is a linear interpolation from B19001 brackets, not Census's published B19013 (computed from + finer internal distributions; no VRE file exists for it). Its replicate deviations are therefore those of + the interpolated statistic. +* Correlation is reported, not covariance in units: covariance in units scales with each area's + variance, which the published MOEs already give; only the correlation structure is new information. +* Distance bins use centroid distance; large rural tracts make "0-2 km" mostly urban pairs. diff --git a/docs/audits/inference_validation.md b/docs/audits/inference_validation.md index f26bed4..e21dd2a 100644 --- a/docs/audits/inference_validation.md +++ b/docs/audits/inference_validation.md @@ -1,6 +1,6 @@ # Inference validation (PLAN_ROUND_8 workstream F1) -Generated by `scripts/inference_validation.py` (full configuration) on 2026-11-08; Python 3.14.4; 19.6 min. Seed 20261102. Do not edit by hand. +Generated by `scripts/inference_validation.py` (full configuration) on 2026-11-09; Python 3.14.4; 19.5 min. Seed 20261102. Do not edit by hand. Nominal interval level 90%; acceptance band 88%-92% (PLAN_ROUND_8 sec. 14.3). Monte Carlo SE = sqrt(c(1-c)/R) (Fay-Herriot: between-replication SE of the per-replication coverage, since areas share a fit). Rows with status `info` are comparisons (naive fits, targets a method does not claim) and are not gated. diff --git a/docs/audits/lisa_validation.md b/docs/audits/lisa_validation.md new file mode 100644 index 0000000..dfa4bff --- /dev/null +++ b/docs/audits/lisa_validation.md @@ -0,0 +1,358 @@ +# LISA under ACS sampling error (PLAN_ROUND_16 workstream B) + +Simulation study of local Moran's I (LISA) and three alternatives when the mapped values are survey estimates. It covers sampling errors that are independent, heterogeneous or spatially correlated. + +- **Run:** 2026-11-08, Python 3.14.4, numpy 2.5.4, pandas 3.1.1, esda 2.10.0. +- **Command:** + + ``` + python scripts/inference_validation.py --studies lisa_validation_null lisa_validation_clusters --jobs 2 --out + ``` + +- **Configuration:** full, 400 maps per study, seed 20261102 (+1600 for the null study, +1601 for the cluster study). +- **Runtime:** single process on a shared, heavily loaded 16-core machine. `lisa_validation_null` took 25.7 min and `lisa_validation_clusters` 22.6 min. +- **Raw rows:** `--out` also writes a CSV next to the report, one row per scenario, error structure, method, weights and metric. `coverage` is the metric's mean and `mc_se` its Monte Carlo SE. CSVs under `docs/audits` are not committed (`.gitignore`); the tables below are taken from that file. +- **Code:** `census/lisa_validation.py`. All candidate methods are prototypes in validation code. **No tool default changed:** `local_morans` still uses two-sided p-values, 9999 permutations and BH at alpha 0.10. + +Every row has status `info`. The studies run only when named with `--studies`, because each takes more than the ~5 min budget for default-report studies. + +## 1. Questions kept apart + +1. **Local similarity beyond chance:** LISA's significance, the question LISA answers. +2. **Stability across replicate maps:** `replicate_quadrant_share`. It is a diagnostic, not a calibrated probability. +3. **Latent elevation:** whether an area's true value is beyond a threshold. This is Wong's (2021) question, and Sun et al.'s (2015). + +Because the truth is known in a simulation, **evidence of a genuine pattern** is scored against it. Significance (1) and stability (2) are reported separately from that evidence. Two error situations are also kept distinct: + +- **heterogeneous variances:** errors independent, SEs varying across space; +- **correlated errors:** errors covary between neighbours. + +The Round 15 "38%" result (`local_morans_size`) is about the first situation only. + +## 2. Design + +### Map + +- **Lattice:** 12 x 12, 144 areas, rook contiguity. +- **True values:** theta_i ~ N(50, 10²), independent. +- **Cluster scenarios:** the central 4 x 4 block is raised by 25, i.e. 2.5 true-value SDs. This is stronger than Round 15's 1.5 SD, where power was 1%. +- **Estimates:** y = theta + e. + +### Scenarios (B1) + +| # | Scenario (`scenario` column) | Truth | Sampling errors | +| :--- | :--- | :--- | :--- | +| 1 | `independent_homogeneous` | no clustering | independent, SE 6.7 for every area (median tract CV 13.4% x 50; reliability about 0.69) | +| 2 | `heterogeneous_independent` | no clustering | independent; SE = tract-profile CVs x 50, sorted along the lattice diagonal (the Round 15 "patterned" case) | +| 3 | `correlated` | no clustering | SE 6.7 for every area; correlation matrix R with mean rook-neighbour correlation r1 | +| 4 | `cluster_independent` | hot spot | as 1 | +| 5 | `cluster_correlated` | hot spot | as 3 | + +Each scenario is also analysed with no sampling error (`no_sampling_error`, the true values). This is the reference for the decomposition. + +### Error correlation (stated assumption, not an ACS measurement) + +- R is the SAR(1) covariance (I − λW)⁻¹(I − λW')⁻¹, with W the row-standardized rook matrix, rescaled to unit diagonal. +- λ is solved so that the mean rook-neighbour correlation equals r1. +- The SE stays homogeneous in scenarios 3 and 5, so they isolate covariance from heterogeneity. +- The grid spans -0.2 to 0.5, so workstream C's replicate-based neighbour correlations can be located in it. +- Negative r1 represents a possible effect of weighting controls. It gives a checkerboard structure in which second-order neighbours are positively correlated. + +| r1 (target = achieved) | λ | mean 2nd-order correlation | +| ---: | ---: | ---: | +| -0.20 | -0.350 | 0.043 | +| -0.10 | -0.181 | 0.011 | +| 0 | 0 | 0 | +| 0.10 | 0.181 | 0.011 | +| 0.25 | 0.427 | 0.067 | +| 0.50 | 0.724 | 0.256 | + +### Paired draws and replicates + +All r1 values and scenario 2 in one repetition share the true values and the standard-normal draw z (e = S L z, L the Cholesky factor of R). Differences between error structures are therefore paired (common random numbers). + +Replicates follow the ACS VRE scaling, with 80 maps per analysed map: + +- deviations d_r = S L z_r / 2, with z_r fresh for each replicate; +- each area's deviations are rescaled so that 4/80 Σ_r d_ir² = s_i² exactly, as in `census.simulation.make_replicates`; +- the replicate maps are y + d_r. + +Their cross-area correlation is the scenario's error correlation. + +## 3. Methods (B2), all on the same maps + +1. **`lisa`:** `census.spatial.local_morans` with its defaults. An area is flagged when `significant`; the cluster label is HH or LL. +2. **`lisa_stable`:** the same call with the replicates. An area is flagged when significant **and** `replicate_quadrant_share` ≥ 0.8 (a stated stability threshold). + - It is reported as a diagnostic filter, not as a test. + - `stable_only` flags HH or LL areas with share ≥ 0.8 and ignores significance. It shows what stability measures on its own. +3. **`lisa_eblup`:** `local_morans` (defaults) on spatial Fay-Herriot EBLUPs. + - Fit: `census.sae_spatial`, REML, intercept-only model, rook W, D = s_i². + - The model assumes independent sampling errors. + - Smoothing is measured directly: estimated ρ, Moran's I of the EBLUPs against the truth, hot-spot contrast relative to the truth, and the paired change in the false-cluster map indicator against `lisa` on the direct estimates. +4. **`wong_cl0.5`, `wong_cl0.75`:** the Wong (2021) heuristic hot spot / cold spot procedure (section 6), at the confidence levels of the paper's simulated examples. + - Thresholds: 50 ± 1.645 x 10. These are an analyst's domain thresholds, the true 5% tails. + - It is scored against its own target, not against LISA's false-positive definitions. + +## 4. Metrics (B3) + +Each metric is computed per map, then averaged over the 400 maps. The MC SE is the between-map SD / √400, because the areas of one map share a fit and a permutation run. Null areas are every area in scenarios 1-3 and the areas outside the block in 4-5. + +| Metric | Definition | +| :--- | :--- | +| `fpr` | flagged null areas / null areas (per-area false-positive rate) | +| `fdr` | false discovery proportion, flagged null / flagged (0 when nothing is flagged), averaged: the FDR that BH targets. **Under the complete null (scenarios 1-3) it equals the family-wise rate, the share of maps with any flag**: the quantity behind Round 15's "38%" | +| `false_cluster_map_k2`, `_k4` | **false-cluster map rate:** share of maps with at least one rook-connected component of areas flagged with the same cluster label (HH/LL; hot/cold) of size ≥ k made only of null areas. k = 2 and 4 are stated. Not Sun et al.'s (2015) false cluster rate, which weights predefined clusters | +| `power` | block areas flagged with the correct label (HH; hot) / 16 | +| `jaccard` | \|F ∩ T\| / \|F ∪ T\|, with F the areas flagged with the correct label and T the block (cluster-location accuracy) | +| `halo_share` | share of false discoveries that are rook neighbours of the block | +| W sensitivity | `lisa` also run with queen contiguity, k-nearest neighbours (k = 6) and a **misspecified W** (rook contiguity of a fixed random permutation of the areas, unrelated to the map). Run on the no-error, r1 = 0, r1 = 0.25 and heterogeneous maps | + +Wong-specific metrics: + +| Metric | Definition | +| :--- | :--- | +| `exceed_recall` | areas whose true value is beyond a threshold that are flagged with that label | +| `seed_precision` | seeds whose true value is beyond their threshold | +| `member_near_share` | grown (non-seed) members whose true value is within 5 (0.5 SD) of their seed's true value | +| `member_beyond_share` | grown members whose true value is itself beyond the threshold | +| `growth_per_seed` | grown members per seed | + +## 5. Results + +Values are the mean over 400 maps, with the MC SE in parentheses. All LISA rows use rook W unless stated otherwise. + +### 5.1 No true clustering (scenarios 1-3) + +| Map | `lisa` maps with any flag (= FDR) | `lisa` fpr | `lisa` false-cluster map k2 | `lisa_eblup` maps with any flag | `lisa_eblup` false-cluster map k2 / k4 | +| :--- | ---: | ---: | ---: | ---: | ---: | +| no sampling error | 0.085 (0.014) | 0.001 | 0.000 | | | +| independent homogeneous (r1 = 0) | 0.060 (0.012) | 0.001 | 0.000 | 0.070 (0.013) | 0.003 (0.003) / 0.000 | +| heterogeneous, independent | **0.378 (0.024)** | 0.004 | 0.007 (0.004) | 0.133 (0.017) | 0.000 / 0.000 | +| correlated r1 = -0.20 | 0.085 (0.014) | 0.001 | 0.000 | 0.110 (0.016) | 0.000 / 0.000 | +| correlated r1 = -0.10 | 0.062 (0.012) | 0.001 | 0.000 | 0.075 (0.013) | 0.000 / 0.000 | +| correlated r1 = +0.10 | 0.072 (0.013) | 0.001 | 0.000 | 0.072 (0.013) | 0.003 (0.003) / 0.000 | +| correlated r1 = +0.25 | 0.087 (0.014) | 0.001 | 0.003 (0.003) | 0.107 (0.016) | 0.013 (0.006) / 0.007 (0.004) | +| correlated r1 = +0.50 | **0.188 (0.020)** | 0.002 | 0.013 (0.006) | **0.340 (0.024)** | **0.098 (0.015) / 0.035 (0.009)** | + +Reading: + +- **Round 15's 38% is reproduced** (0.378) and characterized: + - it is the share of maps with **at least one significant area**, not with a contiguous cluster; + - contiguous false clusters of size ≥ 2 occur in 0.7% of maps under rook W; + - only 0.4% of areas are flagged. +- **Correlated errors** inflate the rate only for clearly positive neighbour correlation: 0.188 at r1 = 0.5, against 0.060 at r1 = 0. + - At -0.2 to +0.25 the rate stays at 0.06-0.09, within MC error of the no-error reference (0.085). Size at that reference is within nominal: BH at 0.10, conservative under dependence. +- **Heterogeneous variances** inflate the rate more than any correlation in the grid. These are two different failure mechanisms (section 5.4). + +### 5.2 Hot spot (scenarios 4-5) + +| Map | `lisa` power | `lisa` Jaccard | `lisa` FDR | `lisa` false-cluster map k2 | `lisa_eblup` power | `lisa_eblup` Jaccard | `lisa_eblup` FDR | `lisa_eblup` false-cluster map k2 / k4 | +| :--- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | +| no sampling error | 0.298 (0.007) | 0.298 (0.007) | 0.051 (0.005) | 0.003 (0.003) | | | | | +| r1 = 0 | 0.185 (0.007) | 0.185 (0.007) | 0.053 (0.007) | 0.000 | 0.302 (0.008) | 0.302 (0.008) | 0.067 (0.006) | 0.030 (0.009) / 0.005 (0.004) | +| r1 = -0.20 | 0.184 (0.007) | 0.184 (0.007) | 0.066 (0.007) | 0.000 | 0.283 (0.008) | 0.283 (0.008) | 0.065 (0.006) | 0.005 (0.004) / 0.003 (0.003) | +| r1 = -0.10 | 0.181 (0.007) | 0.181 (0.007) | 0.055 (0.007) | 0.000 | 0.287 (0.008) | 0.287 (0.008) | 0.056 (0.006) | 0.013 (0.006) / 0.003 (0.003) | +| r1 = +0.10 | 0.184 (0.007) | 0.184 (0.007) | 0.049 (0.006) | 0.003 (0.003) | 0.313 (0.008) | 0.312 (0.008) | 0.074 (0.007) | 0.037 (0.010) / 0.010 (0.005) | +| r1 = +0.25 | 0.189 (0.007) | 0.189 (0.007) | 0.070 (0.008) | 0.018 (0.007) | 0.329 (0.008) | 0.328 (0.008) | 0.099 (0.007) | 0.090 (0.014) / 0.025 (0.008) | +| r1 = +0.50 | 0.213 (0.007) | 0.213 (0.007) | 0.117 (0.009) | 0.037 (0.010) | 0.376 (0.009) | 0.374 (0.009) | 0.184 (0.009) | **0.260 (0.022) / 0.133 (0.017)** | + +Reading: + +- With rook W, LISA finds under a third of a 2.5-SD hot spot even without sampling error (0.298). LISA's local statistic is diluted at the block's edge and corners. +- Independent sampling error at reliability 0.69 cuts power to 0.185. +- Positive error correlation raises the apparent power slightly (0.213 at 0.5) and the FDR more (0.117). + +### 5.3 Smoothing by the spatial Fay-Herriot model (B2.3) + +| Map | ρ̂ (null) | Moran's I: truth / direct / EBLUP (null) | Δ false-cluster map k2 (null) | ρ̂ (hot spot) | hot-spot contrast / true contrast: direct / EBLUP | Δ false-cluster map k2 / k4 (hot spot) | +| :--- | ---: | :--- | ---: | ---: | :--- | :--- | +| r1 = 0 | -0.006 (0.008) | -0.008 / -0.009 / -0.012 | +0.003 (0.003) | 0.453 (0.006) | 0.994 / 0.858 | +0.030 (0.009) / +0.005 (0.004) | +| heterogeneous | -0.010 (0.008) | -0.008 / -0.010 / -0.016 | -0.007 (0.004) | | | | +| r1 = -0.20 | -0.161 (0.008) | -0.008 / -0.070 / -0.126 | 0.000 | 0.370 (0.006) | 0.996 / 0.848 | +0.005 (0.004) / +0.003 (0.003) | +| r1 = +0.10 | 0.072 (0.008) | -0.008 / 0.021 / 0.045 | +0.003 (0.003) | 0.492 (0.005) | 0.994 / 0.862 | +0.035 (0.009) / +0.007 (0.004) | +| r1 = +0.25 | 0.186 (0.008) | -0.008 / 0.066 / 0.130 | +0.010 (0.005) | 0.546 (0.005) | 0.993 / 0.866 | +0.072 (0.013) / +0.022 (0.007) | +| r1 = +0.50 | 0.354 (0.008) | -0.008 / 0.134 / **0.270** | **+0.085 (0.014)** | 0.623 (0.005) | 0.991 / 0.870 | **+0.223 (0.021) / +0.128 (0.017)** | + +Δ is the paired difference between `lisa_eblup` and `lisa` on the same map. The hot-spot Moran's I of the truth is 0.270; the direct estimates give 0.208 at r1 = 0, and the EBLUPs 0.343-0.493. + +### 5.4 Sensitivity to W, and the decomposition + +| Map | W | maps with any flag (null) | false-cluster map k2 (null) | power (hot spot) | FDR (hot spot) | halo share (hot spot) | false-cluster map k2 (hot spot) | +| :--- | :--- | ---: | ---: | ---: | ---: | ---: | ---: | +| no sampling error | rook | 0.085 (0.014) | 0.000 | 0.298 (0.007) | 0.051 (0.005) | 0.032 (0.017) | 0.003 (0.003) | +| no sampling error | queen | 0.077 (0.013) | 0.022 (0.007) | 0.547 (0.007) | 0.116 (0.006) | 0.390 (0.026) | 0.095 (0.015) | +| no sampling error | kNN 6 | 0.077 (0.013) | 0.018 (0.007) | 0.467 (0.006) | 0.112 (0.006) | 0.474 (0.027) | 0.065 (0.012) | +| no sampling error | misspecified | 0.080 (0.014) | 0.000 | 0.000 | 0.049 (0.011) | | 0.000 | +| r1 = 0 | rook | 0.060 (0.012) | 0.000 | 0.185 (0.007) | 0.053 (0.007) | 0.056 (0.024) | 0.000 | +| r1 = 0 | queen | 0.095 (0.015) | 0.015 (0.006) | 0.441 (0.007) | 0.116 (0.006) | 0.386 (0.027) | 0.090 (0.014) | +| r1 = 0 | kNN 6 | 0.068 (0.013) | 0.010 (0.005) | 0.362 (0.007) | 0.117 (0.007) | 0.429 (0.029) | 0.058 (0.012) | +| r1 = 0 | misspecified | 0.080 (0.014) | 0.003 (0.003) | 0.000 | 0.062 (0.012) | | 0.000 | +| heterogeneous | rook | 0.378 (0.024) | 0.007 (0.004) | | | | | +| heterogeneous | queen | 0.372 (0.024) | 0.125 (0.017) | | | | | +| heterogeneous | kNN 6 | 0.375 (0.024) | 0.098 (0.015) | | | | | +| heterogeneous | misspecified | **0.075 (0.013)** | 0.000 | | | | | +| r1 = +0.25 | rook | 0.087 (0.014) | 0.003 (0.003) | 0.189 (0.007) | 0.070 (0.008) | 0.053 (0.022) | 0.018 (0.007) | +| r1 = +0.25 | queen | 0.205 (0.020) | 0.062 (0.012) | 0.456 (0.008) | 0.185 (0.008) | 0.316 (0.021) | 0.215 (0.021) | +| r1 = +0.25 | kNN 6 | 0.172 (0.019) | 0.060 (0.012) | 0.379 (0.007) | 0.166 (0.008) | 0.363 (0.024) | 0.158 (0.018) | +| r1 = +0.25 | misspecified | 0.072 (0.013) | 0.003 (0.003) | 0.000 | 0.052 (0.011) | | 0.000 | + +The halo share for the misspecified W is about 0.09 with large SEs (few discoveries) and is omitted. + +Decomposition, comparing the same truth across rows: + +- **Sampling uncertainty (independent, homogeneous):** with rook W the null rate does not rise (0.085 → 0.060). Power falls from 0.298 to 0.185 (-0.113), and Jaccard likewise. This error is a loss of power, not false positives. +- **Heterogeneous variances:** the null rate rises 0.060 → 0.378 under rook, queen and kNN alike. It falls back to 0.075 under the misspecified W. The excess comes from the SE gradient being **aligned with the neighbourhood structure**: neighbours share similar SEs, so high-variance regions contain extreme values next to each other. It is not an artefact of a particular contiguity rule. +- **Correlated errors:** the null rate is 0.087 at r1 = 0.25 (rook) and 0.188 at r1 = 0.5. The permutation null assumes exchangeable values; positively covarying neighbour errors make neighbours alike without any true pattern. The misspecified W removes the excess (0.072). Negative r1 down to -0.2 did not inflate false flags. +- **W choice (not misspecification of the truth):** + - queen and kNN-6 double power over rook (0.44-0.55 against 0.19-0.30); + - they also double the FDR (0.11-0.19) and put 32-47% of false discoveries in the ring around the true hot spot; + - under correlated errors they amplify false clusters (false-cluster map k2 0.215 queen and 0.158 kNN against 0.018 rook at r1 = 0.25, hot spot). + - The tool's `spatial_autocorrelation` default W is queen. +- **Misspecified W** removes both power (0.000) and the dependence-driven false positives. It keeps the nominal null rate, as permutation inference should when W is unrelated to the values. + +### 5.5 Stability is not evidence (B4) + +`replicate_quadrant_share` (threshold 0.8): + +| Map | share of significant areas that are stable | mean share, significant null areas | mean share, significant true-cluster areas | `stable_only` fpr | `stable_only` false-cluster map k4 | +| :--- | ---: | ---: | ---: | ---: | ---: | +| null, r1 = 0 | 0.806 (0.077) | 0.908 (0.025) | | 0.324 (0.002) | 0.998 (0.003) | +| null, heterogeneous | 0.691 (0.035) | 0.851 (0.012) | | 0.321 (0.002) | 0.998 (0.003) | +| null, r1 = +0.50 | 0.858 (0.034) | 0.926 (0.014) | | 0.394 (0.003) | 1.000 | +| hot spot, r1 = 0 | 0.949 (0.008) | 0.924 (0.013) | 0.980 (0.003) | 0.355 (0.003) | 1.000 | +| hot spot, r1 = +0.50 | 0.966 (0.005) | 0.948 (0.007) | 0.986 (0.002) | 0.417 (0.003) | 1.000 | + +The null-scenario stable shares are averaged over the maps that have significant areas. + +- **Filtering by stability changes almost nothing.** For example, the null maps with any flag in the heterogeneous case go from 0.378 to 0.285, and hot-spot power from 0.185 to 0.182. +- **Falsely significant areas are almost as stable as true ones** (0.91-0.95 against 0.98). +- **Stability alone is not evidence of a pattern:** + - it marks a third or more of null areas as stable HH/LL; + - it gives a stable null "cluster" of size ≥ 4 on essentially every map; + - positive error correlation makes null labels **more** stable (0.926 at r1 = 0.5). +- The reason: a replicate map perturbs the published map by one more draw of the same error. A spurious pattern already in the estimates is not removed by that perturbation, and correlated perturbations preserve the quadrant structure. + +The three quantities are therefore separate: + +- **stability:** high almost everywhere; +- **significance:** LISA's BH flags, which control false discoveries reasonably except under heterogeneity aligned with W, or strong positive correlation; +- **genuine-pattern evidence:** only measurable here because the truth is known. + +### 5.6 Wong (2021) against its own target (latent elevation) + +| Map | CL | exceed recall | seed precision | member near seed | member beyond threshold | growth per seed | areas flagged | +| :--- | :--- | ---: | ---: | ---: | ---: | ---: | ---: | +| null, r1 = 0 | 0.5 | 0.750 (0.006) | 0.415 (0.005) | 0.446 (0.015) | 0.130 (0.010) | 0.185 (0.006) | 0.207 | +| null, r1 = 0 | 0.75 | 0.722 (0.007) | 0.415 (0.005) | 0.454 (0.029) | 0.208 (0.024) | 0.037 (0.002) | 0.182 | +| null, heterogeneous | 0.5 | 0.811 (0.006) | 0.316 (0.004) | 0.329 (0.007) | 0.050 (0.003) | **1.194 (0.049)** | 0.501 | +| null, heterogeneous | 0.75 | 0.729 (0.006) | 0.316 (0.004) | 0.330 (0.016) | 0.090 (0.010) | 0.102 (0.004) | 0.254 | +| null, r1 = -0.20 | 0.5 | 0.753 (0.006) | 0.414 (0.005) | 0.438 (0.015) | 0.154 (0.011) | 0.167 (0.006) | 0.205 | +| null, r1 = +0.25 | 0.5 | 0.742 (0.006) | 0.415 (0.005) | 0.446 (0.013) | 0.098 (0.008) | 0.217 (0.006) | 0.213 | +| null, r1 = +0.50 | 0.5 | 0.733 (0.006) | 0.420 (0.005) | 0.505 (0.012) | 0.060 (0.006) | 0.263 (0.007) | 0.219 | +| hot spot, r1 = 0 | 0.5 | 0.833 (0.004) | 0.599 (0.004) | 0.425 (0.014) | 0.180 (0.011) | 0.146 (0.005) | 0.272 | +| hot spot, r1 = 0 | 0.75 | 0.807 (0.004) | 0.599 (0.004) | 0.435 (0.028) | 0.227 (0.023) | 0.030 (0.002) | 0.245 | +| hot spot, r1 = +0.50 | 0.5 | 0.838 (0.004) | 0.604 (0.004) | 0.484 (0.011) | 0.143 (0.009) | 0.203 (0.005) | 0.285 | + +Growth per seed over the whole r1 grid, at CL 0.5: + +| Map | -0.2 | -0.1 | 0 | 0.1 | 0.25 | 0.5 | +| :--- | ---: | ---: | ---: | ---: | ---: | ---: | +| null | 0.167 | 0.173 | 0.185 | 0.196 | 0.217 | 0.263 | +| hot spot | 0.135 | 0.136 | 0.146 | 0.157 | 0.176 | 0.203 | + +The hot-spot power of the hot label against the block is 0.82 at CL 0.5 and 0.78 at CL 0.75 (Jaccard 0.45-0.46). That is much higher than LISA's, because the block's true values are mostly beyond the threshold. + +- **Seeds:** at ACS-like reliability (0.69) most seeds are not truly beyond the threshold. Precision is 0.42 with no true clusters and 0.32 under heterogeneous SEs. Thresholding point estimates selects noise, and the paper's procedure has no step that accounts for a seed's own SE. +- **Growth:** + - it rises with the SEs, as the paper says (sec. 5.2): 1.19 grown members per seed under the SE gradient against 0.19 with homogeneous SEs; + - it rises with positive error correlation, monotonically from 0.167 (r1 = -0.2) to 0.263 (r1 = 0.5) at CL 0.5; + - this confirms the expectation that the independence-based difference test under-rejects when seed and neighbour errors covary positively, and over-rejects when they covary negatively. +- **Grown members:** fewer than half are truly within 0.5 SD of their seed (0.33-0.52), and few are themselves beyond the threshold. + +## 6. Wong (2021): what the paper specifies and what was implemented + +**Source:** Wong, D.W.S. (2021). Issues in the Current Practices of Spatial Cluster Detection and Exploring Alternative Methods. *IJERPH* 18(18):9848, doi:10.3390/ijerph18189848, CC BY. + +**Read:** the full text, through PubMed Central PMC8466737 (the BioC export and the JATS XML for Eq. 1). The MDPI page returned HTTP 403. + +Verified from the text: + +- **Thresholds and seeds** (sec. 4.2): the analyst sets low and high thresholds from domain knowledge (the "global criteria"). Areas whose **estimates** are beyond them are seeds. +- **Growing:** + - a first-order neighbour joins when its estimate is "not statistically different" from the **seed's** estimate, by "a t-test given a confidence level"; + - second- and higher-order neighbours of members are compared with the seed again; + - growth stops when every candidate differs; + - a seed with no joining neighbour is a single-unit spot; + - multiple seeds give multiple spots. +- **Join rule, from the worked example** (sec. 5.1 and Table 2): + - Louisiana (p = 0.65 against Mississippi) joins at CL ≈ 0.648; + - Rhode Island and New York (p = 0.52 and 0.50 against Connecticut) join just below CL 0.50; + - Alabama (p = 0.13 against Mississippi) joins at CL 0.128; + - at CL 0.95 no neighbour joins. + + The operational rule is therefore **join when p ≥ CL**: lowering CL grows the spots. This is the paper's convention and is not "fail to reject at α = 1 − CL", which would act in the opposite direction. +- **Eq. (1):** prob(x) = C(n, x) CL^x (1 − CL)^(n−x), "assumes the independence of events". The paper's value 6.9674e-18 for n = 42, x = 1, CL = 0.648 was reproduced. + - **Not implemented as an inference.** It is the probability of x joins if each join occurred with probability CL, not a null distribution for "no clustering". +- **Not specified, and our choices:** + - test formula, degrees of freedom and sidedness: two-sided z = (y_s − y_j)/√(s_s² + s_j²), the ACS comparison formula, assuming independent errors; + - how spots from different seeds meet: an area claimed by a hot and a cold seed is labelled `both` (not observed with these thresholds); + - growth uses rook neighbours. +- **Not implemented:** + - the class separability classification of sec. 4.1, which is a choropleth-break method; + - the paper's web application. + +The paper reports no accuracy metrics for its simulations. + +## 7. Conclusions + +1. **The Round 15 caveat overstated the finding.** "Spurious clusters in 38% of maps" under spatially patterned SEs is, precisely: + - 38% of maps with at least one significant area (0.4-0.5% of areas); + - under **heterogeneous but independent** errors; + - with contiguous false clusters in under 1% of maps under rook W. + + It said nothing about correlated errors. The caveat text is corrected (section 9). +2. **The two error situations fail differently:** + - Heterogeneous variances aligned with the neighbourhood structure raise the null rate strongly (0.38), under any contiguity-based W. + - Positively correlated errors raise it only at stronger correlation (0.19 at r1 = 0.5; 0.06-0.09 for r1 between -0.2 and 0.25). + - Where workstream C's measured neighbour correlations fall in -0.2 .. 0.5 determines which regime real ACS maps are in. That reading belongs to C's results; nothing here assumes a value. +3. **Sampling error costs power more than size.** Under independent homogeneous errors the null rate stays nominal, and power drops by a third. +4. **`replicate_quadrant_share` should not be offered as a check against false positives.** Falsely significant areas are as stable as true ones, and stability rises with positive error correlation. +5. **Spatial Fay-Herriot smoothing induces and alters patterns:** + - Without true clustering and with independent errors it is harmless (ρ̂ ≈ 0). + - With correlated errors, ρ̂ absorbs the error correlation (ρ̂ 0.35 at r1 = 0.5 with truth ρ = 0). Moran's I doubles (0.134 → 0.270 against a true -0.008), maps with any flag rise from 0.19 to 0.34, and false clusters of ≥ 2 areas rise by 8.5 points. + - With a true hot spot it raises power (0.185 → 0.302, matching the no-error reference) but attenuates the hot-spot contrast by about 14%. It adds false clusters around it: +3 points at r1 = 0, +22 points at r1 = 0.5. + - "Rerun on Fay-Herriot estimates" is therefore not a neutral sensitivity check. +6. **W choice is a power/precision trade-off,** not a correctness question. Queen and kNN double power, and double FDR and halo false discoveries. Under correlated errors they amplify false clusters most. +7. **Wong (2021) answers a different question** (latent elevation). As specified it: + - selects seeds by point estimate, so precision is low at ACS reliability (0.32-0.60); + - grows spots on absence of evidence, so growth rises with SEs and with positive error correlation. + + It is not a replacement for LISA and is not recommended as a tool method in this form. + +No method is preferred by false positives alone. LISA on EBLUPs has the best power but the worst induced false clusters under correlated errors. Rook LISA has the fewest false clusters but low power. The stability filter changes neither. + +## 8. Proposals (not implemented) + +- **P-B1. No change to the `local_morans` defaults.** The evidence does not support a different default p-value, permutation count or correction. +- **P-B2. Caveat and guidance:** + - stop recommending `replicate_quadrant_share` or Fay-Herriot reruns as false-positive checks; + - describe `replicate_quadrant_share` as sampling stability only; + - describe EBLUP LISA as higher power with smoothing-induced clusters, larger when errors are positively correlated. + + The caveat text was corrected only where it mischaracterized the finding. A fuller rewrite of the guidance is an owner decision. +- **P-B3. Correlated-error null (later round, conditional on C):** if workstream C finds neighbour error correlations that are estimable and positive, prototype a conditional-permutation alternative. It would simulate the null distribution of each Ii from the replicate-based (structured, regularized) error covariance instead of exchangeable permutations, and be validated with this harness (scenarios 3 and 5). Not before C's evidence. +- **P-B4. Latent elevation (question 1):** if wanted, the principled route is posterior exceedance probabilities, P(theta_i > threshold), from a model with known sampling variances, thresholded as in Sun et al. (2015, Procedure 1). + - A spatial Fay-Herriot Gaussian approximation (EBLUP, MSE) would be **our adaptation**, not a paper method. It needs its own validation, using this harness's `exceed_recall` and `seed_precision`. + - Wong's point-estimate seeds are the failure to avoid. +- **P-B5. Weights guidance:** state in the LISA notes that queen and kNN roughly double power and FDR relative to rook on a lattice, and put most extra false discoveries next to real clusters. The `spatial_autocorrelation` default (queen) stays. + +## 9. Changes made in this workstream + +- `census/lisa_validation.py`: the scenarios, error-correlation calibration, replicate generation, the Wong procedure, metrics, and the studies `lisa_validation_null` and `lisa_validation_clusters`, registered in `census.simulation.STUDIES` as the on-demand `lisa` group. +- `scripts/inference_validation.py`: routes the `lisa` group only when named with `--studies`. +- `census/spatial.py` (`LOCAL_MORAN_CAVEAT` and docstrings) and `census/workflows_geo.py` (docstring) are corrected: + - "spatially patterned sampling error produced spurious clusters in 38% of maps" became "heterogeneous but independent errors gave at least one significant area in 38% of maps (0.5% of areas); correlated errors were not tested by that study"; + - `replicate_quadrant_share` is described as a stability check, not a probability that a cluster is real; + - the unvalidated recommendation to "check replicate_quadrant_share or rerun on Fay-Herriot estimates" was removed. Sections 5.3 and 5.5 show neither is a false-positive check. +- `docs/audits/CHECK_MAP.md`: the check-map rows and the study description. +- `tests/test_lisa_validation.py`: building blocks (error-correlation calibration, replicate scaling and correlation, Wong's join rule against the paper's worked example, metrics, misspecified W), registration, and seeded tiny runs. diff --git a/docs/audits/spatial_regression_validation.md b/docs/audits/spatial_regression_validation.md new file mode 100644 index 0000000..338f79d --- /dev/null +++ b/docs/audits/spatial_regression_validation.md @@ -0,0 +1,396 @@ +# Spatial regression under ACS sampling error: validation (PLAN_ROUND_16 workstream A) + +**Date:** 2026-11-08. **Status:** findings and proposals; every study row is `info` (gating is an owner decision). +**Code:** `census/simulation_spreg.py` (studies and study-only estimators), `census/spatial.py` (lag-model effects). +**Tests:** `tests/test_spatial_regression_validation.py`. +**Reproduce:** `python scripts/inference_validation.py --studies spatial_error_attenuation spatial_error_attenuation_correlated spatial_dgp_recovery common_factor_lr common_factor_lr_acs --jobs 5 --out /tmp/r16a.md` +(seed 20261102; Python 3.14, spreg 1.9.1; BLAS pinned to one thread; about 28 min wall with 5 processes on a loaded 16-core machine). +Single-process runtimes: `spatial_error_attenuation` 28 min, `spatial_error_attenuation_correlated` 27 min, `spatial_dgp_recovery` 26 min, `common_factor_lr` 27 min, `common_factor_lr_acs` 6.5 min. +All exceed the 5-minute budget, so none is in the default full report; they run only when named. + +Principle (PLAN_ROUND_16 sec. 0): validation first. No estimator was added to the tool in this workstream. The tool gained the lag-model effects (A2, owner-approved) and measured caveats. + +## Common design + +- **Lattice and weights.** 12 x 12 lattice (144 areas), row-standardized W: rook, queen, or kNN with k = 6. kNN uses cell centres with a fixed 1e-6 jitter, so ties break the same way on every machine. The kNN W is asymmetric. +- **Variables.** x ~ N(50, 24^2), beta = 0.5, innovations e ~ N(0, 8^2). These match the Round 15 study `spatial_regression_coverage`. +- **Outcome sampling error.** Independent d_i ~ N(0, s_i^2). s_i = CV_i x mean(y*), with CV_i from the tract profile (median 13.4%; `census.simulation.CV_PROFILES`). "CV x k" scales every CV by k. +- **Correlated sampling error (stated assumption).** d = S L z, where S = diag(s) and L L' = R(gamma). R(gamma) is the correlation matrix of a SAR(gamma) process on the same W. With gamma = 0.3 and 0.6, the mean first-order neighbour error correlation is: + - rook: 0.17 and 0.38; + - queen: 0.11 and 0.29; + - kNN: 0.12 and 0.31. + + This is a hypothesis about the structure, not an estimate of the ACS covariance (workstream C). +- **Inference reported.** Nominal 90% Wald intervals. +- **Monte Carlo SEs.** + - coverage and rejection rates: sqrt(p(1-p)/R); + - bias: sd / sqrt(R); + - RMSE: by the delta method. +- **Seeds.** Every cell has its own seed, cfg.seed + a fixed offset from the cell's position, so results do not depend on run order or process. `tests/test_spatial_regression_validation.py` checks this. + +## Verification of the study estimators (offline tests) + +| Check | Result | +| :--- | :--- | +| `sem_ml` and `sdm_ml` (eigenvalue-based concentrated ML) vs spreg `ML_Error` and `ML_Lag(slx_lags=1)`, rook/queen/kNN | log-likelihoods equal to 5 decimals; rho and coefficients equal | +| Nesting: SDM log-likelihood at the SEM estimates with theta = -lambda beta (constant (1 - lambda) alpha) | equals the SEM log-likelihood (same data, same full Jacobian, same constants); LR >= 0 | +| `me_sem_fit` (diagonal Sigma, ML) vs `census.sae_spatial` ML | rho, A and beta equal (4 decimals) | +| `me_sem_fit` (diagonal Sigma, REML) vs `spatial_fay_herriot` | rho and rho_se equal (4 decimals) | +| `me_sem_fit` with Sigma = 0 | equals `sem_ml` | +| Lag effects vs independent S(W) = (I - rho W)^-1 algebra, with and without an island | equal to 10 decimals (GMM and ML) | +| Lag effects vs spreg `sp_multipliers['full']` | equal to 6 decimals | +| Delta-method SEs vs numerical gradient | equal (rtol 1e-6) | +| Delta-method SEs vs Monte Carlo draws, n = 400 | within 5% | +| spreg RS additivity (Koley & Bera): RS_rho,gamma = RS_rho + RS*_gamma = RS*_rho + RS_gamma | holds, but by construction: spreg defines the robust tests as these differences | + +## A1. Attenuation of lambda in the error model + +### Source verification + +- **Lorincz-Comi et al. (2021), *Spatial Statistics* 45:100523: not implemented.** + - The abstract is accessible, but it does not state the lambda correction. + - The coordinator's full-text reading (docs/literature/full_text_reading_notes.md) shows it is a method-of-moments **covariate** measurement-error correction for GWR. It does not address lambda or SEM/SAR models. Its global analogue is the existing `regression_calibration`. +- **The candidate tested is the engine's existing spatial Fay-Herriot model** (`census.sae_spatial`; Petrucci & Salvati 2006; Molina & Marhuenda 2015, *The R Journal* 7(1), eqs. (1) and (6), read in full; Donegan, Chun & Griffith 2021 per the coordinator's notes): + - y = X beta + u + e, with u = rho W u + eps (SAR(1)); + - e_d independent N(0, psi_d), psi_d **known**. + + This is algebraically the SEM with classical independent outcome error: V = sigma^2 [(I - rho W)'(I - rho W)]^-1 + diag(SE^2). Its rho is the SEM's lambda corrected for that error. +- **What the correction needs.** Only the sampling **variances**, and it **assumes independence**: the source's model statement is "e_d ind N(0, psi_d)". No accessible source gives a version with error covariances. The study's dense-Sigma variant (V = A C(lambda) + Sigma) is an **oracle** that knows the true Sigma, not a method. + +### Design + +- **Independent errors** (`spatial_error_attenuation`): W in {rook, queen, kNN}, lambda in {0, 0.3, 0.6}, CV x {0, 0.5, 1, 2}; 300 repetitions per cell. Estimators: + - the tool default, spreg ML_Error ("naive"); + - spatial Fay-Herriot rho by REML (`spatial_fay_herriot`'s default; SE from the REML information, as its `rho_se`); + - spatial Fay-Herriot rho by ML. +- **Correlated errors** (`spatial_error_attenuation_correlated`): the same W and lambda, CV x 1, gamma in {0.3, 0.6}, plus the oracle. + +### Results: lambda, independent errors (300 reps per cell; MC SE of coverage <= 0.029, of bias <= 0.02) + +**Rerun 2026-11-09 on the shipped estimator.** The SFH rows (both A1 studies) now call +`census.spatial.error_model_known_variances`, the function behind `spatial_regression(correct_outcome_error=True)` +(census.sae_spatial: A >= 0, |rho| <= 0.999, six starts, score-based convergence check), instead of the study-only +`me_sem_fit` (log A parameterization, |lambda| <= 0.99, two starts, L-BFGS-B success flag), so the coverage below +describes the tool. Full configuration, `--jobs 0`, 51.7 min. Every SFH number moved by at most 0.007 (the old run +dropped its 0.6% non-converged fits; the shipped optimizer converged in all 8,100 fits per estimator); naive and +oracle rows are unchanged. + +Coverage of nominal 90% intervals: + +| W | lambda | CV x | naive | SFH REML | SFH ML | +| :--- | ---: | ---: | ---: | ---: | ---: | +| rook | 0 | 1 | 0.920 | 0.890 | 0.867 | +| rook | 0.3 | 0.5 / 1 / 2 | 0.790 / 0.533 / 0.247 | 0.873 / 0.880 / 0.873 | 0.860 / 0.880 / 0.877 | +| rook | 0.6 | 0.5 / 1 / 2 | 0.657 / 0.140 / 0.003 | 0.913 / 0.863 / 0.863 | 0.910 / 0.870 / 0.883 | +| queen | 0.3 | 0.5 / 1 / 2 | 0.823 / 0.683 / 0.420 | 0.900 / 0.893 / 0.883 | 0.877 / 0.883 / 0.893 | +| queen | 0.6 | 0.5 / 1 / 2 | 0.690 / 0.343 / 0.033 | 0.880 / 0.933 / 0.897 | 0.880 / 0.927 / 0.907 | +| kNN | 0.3 | 0.5 / 1 / 2 | 0.850 / 0.660 / 0.417 | 0.917 / 0.900 / 0.867 | 0.907 / 0.890 / 0.883 | +| kNN | 0.6 | 0.5 / 1 / 2 | 0.757 / 0.330 / 0.017 | 0.880 / 0.887 / 0.920 | 0.893 / 0.883 / 0.920 | + +Bias (lambda-hat minus lambda) at CV x 1: + +| W | lambda | naive | SFH REML | SFH ML | +| :--- | ---: | ---: | ---: | ---: | +| rook | 0.3 / 0.6 | -0.175 / -0.274 | -0.025 / -0.017 | -0.045 / -0.035 | +| queen | 0.3 / 0.6 | -0.178 / -0.254 | -0.020 / -0.031 | -0.053 / -0.057 | +| kNN | 0.3 / 0.6 | -0.183 / -0.248 | -0.033 / -0.031 | -0.064 / -0.055 | + +RMSE at CV x 1, lambda 0.6: + +| Estimator | rook | queen | kNN | +| :--- | ---: | ---: | ---: | +| naive | 0.294 | 0.286 | 0.283 | +| SFH REML | 0.116 | 0.131 | 0.144 | + +At lambda 0 the corrected estimators have larger RMSE than the naive one (rook 0.165 vs 0.105), the price of estimating one more variance component. + +**Other observations:** +- **Bias grows with the error.** Naive bias at lambda 0.6 is -0.12, -0.27 and -0.46 at CV x 0.5, 1 and 2 (rook). +- **The tool's error-model beta is unaffected.** Its intervals covered 0.85-0.93 in every cell, corrected or not (the outcome error is independent of x). +- **SE calibration.** The ratio mean SE / SD of the estimates is 0.84-1.08 for every estimator. +- **Numerical stability.** + - Boundary estimates (|lambda| >= 0.98, or the variance component A at 0): 0 for naive. For SFH, 0 except at CV x 2 with lambda <= 0.3 on queen/kNN (0.7-2.0%). + - Non-convergence: 0 of 8,100 fits for REML and ML (shipped convergence rule: optimizer success or a projected, scaled score below 1e-3; the study-only `me_sem_fit` had 50 and 37 failures by its L-BFGS-B success flag). Naive: 0. + +### Results: correlated errors (300 reps per cell) + +Coverage of nominal 90% intervals: + +| W | lambda | gamma | naive | SFH REML | SFH ML | oracle (true Sigma) | +| :--- | ---: | ---: | ---: | ---: | ---: | ---: | +| rook | 0 | 0.3 / 0.6 | 0.840 / 0.517 | 0.830 / 0.593 | 0.860 / 0.643 | 0.900 / 0.883 | +| rook | 0.3 | 0.3 / 0.6 | 0.763 / 0.890 | 0.787 / 0.647 | 0.813 / 0.683 | 0.847 / 0.856 | +| rook | 0.6 | 0.3 / 0.6 | 0.303 / 0.633 | 0.913 / 0.810 | 0.920 / 0.830 | 0.920 / 0.913 | +| queen | 0 | 0.3 / 0.6 | 0.827 / 0.550 | 0.833 / 0.657 | 0.840 / 0.693 | 0.880 / 0.889 | +| queen | 0.3 | 0.3 / 0.6 | 0.810 / 0.867 | 0.880 / 0.740 | 0.910 / 0.760 | 0.936 / 0.893 | +| queen | 0.6 | 0.3 / 0.6 | 0.517 / 0.750 | 0.870 / 0.827 | 0.887 / 0.873 | 0.886 / 0.930 | +| kNN | 0 | 0.3 / 0.6 | 0.820 / 0.517 | 0.803 / 0.600 | 0.837 / 0.643 | 0.863 / 0.897 | +| kNN | 0.3 | 0.3 / 0.6 | 0.833 / 0.883 | 0.867 / 0.717 | 0.897 / 0.770 | 0.923 / 0.896 | +| kNN | 0.6 | 0.3 / 0.6 | 0.487 / 0.717 | 0.870 / 0.793 | 0.877 / 0.817 | 0.890 / 0.857 | + +Bias at lambda 0 with gamma 0.6: + +| Estimator | rook | queen | kNN | +| :--- | ---: | ---: | ---: | +| naive | +0.18 | +0.19 | +0.21 | +| SFH REML | +0.19 | +0.20 | +0.22 | +| oracle | -0.001 | -0.05 | -0.03 | + +**How to read this:** +- **Spatially correlated sampling error is read as spatial dependence of the disturbances.** With no true dependence (lambda 0), both the naive estimate and the independence-based correction report lambda of about 0.2. +- **The independence-based correction breaks under correlation**, as expected: + - coverage 0.59-0.69 at gamma 0.6 and lambda 0; + - 0.65-0.77 at lambda 0.3. +- **The naive coverage of 0.87-0.89 at lambda 0.3, gamma 0.6 is a coincidence**: attenuation from the error variance and inflation from the error correlation cancel. +- **The oracle stays near nominal (0.847-0.936)**, so the failure comes from the covariance assumption, not from the estimator. +- **beta intervals stayed at 0.85-0.94** in every correlated cell. + +### Conclusion and proposal (A1) + +- **Independent errors: the existing spatial Fay-Herriot model validates as the lambda correction.** + - REML coverage 0.86-0.93 in every lambda > 0 cell (all W, CV x 0.5-2), vs 0.003-0.85 for the tool default; 0.83-0.89 at lambda 0 (lowest at CV x 2 on rook). + - Bias at most 0.11 in magnitude (CV x 2), usually <= 0.035. + - Stable: no non-convergence in 16,200 fits, rare boundary fits (at most 2% of a cell). +- **It fails when sampling errors are spatially correlated.** Whether ACS errors are correlated enough to matter is the workstream C question (C1/C2). +- **Not implemented as a default** (PLAN_ROUND_16 A1: proposal only; and its key assumption is untested for the ACS until C reports). +- **Proposal for the owner:** expose `spatial_fay_herriot(...).provenance['rho']` / `rho_se` as the "lambda corrected for independent outcome sampling error" route, from `spatial_regression(model='error')`: + - either as an option, `correct_outcome_error=True`, that refits through `census.sae_spatial` and reports rho, rho_se and the GLS beta; + - or as a cross-reference in the error-model notes. + + The route should carry the independence caveat, with the C1 neighbour-correlation evidence attached. If C shows non-negligible neighbour correlation, the oracle results say a correction would need an estimated error covariance (a research extension). +- **Wording.** In the SEM, lambda describes the **diffusion of shocks** through the disturbances (Lacombe & LeSage 2015), not a substantive spillover from covariates. The attenuation is measurement error in y diluting that diffusion. + +## A2. Effects for the lag model + +### What spreg 1.9.1 provides + +`ML_Lag`, `GM_Lag` and the regime and panel variants accept `spat_impacts` = "simple" / "full" / "power" / "all": +- `spreg.sputils.spmultiplier` computes the average multipliers (ati = 1/(1-rho); adi = tr(S)/n by "full" (dense inverse) or "power" (power series); aii = ati - adi); +- `spreg.output._summary_impacts` / `sputils._sp_effects` multiply them by beta; +- results go to `reg.sp_multipliers` and the printed summary; +- `i_multipliers` gives observation-level multipliers. + +**There is no inference** (no SEs or intervals for the impacts), and the default "simple" method sets adi = 1, i.e. direct = beta. That is not the LeSage & Pace direct effect. + +### Tool change + +`spatial_regression(model='lag')` returns `ModelResult.effects`: per covariate, the average direct, indirect and total effects. + +- **Point estimates.** Computed from the eigenvalues of W, once per fit: direct = beta x mean 1/(1 - rho w_i); total = beta / (1 - rho); with islands, total = beta (1 - s + s/(1 - rho)), s the share of areas with neighbours. These equal spreg's "full" multipliers and the independent S(W) algebra (tests). +- **Uncertainty.** + - se_model: delta method from the fit's (beta, rho) covariance; + - se_sampling: from the 80 replicate refits when replicates are given; + - intervals follow the fit's target. + + The coefficient covariance is in `diagnostics['coefficient_vcov']`, so users can draw simulated (LeSage & Pace) intervals. Effects are not computed above 5,000 areas (dense eigenvalues); a note says so. +- **Effects first:** + - `to_dict()` puts `effects` before `coefficients`; + - `summary()` starts "EFFECTS FIRST ..."; + - the first note says beta is NOT the marginal effect, and the coefficient table has an `interpretation` column saying so; + - `report_tables.effects_table` renders Direct / Indirect / Total; + - `regression_table` adds a note for lag models; + - the workflow's `next_step` says to report the effects table first. +- **Labelling of the rho/lambda p-value.** A note says it tests the parameter = 0 within the fitted model and "is not a specification test" (Juhl 2021, sec. 3.2). + +### Effects coverage (from A3, SAR lag DGP, rook 12x12, 400 reps; MC SE <= 0.025) + +| rho | outcome error | estimator | direct | indirect | total | indirect, simulated interval | total, simulated interval | +| ---: | :--- | :--- | ---: | ---: | ---: | ---: | ---: | +| 0.3 | ACS | GMM [default] | 0.918 | 0.868 | 0.890 | 0.898 | 0.910 | +| 0.6 | ACS | GMM [default] | 0.870 | 0.805 | 0.832 | 0.965 | 0.980 | +| 0.3 | none | GMM | 0.912 | 0.892 | 0.892 | 0.882 | 0.880 | +| 0.6 | none | GMM | 0.902 | 0.928 | 0.925 | 0.905 | 0.905 | +| 0.3 | ACS | ML | 0.902 | 0.400 | 0.580 | 0.490 | 0.648 | +| 0.6 | ACS | ML | 0.902 | 0.002 | 0.018 | 0.008 | 0.032 | +| 0.3 / 0.6 | none | ML | 0.908 / 0.890 | 0.865 / 0.845 | 0.858 / 0.860 | 0.878 / 0.872 | 0.872 / 0.880 | + +**Delta method vs simulated intervals:** +- The tool's delta-method intervals are near nominal for GMM without sampling error and at rho 0.3 with it. +- At rho 0.6 with ACS error, GMM's rho is imprecise (SD about 0.26). Some samples have rho-hat near 1, so the indirect and total effects are heavy-tailed: + - mean bias +0.79, RMSE 4.6, against a true indirect effect of 0.685; + - the delta method under-covers (0.80-0.83); + - simulated percentile intervals over-cover (0.97-0.98). +- ML lag effects inherit ML's rho failure under outcome error (Round 15). +- The direct effect is always well covered (about beta). + +A measured caveat (`LAG_EFFECTS_CAVEAT`) is now in the lag-model notes. + +**Proposal:** +1. Add simulated (LeSage & Pace) percentile intervals and medians as an option, `effects_intervals="simulated"`. The draws are cheap once the eigenvalues are known. +2. Consider making them the default at high rho, after the owner reviews this table. The full text of LeSage & Pace 2014 recommends simulated intervals and medians, because total effects can be asymmetric. + +## A3. DGP set: what each model recovers (rook 12x12, 400 reps per cell, with and without ACS outcome error) + +DGPs: +- **SEM:** lambda 0.6. +- **SAR lag:** rho 0.3 and 0.6. +- **SDM:** rho 0.4, beta 0.5, theta 0.3. +- **Omitted spatially correlated covariate:** z SAR(0.7), corr(x, z) = 0.5, coefficient 8 on standardized z. +- **Juhl (2021) Eq. 5:** y = x beta + (I - rho W)^-1 (x gamma + v), rho 0.5, gamma 0.25 (= beta/2). This is an SDM with beta + gamma and theta = -rho beta. + +All four tool estimators were fitted through `spatial_regression`. The true average effects follow LeSage & Pace (dy/dx' = (I - rho W)^-1 (beta I + theta W)). + +### Disturbance parameter (error model, SEM DGP) + +lambda 0.6 is recovered without sampling error: +- ML: coverage 0.872, bias -0.026; +- GMM: coverage 0.882, bias -0.022. + +With ACS error it is attenuated: +- ML: coverage 0.205, bias -0.263; +- GMM: coverage 0.170, bias -0.264. + +Remedy: A1. The error model's beta (= its direct and total effect) covers 0.88-0.90 either way. In DGPs without an SEM structure, lambda has no true value. Mean lambda-hat with ACS error was 0.09-0.27, always positive: it absorbs any unmodelled spatial structure. + +### Substantive effects (coverage of nominal 90% intervals; ACS outcome error / no sampling error) + +| DGP (true direct, indirect, total) | lag GMM [default]: dir / ind / tot | lag ML: dir / ind / tot | error ML [default]: beta covers direct / total | +| :--- | :--- | :--- | :--- | +| SEM lambda 0.6 (0.5, 0, 0.5) | 0.92/0.91/0.82 ; 0.92/0.83/0.74 | 0.89/0.26/0.48 ; 0.82/0.02/0.07 | 0.88/0.88 ; 0.90/0.90 | +| SAR rho 0.3 (0.513, 0.201, 0.714) | 0.92/0.87/0.89 ; 0.91/0.89/0.89 | 0.90/0.40/0.58 ; 0.91/0.87/0.86 | 0.89/0.01 ; 0.77/0.00 | +| SAR rho 0.6 (0.565, 0.685, 1.25) | 0.87/0.81/0.83 ; 0.90/0.93/0.93 | 0.90/0.00/0.02 ; 0.89/0.85/0.86 | 0.84/0.00 ; 0.01/0.00 | +| SDM rho 0.4, theta 0.3 (0.561, 0.772, 1.333) | 0.63/0.63/0.63 ; 0.83/0.85/0.85 | 0.88/0.00/0.00 ; 0.74/0.84/0.81 | 0.79/0.00 ; 0.01/0.00 | +| Omitted covariate (0.5, 0, 0.5) | 0.06/0.45/0.02 ; 0.00/0.17/0.00 | 0.04/0.30/0.01 ; 0.00/0.05/0.00 | 0.05/0.05 ; 0.02/0.02 | +| Juhl rho 0.5, gamma 0.25 (0.770, 0.230, 1.0) | 0.86/0.78/0.81 ; 0.88/0.75/0.78 | 0.86/0.85/0.87 ; 0.87/0.82/0.82 | 0.80/0.00 ; 0.62/0.00 | + +**Findings:** +1. **Where the lag model fits, the default GMM lag recovers the effects.** This covers the SEM (indirect effect about 0), SAR and Juhl DGPs, with and without outcome error, with the rho 0.6 / ACS caveat above. + - ML lag reports spurious indirect effects under an SEM (mean 0.16-0.25, coverage 0.02-0.26), because ML rho picks up the disturbance dependence. + - GMM lag does not (rho about 0). +2. **Under an SDM with theta > 0, GMM lag breaks down.** Its instruments (W X, W^2 X) are excluded regressors in the true model. Without sampling error, rho-hat averages 0.89 (coverage 0), and the indirect and total effects explode (mean bias +5.3 / +5.4 without error, +6.9 / +7.1 with). Their delta-method coverage (0.85) is meaningless. + - ML lag has rho bias +0.21 without sampling error and is moderately biased in the effects (indirect -0.09; coverage 0.84). + - **No production SDM estimator** was added (scope). The tool caveat now warns about W X effects. +3. **The error model's beta is the direct = total effect only when the DGP has no substantive spillover.** Under SAR/SDM/Juhl it misses the total effect entirely (coverage 0.00-0.02). Under the Juhl DGP it is biased by gamma (+0.25), as Juhl and Lacombe & LeSage state: an SEM with a correlated omitted variable is not unbiased. +4. **An omitted spatially correlated covariate that is correlated with x biases every model** (beta +0.13 to +0.15). The lag models turn the omission into a spurious indirect effect (0.15-0.24). No model in the tool fixes omitted-variable bias. +5. **Outcome sampling error mainly hurts ML** (lag rho, effects, error lambda); GMM lag is robust apart from the high-rho tails. +6. **OLS-based RS tests (spreg; Koley & Bera 2024)**, rejection rates at 0.05 (400 reps): + + | DGP, scenario | RS_lag | RS_WX | RS joint | RS*_WX (robust) | RS*_lag (robust) | + | :--- | ---: | ---: | ---: | ---: | ---: | + | SEM lambda 0.6, no error | 0.975 | 0.118 | 1.000 | 0.965 | 0.998 | + | SEM lambda 0.6, ACS error | 0.748 | 0.078 | 0.812 | 0.462 | 0.858 | + | SAR rho 0.6, no error | 1.000 | 0.992 | 1.000 | 0.025 | 1.000 | + | SDM, no error | 1.000 | 1.000 | 1.000 | 0.970 | 0.948 | + | Juhl, no error | 0.992 | 0.702 | 0.998 | 0.628 | 0.985 | + + - Under a pure SEM the robust W X test rejects 97% of the time (gamma = -beta lambda is not 0) and the robust lag test 99.8%. These tests point to an SDM or lag model when the truth is an SEM. + - They are not SEM-vs-SDM tests and are **not exposed** in the tool. The SEM-vs-SDM question needs the common-factor test (A4). + - Sampling error lowers their power. + +## A4. Common-factor LR test (SEM as SDM with theta = -rho beta) + +### Checks + +- **Nesting.** (I - lambda W) y = (I - lambda W) X beta + e gives y = lambda W y + X beta - lambda W X beta + e. That is the SDM with rho = lambda, theta = -lambda beta and constant (1 - lambda) alpha (row-standardized W). There is one restriction per covariate. +- **Likelihood comparability.** spreg's `ML_Lag(slx_lags=1)` (the SDM) and `ML_Error` share the same constants, the full log|I - rho W| Jacobian, sigma^2 = e'e/n and rho, lambda in (-1, 1). This was verified in code and numerically (tests). +- **The studies** use eigenvalue-based reimplementations equal to spreg to 5 decimals. + +### Size and power, Juhl's design (no sampling error) + +Design: queen W, beta 2, x, v ~ N(0, 1), 1000 repetitions per cell (MC SE <= 0.016). Residual bootstrap on the first 200 (MC SE <= 0.035; B = 100; Juhl sec. 3.3). + +Size (gamma = 0), chi-square(1) critical value, nominal 0.05: + +| rho | n = 49 | 100 | 225 | 400 | +| ---: | ---: | ---: | ---: | ---: | +| 0 | 0.066 | 0.054 | 0.052 | 0.060 | +| 0.2 | 0.067 | 0.059 | 0.053 | 0.044 | +| 0.4 | 0.077 | 0.049 | 0.064 | 0.049 | +| 0.6 | 0.078 | 0.073 | 0.070 | 0.060 | +| 0.8 | 0.090 | 0.066 | 0.041 | 0.052 | + +Size with Juhl's residual-bootstrap critical value (200 reps per cell): + +| rho | n = 49 | 100 | 225 | 400 | +| ---: | ---: | ---: | ---: | ---: | +| 0 | 0.045 | 0.050 | 0.065 | 0.070 | +| 0.2 | 0.080 | 0.080 | 0.035 | 0.045 | +| 0.4 | 0.065 | 0.050 | 0.095 | 0.040 | +| 0.6 | 0.065 | 0.075 | 0.055 | 0.070 | +| 0.8 | 0.080 | 0.080 | 0.040 | 0.050 | + +Power at gamma = 0.5 and 1.0 (chi-square(1)): + +| rho | gamma | n = 49 | 100 | 225 | 400 | +| ---: | ---: | ---: | ---: | ---: | ---: | +| 0.2 | 0.5 / 1.0 | 0.082 / 0.104 | 0.070 / 0.113 | 0.114 / 0.224 | 0.128 / 0.317 | +| 0.4 | 0.5 / 1.0 | 0.108 / 0.201 | 0.123 / 0.338 | 0.212 / 0.584 | 0.321 / 0.825 | +| 0.6 | 0.5 / 1.0 | 0.191 / 0.386 | 0.211 / 0.629 | 0.398 / 0.908 | 0.605 / 0.990 | +| 0.8 | 0.5 / 1.0 | 0.263 / 0.587 | 0.340 / 0.828 | 0.608 / 0.982 | 0.803 / 1.000 | + +At rho = 0 the restriction holds for any gamma (theta = 0 = -0 x beta), so the "power" there is size (0.044-0.075). + +**Reading:** +- **The LR test's size is close to nominal for n >= 100** (0.041-0.073). It is mildly liberal at n = 49 (0.066-0.090) and at rho 0.6 (0.060-0.078). +- **This confirms that the LR test does not have the Wald test's problem** (Juhl, Table 2: 0.0-11.5%), but it is not exact. +- **The residual bootstrap gives 0.035-0.095.** Within its larger MC error (0.015 at 0.05) it is not demonstrably better. It is not needed for n >= 100. +- **Power is low** for small rho or small n. For gamma / beta = 0.25 and rho <= 0.4 it is under 0.35 at every n. + +### With ACS outcome error (`common_factor_lr_acs`) + +Design: queen 12x12, the scale of the other studies; 500 reps for chi-square, the first 200 for the bootstraps. + +| rho | gamma | error | chi-square(1) | Juhl bootstrap | ME-aware bootstrap (adaptation) | +| ---: | ---: | :--- | ---: | ---: | ---: | +| 0.3 | 0 (H0) | none / ACS | 0.058 / 0.060 | 0.065 / 0.045 | - / 0.065 | +| 0.6 | 0 (H0) | none / ACS | 0.054 / 0.060 | 0.050 / 0.060 | - / 0.055 | +| 0.3 | 0.25 | none / ACS | 0.172 / 0.118 | 0.170 / 0.150 | - / 0.135 | +| 0.6 | 0.25 | none / ACS | 0.518 / 0.388 | 0.495 / 0.415 | - / 0.410 | + +Independent outcome error keeps the size (the SEM-plus-noise null is not an SEM, but the LR barely reacts) and costs power (0.52 to 0.39 at rho 0.6). + +### SEM estimates across gamma (Lacombe & LeSage 2015; Juhl design, no sampling error) + +Mean ML estimates, 1000 repetitions per cell (true beta = 2; the DGP's rho is the SEM's "lambda" when gamma = 0): + +| rho | gamma | SEM lambda-hat, n = 49 / 100 / 225 / 400 | SEM beta-hat, n = 400 | +| ---: | ---: | :--- | ---: | +| 0.4 | 0 | 0.281 / 0.336 / 0.369 / 0.384 | 1.999 | +| 0.4 | 0.5 | 0.298 / 0.363 / 0.384 / 0.395 | 2.487 | +| 0.4 | 1.0 | 0.316 / 0.379 / 0.408 / 0.413 | 2.976 | +| 0.6 | 0 | 0.481 / 0.545 / 0.572 / 0.583 | 1.999 | +| 0.6 | 0.5 | 0.507 / 0.562 / 0.588 / 0.599 | 2.476 | +| 0.6 | 1.0 | 0.536 / 0.608 / 0.631 / 0.644 | 2.943 | +| 0.8 | 1.0 | 0.765 / 0.816 / 0.837 / 0.841 | 2.910 | + +**Reading** (consistent with Lacombe & LeSage 2015 and Juhl 2021): +- **ML lambda has a small-sample downward bias** (-0.12 at n = 49, rho 0.6, gamma 0). +- **An omitted variable correlated with x pushes the SEM's lambda up.** At n >= 225 with gamma = 1, lambda-hat exceeds the true rho. +- **The SEM's beta absorbs gamma.** beta-hat is about beta + gamma (2.48 at gamma 0.5, 2.94 at gamma 1), slightly less at high rho. The SEM reports it as both the direct and the total effect, while the true total effect is beta + gamma / (1 - rho). The SEM therefore misses the indirect effect gamma rho / (1 - rho) entirely (see A3 for the Juhl DGP). +- **The SDM's rho-hat matches the SEM's lambda-hat under H0** (the two models coincide). +- **A diagnostic for users:** + - a gap between the SEM and OLS beta; + - significant lag-model indirect effects; + - a rejected common-factor LR. + + Each is an omitted-variable signal (Lacombe & LeSage). None is decisive alone, given the LR's power. + +Reproducibility: the `common_factor_lr` rows from the 5-process run and a separate single-process rerun are identical. + +### Conclusion (A4) + +**The LR common-factor test is valid** (size 0.04-0.09, near nominal for n >= 100) **but weak** in the sample sizes typical of county or tract analyses in one state. A significant result is informative; a non-significant one is weak evidence for the SEM. + +It is **not added to the tool** in this round, for two reasons: +1. It needs the SDM fit (spreg ML_Lag with slx_lags=1), which the plan excludes as a production estimator. +2. Its power is low. + +**Proposal:** if the owner wants an SEM-vs-SDM check, add `common_factor_test(formula, data, w)`. It would: +- fit both models by spreg ML on the same data; +- report the LR statistic with the chi-square(1) p-value per restriction (k - 1 df); +- add Juhl's residual bootstrap p-value for n < 100 (measured: chi-square size up to 0.09 at n = 49); +- carry a power caveat from the table above, and no SDM coefficients as results; +- not label the OLS RS tests or the rho = 0 test as specification tests. + +The Bayesian model-probability approach (Lacombe & LeSage 2015) needs an SDM estimator and marginal likelihoods: proposal only. + +## Files + +- `census/spatial.py`: `weights_eigenvalues`, `lag_multipliers`, `lag_effects`, effects in `spatial_regression`, `LAG_EFFECTS_NOTE`, `LAG_EFFECTS_CAVEAT`, `SPATIAL_PARAMETER_TEST_NOTE`, `diagnostics['coefficient_vcov']`, and the `interpretation` column. +- `census/results.py`: `ModelResult.effects`, ordered first in `to_dict()`, plus the summary lead. +- `census/report_tables.py`: `effects_table`, and the lag-model note in `regression_table`. +- `census/workflows_geo.py`: docstring and `next_step` (effects first). +- `census/simulation_spreg.py` (new): the studies, `me_sem_fit`, `sem_ml`, `sdm_ml`, `common_factor_lr`, `lr_residual_bootstrap`, `true_effects` and lattice helpers. +- `census/simulation.py`: registration (group `spreg_validation`). +- `scripts/inference_validation.py`: routing when named. +- `tests/test_spatial_regression_validation.py` (new) and `tests/test_workflows_geo.py`. diff --git a/docs/claude_setup.md b/docs/claude_setup.md index 8238016..a280a08 100644 --- a/docs/claude_setup.md +++ b/docs/claude_setup.md @@ -81,7 +81,7 @@ claude mcp add census -- python mcp_server.py Or with an absolute path: ```bash -claude mcp add census -- python c:\Users\hobo\Desktop\US-Census-5-Year-ACS-Data-Batch-Downloader\mcp_server.py +claude mcp add census -- python C:\path\to\Census-Research-Engine\mcp_server.py ``` ### Option B: Project Configuration (`.claude/config.json`) @@ -93,7 +93,7 @@ Create or edit `.claude/config.json` in the workspace root: "census": { "command": "python", "args": [ - "c:\\Users\\hobo\\Desktop\\US-Census-5-Year-ACS-Data-Batch-Downloader\\mcp_server.py" + "C:\\path\\to\\Census-Research-Engine\\mcp_server.py" ], "env": { "CENSUS_API_KEY": "${CENSUS_API_KEY}" @@ -112,7 +112,7 @@ To enable the tool across all projects in Claude Code, add the server under `"mc "census": { "command": "python", "args": [ - "c:\\Users\\hobo\\Desktop\\US-Census-5-Year-ACS-Data-Batch-Downloader\\mcp_server.py" + "C:\\path\\to\\Census-Research-Engine\\mcp_server.py" ] } } @@ -135,7 +135,7 @@ Add the `census` entry under `"mcpServers"`: "census": { "command": "python", "args": [ - "c:\\Users\\hobo\\Desktop\\US-Census-5-Year-ACS-Data-Batch-Downloader\\mcp_server.py" + "C:\\path\\to\\Census-Research-Engine\\mcp_server.py" ], "env": { "PYTHONIOENCODING": "utf-8" diff --git a/docs/literature/full_text_reading_notes.md b/docs/literature/full_text_reading_notes.md new file mode 100644 index 0000000..8b7de20 --- /dev/null +++ b/docs/literature/full_text_reading_notes.md @@ -0,0 +1,285 @@ +# Full-text reading notes: Round 16 spatial methods + +Read 2026-11-08 from the full texts retrieved through open-access routes: +- PubMed Central articles: the NCBI E-utilities XML, equations included; +- Juhl: the Cambridge Core CC-BY PDF; +- Arthur: the arXiv v1 PDF. + +Copyrighted text is not committed; only short quotations and section/equation references. + +**Status labels:** +- **VERIFIED:** stated in the full text I read. +- **ADAPTATION:** my inference for this engine, not a claim of the paper. It must be tested before use. +- **NOT READ:** supplements and web appendices. + +**Retrieved by the owner (2026-11-08):** these two were blocked to scripted requests (HTTP 403) and were not circumvented. The owner downloaded them in a browser into `literature_pdfs/`, which is gitignored; see sections 7 and 8. +- LeSage & Pace (2014): MDPI version of record. +- Koley & Bera: SSRN preprint, abstract 4252849. The journal version may differ. + +--- + +## 1. Wong (2021): cluster detection and estimate error + +*IJERPH* 18(18):9848, PMC8466737. Full text read. + +**VERIFIED:** +- **What local statistics detect:** local Moran's I and local G detect *similarity* among neighbours, not extreme values. Extreme units with dissimilar neighbours can be missed, and moderate-value units can form clusters (sec. 3.1; simulated hexagon examples). +- **Estimate error:** local statistics assume accurate or uniformly reliable estimates (sec. 3.2). Within one detected cluster, estimates can differ significantly, while units inside and outside it may not (Table 2, obesity data). +- **Class separability classification** (sec. 4.1): class breaks chosen to maximize the confidence that estimates on either side differ. The top and bottom classes are read as hot and cold spots, which need not be contiguous. +- **Heuristic hot/cold-spot procedure** (sec. 4.2): + 1. The analyst sets low and high thresholds from domain knowledge (a "global criterion"). + 2. Units whose *estimates* are beyond a threshold are seeds. + 3. First-order neighbours are compared with the **seed** estimate by "a t-test given a confidence level". A neighbour **not significantly different** joins the cluster. + 4. Growth continues to higher-order neighbours, always comparing against the seed. It stops when all frontier neighbours differ. + 5. A seed with no such neighbour is a single-unit spot. +- **Multiple testing:** Bonferroni is "not applicable" because membership rests on *failing to reject*. A binomial probability, Eq. (1), "assumes the independence of events". +- **Error size drives membership:** larger errors make estimates "more likely to be statistically indifferent and thus produced larger HSCS" (sec. 5.2). +- **Simulations:** + - errors random, proportional or inversely proportional to the estimates, "associated with the estimates, not with locations"; + - no accuracy metrics reported; + - the empirical example is state BRFSS obesity, not ACS. +- **Not specified:** the t-test formula (how SEs combine, degrees of freedom, sidedness); tie-breaking when clusters from different seeds meet. + +**ADAPTATION / implications:** +- **Different estimand from Local Moran's I.** It flags units *beyond a threshold* plus neighbours not shown to differ. Under "no spatial clustering" it still reports spots wherever extreme values occur, so false-positive comparisons with Local Moran's I compare unlike targets. Evaluate it against its **own** target: recovery of units whose *true* value is beyond the threshold, and the inclusion of neighbours whose true value is near the seed's. +- **Absence of evidence becomes membership.** Larger SEs mean larger clusters. This is a measurable property, and expected to be worse for high-CV tracts. +- **Correlated errors break the test.** Positively correlated errors between a seed and its neighbour make an independence-based difference test under-reject, so clusters grow. Negatively correlated errors do the opposite. This is a hypothesis to measure in B scenarios 3 and 5. + +## 2. Koo, Wong & Chun (2019): global spatial autocorrelation with reliability information + +*The Professional Geographer* 71(3):551-565, PMC6884366. Full text read; equations recovered from MathML. + +**VERIFIED:** +- **What "biased upward" means.** The claim that conventional statistics ignoring error are biased upward comes from comparing published estimates with **new draws from N(estimate, SE)**, i.e. a second layer of noise on top of the sampling error already in the estimates. In their words, estimates "with larger errors are less positively autocorrelated". This is **consistent with**, not contrary to, attenuation of the *true* process's autocorrelation by sampling error. +- **The SBC** (Eqs. 2-4): the Bhattacharyya coefficient between neighbouring N(μᵢ, σᵢ²) and N(μⱼ, σⱼ²), averaged over binary neighbour weights. Eq. (3) as printed has a sign slip; the coefficient is exp(-BD). + - Significance is by permutation, shuffling estimates and SEs separately. + - On the 50×50 lattice, the SBC is significant "even with a large CV ... although the estimates do not show spatial autocorrelation (ρ = 0)". The authors say errors have "an overwhelming influence". + - It "is not compatible" with conventional statistics for the direction of autocorrelation. + +**ADAPTATION / implications:** +- **Not a correction.** The SBC measures overlap of *error distributions*, not autocorrelation of the latent process. It can be significant with no spatial structure, so it is neither a correction nor a test of true spatial autocorrelation for our purposes. At most it is a descriptive reliability index. **Do not implement as a correction.** + +## 3. Donegan, Chun & Griffith (2021): Bayesian inference with survey SEs and spatial structure + +*IJERPH* 18(13):6856, PMC8297362. Full text read (main text; online supplement not read). + +**VERIFIED:** +- **Classical measurement error** (Eq. 1): observed = true + error. With spatial data, "observational error tends to decrease the degree of observed SA", and spatial variation in data quality produces spatially varying consequences (sec. 2.3). +- **Empirical illustration:** Milwaukee tract insurance coverage has SAR ρ of 0.82 raw against 0.89 (0.86-0.92) after modelling (sec. 3.3). +- **Data model:** Gauss(true value, SE), i.e. independent errors with the published SEs. Process model: CAR, (I − ρW)⁻¹M (Eq. 5). The model is applied to a **covariate** (errors-in-variables); the outcome is a Poisson count (Eq. 8). +- **ACS facts used:** + - SEs are spatially autocorrelated (SAR ρ 0.46-0.69 for the SEs themselves; Table 1); + - MOEs come from Successive Differences Replication; + - variance replicate tables let users build SEs for composite variables (their ICE). +- **Limitations stated:** + - non-sampling errors are not modelled; + - "errors themselves are correlated due to bias in the sampling design or survey implementation" is possible, and would need additional information. + - The naive model attenuated the covariate effect relative to the measurement-error model (Fig. 12). + +**ADAPTATION / implications:** +- **The attenuation reading of F1 is supported.** Sampling error added to the *outcome* of a spatial error model inflates the independent part of the disturbance, so the estimated λ is attenuated. +- **The corresponding likelihood**, with independent errors and known sampling variances, has covariance σ²[(I − λW)ᵀ(I − λW)]⁻¹ + diag(SE²). That is the **SAR random-effects (spatial Fay-Herriot) model** the engine already implements in `census.sae_spatial`, where it is a gated, passing release study. +- **Hypothesis for A1:** the spatial Fay-Herriot ρ estimate (and β) is a validated correction for λ attenuation that needs only the sampling variances and assumes independent errors. It must be tested: + - coverage of ρ, not only of the EBLUP; + - behaviour under correlated sampling errors; + - weights sensitivity. +- **Diagonal error covariance:** every method in these papers assumes it. A correction built from MOE variances is independence-based by construction. + +## 4. Sun, Reich, Cai, Guindani & Schwartzman (2015): FDR control in spatial multiple testing + +*JRSS-B* 77(1):59-83, PMC4310249 (author manuscript). Main text read; web appendix with the Theorems 2 and 4 proofs not read. + +**VERIFIED:** +- **Model:** X(s) = μ(s) + ε(s), a latent process plus noise (Eq. 1.1). Hypotheses are about the **latent signal relative to an indifference region**: H₀: μ(s) ∈ A, for example μ ≤ μ₀ (Eq. 2.1). They are not about local autocorrelation. +- **Oracle statistic:** T_OR(s) = P(θ(s) = 0 | data) (Eq. 3.4). Procedure 1 rejects the largest set whose running average of T_OR is ≤ α (Eq. 4.1). For a discrete domain the FDR control is exact (Corollary 1), **given the true model parameters**. +- **Data-driven version:** uses MCMC posterior samples. "The precision of our testing procedure shows some sensitivity to model misspecification" (sec. 6). +- **The FCR** (Eq. 2.3) is a weighted rate over **predefined** clusters with partial-conjunction nulls. This is **not** the same as a "maps with ≥ 1 false contiguous cluster" rate. +- **Simulations:** + - the error covariance (1 − r)I + r·exp(-d/ρε), with r = 0.9, i.e. mostly spatially *correlated* errors; + - BH and related methods were "very conservative". +- **Ozone application:** β̂(s) = β(s) + w(s)ε(s), with known per-site SEs w(s). This is the same structure as ACS estimates with published SEs. + +**ADAPTATION / implications:** +- **Three questions to keep apart in B and in the tool:** + 1. "Is this area's latent value elevated?" (Sun et al.'s and Wong's question); + 2. "Is this area locally similar to its neighbours beyond chance?" (Local Moran's I); + 3. "Is the classification stable across replicates?" +- **A principled route for question 1:** posterior exceedance probabilities from a spatial model with known sampling variances, thresholded with Procedure 1. + - Spatial Fay-Herriot gives a Gaussian approximation (EBLUP, MSE). This would be an adaptation **without** full parameter uncertainty, so it needs its own validation. + - Out of scope for production this round; candidate for a later round. +- **Name collision:** our "false-cluster rate" must be renamed or defined explicitly to avoid confusion with their FCR. + +## 5. Juhl (2021): the Wald test of common factors + +*Political Analysis* 29(2):193-211, doi 10.1017/pan.2020.23, CC BY. Main text read; Supplementary Materials A-C not read. + +**VERIFIED:** +- **Models:** the SEM y = xβ + (I − ρW)⁻¹u is the SDM y = ρWy + xβ + Wxθ + u under θ = −ρβ (Eqs. 1, 3, 4). +- **Omitted variables:** an omitted variable that is spatially clustered **and correlated with x** gives an SDM-like DGP (Eq. 5). The SDM then has E(ρ̂) = ρ and E(θ̂) = −ρβ, but β̂ is biased by γ. If the omitted variable is uncorrelated with x, the result is an SEM and "even a nonspatial OLS model specification would provide unbiased but inefficient parameter estimates". +- **Wald non-invariance:** asymptotic Wald tests of the nonlinear restriction are not invariant to algebraically equivalent forms (Table 1: H0(I)-(IV)). + - Type I error ranged from 0.0% to 11.5% at a nominal 5% (Table 2; n = 49 to 400, queen contiguity, β = 2, ρ from 0 to 0.8, 1000 samples). + - H0(IV) has a non-monotone power function. +- **Bootstrap fix** (sec. 3.3): + 1. fit the SEM; + 2. resample its residuals to build 100 bootstrap samples under the SEM DGP; + 3. recompute w for each; + 4. use the (1 − α) quantile as the critical value. + + This brings the size to about 4-7% for all forms (Table 3). +- **LR test:** invariant to reparameterization. Its good finite-sample performance is asserted via **Supplementary Material C.7 (NOT READ)**. +- **What software reports:** the "Wald tests" reported by spatial software test ρ = 0, not common factors (sec. 3.2). +- **Interpretation:** inference should rest on direct and indirect **impacts**, not coefficients (footnote 23). The SDM is not a general solution to omitted variables: identification is weak and depends on perfect knowledge of W (citing Gibbons & Overman 2012). + +**ADAPTATION / implications for A3/A4:** +- **Use Juhl's design as the template** for the omitted-variable DGP (γ grid) and the size/power study. +- **Measure the LR common-factor test's size directly.** The LR claim rests on an unread supplement. If the size is off, implement Juhl's bootstrap (it is fully specified in the main text). +- **The tool must not label its ρ = 0 test a specification test.** + +## 6. Arthur (2024/2025): resampling autocorrelated spatial data + +*Geographical Analysis* 57(2):302-319, doi 10.1111/gean.12417 (journal version not retrievable). arXiv:2401.05728v1 read in full. + +**VERIFIED:** +- **Method:** null distributions for **bivariate association statistics** (Pearson, Spearman, Kendall correlation between two maps) are built by permuting values and then greedily optimizing so each resample matches the observed global Moran's I. +- **Results:** comparable to sampling from the true DGP in the authors' power-law-spectrum lattice tests. +- **Caveats:** + - Moran's I saturates; the method over-rejects at high autocorrelation and is "not recommend[ed]" above I ≈ 0.7; + - it is for statistics "which sum over all the data points"; + - no treatment of local statistics or of sampling error; + - Python package `spatialsignificance`. + +**ADAPTATION / implications:** +- **Not applicable to the Local Moran's I problem.** It could inform a later null model for *bivariate* map comparisons (for example our `correlation` tools), only with validation and only for I ≤ 0.7. + +## 7. LeSage & Pace (2014): "The Biggest Myth in Spatial Econometrics" + +*Econometrics* 2(4):217-249, doi 10.3390/econometrics2040217. Full text read (numeric table rows skimmed). + +**VERIFIED:** +- **Effects:** for the SDM y = αι + ρWy + Xβ + WXθ + ε, the partial derivatives are ∂y/∂xᵣ′ = (I − ρW)⁻¹(Iβᵣ + Wθᵣ) (Eq. 32). The SAR case is θ = 0 (Eqs. 18-19). The diagonal holds the direct (own-region) effects; the off-diagonal elements hold the indirect (spillover) effects (sec. 3.3). +- **Interpretation:** + - β is not the marginal effect when the model contains Wy. + - When W changes, ρ and β shift in offsetting ways ("required to maintain relatively stable partial derivatives"; Fig. 1). The effects stay stable (Tables 5-8). + - So sensitivity checks must compare **effects, not coefficients** (sec. 3.1, 3.3). +- **Robustness:** observation-level (n²k) effects are **not** robust to W. Scalar summaries averaged over observations are (sec. 4, 5). +- **Effect draws:** medians of simulated effect draws are suggested, since total effects can be asymmetric (footnote 12). Draws from the ML estimates and their covariance are "equivalent" to their MCMC draws. +- **Misspecification signal:** sensitivity to W is often a symptom of model misspecification. For a correctly specified SEM, OLS β is unbiased, so large OLS-versus-SEM differences in β point to misspecification. The Bell & Bockstael re-analysis is the example: 5 of 10 coefficients differ by more than 1.67 SD. A Hausman test is cited to Pace & LeSage (2008; **NOT READ**). +- **Weights similarity:** corr(Wₐu, W_b u) = (mₐ/m_b)^½ for nearest-neighbour matrices (Eq. 7), and the correlation of effects across W is given by Eq. (27). +- **Model choice:** weight matrices can be compared by Bayesian posterior model probabilities (Eq. 34). Flexibility in modelling the spillovers matters more than flexibility in the disturbances (sec. 4). + +**ADAPTATION / implications:** +- **Effects in the tool:** the lag-model output reports average direct, indirect and total effects with simulated intervals, and labels β as not the marginal effect. +- **W sensitivity:** A's weights-sensitivity results compare effects across W, not ρ/β. +- **OLS-versus-SEM β gap:** a candidate diagnostic for the SEM route. It is cheap. The formal Hausman version is unread, so report the gap descriptively unless validated. + +## 8. Koley & Bera: "To use, or not to use the spatial Durbin model?" + +SSRN 4252849 preprint; the journal version is *Spatial Economic Analysis* 19(1):30-56 (2024). Preprint read in full; proofs in its appendix skimmed. + +**VERIFIED:** +- **The hypothesis tested** is the **linear** H0D: γ = 0, the Durbin (WX) term, in y = λWy + αι + Xβ + WXγ + ε. Here λ is the **lag** coefficient. This is **not** the common-factor hypothesis H0S: γ + βλ = 0 (sec. 2): + - accepting H0S points to the SEM; + - accepting H0D points to the SAR. +- **Tests:** the joint RS_λγ, plus Bera-Yoon robust versions RS*_λ and RS*_γ. All need **only OLS** (sec. 3-4). + - Robust to **local** misspecification only, with Monte Carlo suggesting some non-local robustness. + - Additivity identity (Eq. 3.9/4.16): RS_λγ = RS_λ + RS*_γ = RS*_λ + RS_γ. They recommend using it to check computations. +- **Assumptions:** normal errors (asymptotic validity under non-normality is asserted via Fang et al. 2014, not shown); increasing-domain asymptotics; bounded W row sums; nonstochastic X. +- **Recommended sequence:** run RS_λγ first; if it rejects, run RS*_λ and RS*_γ. +- **Monte Carlo:** + - Design: Columbus 49-district W (and W⊗I₂, n = 98); α = 1, β = 1.5; ε ~ N(0, 2); x ~ N(0, 4); λ ∈ [0, 0.6]; γ ∈ [0, 0.35]; 5000 reps. + - Results: at n = 49 the unadjusted tests over-reject, while the adjusted and joint tests under-reject slightly. RS*_γ has flat size under λ ≠ 0, at a cost in power. + - **No SEM DGP is simulated.** + +**ADAPTATION / implications:** +- **They do not identify the SEM.** Under an SEM DGP the SDM representation has γ = −βλ ≠ 0, so RS*_γ is expected to reject and point toward an SDM. That is my derivation, not shown in the paper. The SEM-versus-SDM decision therefore still needs the common-factor test (Juhl's bootstrap or the LR test). +- **They are already in spreg 1.9.1** (`lm_wx`, `lm_spdurbin`, `rlm_wx`, `rlm_durlag`). If the tool adds them, use spreg's implementation and verify it with the paper's additivity identity, not a reimplementation. + +## 9. Lorincz-Comi, Ajayakumar, J. Curtis, Zhang, A. Curtis & Lovell (2021): "Addressing uncertainty in census estimates" + +*Spatial Statistics* 45:100523. Owner-supplied PDF, main text read in full. The supplement (estimator variances, multivariable extension, simulation generators) was **NOT READ**. + +**VERIFIED:** +- **Scope:** this is a method-of-moments measurement-error correction for **geographically weighted regression**, with location-weighted moments. + - Classical model: x = X + u, y = Y + q, with errors normal around the truth. + - The naive slope is attenuated by κ = σ²X/(σ²X + σ²u) when σXu = σuq = 0 (Eq. 2). The correction solves for β₁. + - It allows "nonclassical" σXu ≠ 0 (error correlated with the true value), in which case the bias can go either way (Fig. 1), and correlated x/y errors σuq. +- **Inputs:** σu² is the distance-weighted mean of (MOE/1.645)². +- **Corrected correlation** (Eq. 3) and a **corrected Moran's I** (Eq. 4), which subtracts the error covariance between locations σuq and the error variance σu² from the cross- and own-moments. +- **Inference:** variances of the corrected estimators are only in the supplement (a proposed jackknife). The authors state that existing variance estimators "cannot be applied". +- **What it does not do:** it addresses neither spatial error / lag models nor λ. + +**ADAPTATION / implications:** +- **A1:** confirms that this paper is **not** a λ correction. Its global (non-GWR) analogue is the engine's existing `regression_calibration`. +- **Eq. 4 corrected Moran's I:** the between-area error covariance term is exactly what workstream C estimates from replicates. This makes a corrected global Moran's I a candidate for a later round. Do not implement before: its variance/inference is verified from the supplement (unread), and a simulation validates it under independent and correlated errors. + +## 10. Jung, Thill & Issel (2019): spatial autocorrelation and ACS data uncertainty, a critique + +*IJGIS* 33(6):1155-1175, doi 10.1080/13658816.2018.1554811. Owner-supplied PDF, read in full. + +**VERIFIED:** +- **Global Moran's I is attenuated by reliability:** with independent errors, uncorrelated with the truth, observed I ≈ [Var(θ)/Var(x)] × true I (Eq. 4). This is regression dilution. +- **Ground truth:** Mecklenburg County tract teen birth rates against 2010 census plus birth-certificate counts. Global I was 0.46 on the census-based rates, against 0.128 and 0.025 on the ACS-based rates, matching their reliability ratios of 15.23% and 0.43%. +- **Local Moran's I:** mostly **deflated** in true hot and cold spots, so clusters were missed. Two spurious hot spots appeared. Tests used 10,000 permutations with queen contiguity. +- **Remedies:** empirical Bayes and small-area estimation (the Fay-Herriot family). + +**My reading (not established):** +- **Eq. 9:** the extra local term diag(WΩ)/Var(ε) is zero when Ω (the error covariance) is diagonal, because W has a zero diagonal. So under the paper's own independence assumption, heteroscedastic errors change the *variance*, not the *expected value*, of local I. The term is non-zero only when **neighbouring errors are correlated**: diag(WΩ)ᵢ = Σⱼ wᵢⱼ Cov(εᵢ, εⱼ). +- This matches workstream B: independent errors mainly cost power, while correlated errors add false structure. It also ties directly to C's covariance estimates. + +## 11. Rogerson (2025): "Testing Hypotheses When You Have More Than a Few" + +*Geographical Analysis* 57(2):175-190, doi 10.1111/gean.12412. Owner-supplied PDF, read (closing pages skimmed). + +**VERIFIED:** +- **Family-wise error rate (FWER) methods:** Bonferroni, Šidák, Holm (step-down), Hochberg (step-up, which needs independence or positive dependence). +- **FDR control:** Benjamini-Hochberg (BH) controls the FDR under independence or positive dependence. It notes that nearby local statistics "will almost always be positively correlated". Benjamini-Yekutieli covers general dependence. +- **Ignored autocorrelation inflates FDR:** if spatial autocorrelation is ignored in the individual tests, the realized FDR exceeds the nominal level (an algebraic argument). +- **Estimating the number of true nulls (m₀):** Schweder-Spjøtvoll, Nettleton et al., Storey's λ estimator (λ = 0.5 "usually serves well"). **Adaptive two-stage BH** (Benjamini et al. 2006) uses m₀ for more power. +- **Cluster-level testing** (Heller et al. 2006; Pacifico et al. 2004; Chumbley & Friston 2009) "should often replace" area-level perspectives. + - Example: one 95-unit true cluster plus 5 false units gives an area-level FDR of 5% but a cluster-level FDR of 83%. + - q = 0.2-0.3 "might conceivably be tolerable" for clusters. +- **Illustration:** Gi* on a 55×55 grid. FDR adds rejections over Bonferroni, and adaptive BH adds one more. + +**ADAPTATION / implications:** +- **B:** the current Local Moran's I default uses BH, which is valid under positive dependence. Adaptive (Storey) BH is a candidate for more power and is cheap. +- **Cluster-level reporting:** our "false-cluster map rate" is a step toward it. Report flagged results as clusters with a cluster-level error rate. Proposal only, to be validated with B's harness. + +## 12. Lacombe & LeSage (2015): Bayesian posterior model probabilities and omitted variables + +*Papers in Regional Science* 94(2):365-384, doi 10.1111/pirs.12070. Owner-supplied PDF, read (Monte Carlo tables and appendix skimmed). + +**VERIFIED:** +- **Omitted variables decide the model:** + - an omitted variable uncorrelated with x gives an SEM, with no spillovers; OLS is consistent for β, but its inference is not valid; + - spatially dependent disturbances plus an omitted variable correlated with x give an SDM, y = ρWy + x(β + γ) + Wx(−ρβ) + ε (Eqs. 10-13). +- **γ is identified:** β = θ/(−ρ), then γ = ϕ − β. A significant γ is a test for omitted-variable bias on that variable. They also suggest the t-statistics of the **indirect effects** as a simpler analogous test. +- **SEM disturbances are "diffusion of shocks", not spillovers.** +- **Bayesian model comparison** (LeSage & Parent 2007) gets SEM against SDM right at correlation ≈ 0 and ≈ 0.7 between x and the omitted variable. In between it depends on n: at correlation ≈ 0.2 it points to the SEM at n = 506 and to the SDM at n = 3,107 and 10,000. +- **Biases:** the SEM overestimates ρ when the omitted variable is correlated with x. ML ρ is biased downward in small samples (citing Bao & Ullah 2007). +- **Census data:** variables at county, tract and block-group level show spatial dependence of 0.75-0.95 (citing Pace et al. 2012). + +**ADAPTATION / implications:** +- **A3:** the omitted-correlated-covariate DGP is the central stress test. The Juhl grid already uses this DGP, with γ the same parameter. +- **A4:** evaluate the common-factor/γ route next to LR and the bootstrap. Bayesian model probabilities need a production SDM estimator and a marginal-likelihood implementation, neither of which the engine has, so they stay a proposal. +- **Interpretation:** the lag tool's notes should say that SEM disturbances are diffusion of shocks, not spillovers. This matches the corrected λ wording. + +--- + +## Consequences for Round 16 + +- **A1:** don't implement Lorincz-Comi et al. (closed access, assumptions unverifiable). First test whether the **existing spatial Fay-Herriot model** recovers λ/ρ with correct coverage under outcome sampling error. That is the independence-based correction the literature's data model implies. Then test it under correlated errors. +- **A2:** impacts, not coefficients (Juhl fn. 23; LeSage & Pace 2014, Eq. 32). Report averaged direct/indirect/total effects with simulated intervals. Check W sensitivity on effects, not on ρ/β. +- **A3/A4:** Koley & Bera's OLS-only robust RS tests (in spreg) answer *SAR vs SDM*, not *SEM vs SDM*; use them only alongside a common-factor test. Juhl's DGP grid. Measure LR size, and use the bootstrap critical value if needed. A ρ = 0 test is not a specification test. +- **B:** keep three questions distinct: + 1. latent elevation (Wong, Sun et al.); + 2. local similarity (Local Moran's I); + 3. replicate stability. + + Evaluate Wong's method against its own target, and record its larger-SE-larger-cluster property and its sensitivity to correlated errors. Rename our false-cluster metric, or define it explicitly against Sun et al.'s FCR. Arthur is out of scope for Local Moran's I. +- **Between-area error covariance** (added after sections 9-12) is the common thread. It is: + - the σuq term in the corrected Moran's I (Lorincz-Comi Eq. 4); + - the only source of a non-zero expectation shift in local Moran's I (Jung et al. Eq. 9, as I read it); + - what makes Fay-Herriot smoothing create false clusters (workstream B). + + C's estimates therefore decide which of these corrections is usable. +- **C:** none of these papers estimates between-area sampling covariance; all assume diagonal error covariance. Measuring it from VRE replicates fills a gap the literature leaves open (Donegan et al. name the possibility). diff --git a/docs/local_models.md b/docs/local_models.md index fac8c55..1736bf7 100644 --- a/docs/local_models.md +++ b/docs/local_models.md @@ -46,7 +46,7 @@ Any MCP client that can use a local model works. The server entry is the same ev "mcpServers": { "census": { "command": "python", - "args": ["/path/to/US-Census-5-Year-ACS-Data-Batch-Downloader/mcp_server.py", "--profile", "core"], + "args": ["/path/to/Census-Research-Engine/mcp_server.py", "--profile", "core"], "env": {"CENSUS_API_KEY": "your-key"} } } diff --git a/mcp_server.py b/mcp_server.py index 617eab3..8d025a8 100644 --- a/mcp_server.py +++ b/mcp_server.py @@ -1,5 +1,5 @@ # SPDX-License-Identifier: AGPL-3.0-or-later -"""US Census ACS Batch Downloader: MCP server. +"""Census Research Engine: MCP server. Exposes discovery, extraction, cross-vintage concordance review, and research statistics tools over the Model Context Protocol. The server instructions and diff --git a/pyproject.toml b/pyproject.toml index ec5d828..551b89b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -47,8 +47,8 @@ census-acs = "census.__main__:main" census-acs-mcp = "mcp_server:main" [project.urls] -Repository = "https://github.com/cafeTechne/US-Census-5-Year-ACS-Data-Batch-Downloader" -Changelog = "https://github.com/cafeTechne/US-Census-5-Year-ACS-Data-Batch-Downloader/blob/main/CHANGELOG.md" +Repository = "https://github.com/cafeTechne/Census-Research-Engine" +Changelog = "https://github.com/cafeTechne/Census-Research-Engine/blob/defaultBranch/CHANGELOG.md" [tool.setuptools] py-modules = ["mcp_server"] diff --git a/scripts/acs_error_covariance.py b/scripts/acs_error_covariance.py new file mode 100644 index 0000000..1f3815b --- /dev/null +++ b/scripts/acs_error_covariance.py @@ -0,0 +1,884 @@ +# SPDX-License-Identifier: AGPL-3.0-or-later +"""ACS sampling-error covariance between areas, from the VRE replicates (PLAN_ROUND_16 C1-C2). + +Measures how the sampling errors of ACS 5-year estimates covary between +areas, and how reliably 80 replicates can measure that, for Michigan (26) and +New York (36) tracts and counties. Writes + + docs/audits/acs_sampling_covariance.md methods, tables, conclusion + docs/audits/acs_sampling_covariance.csv one row per (release, state, + geography, variable, pair class) + +Usage: + python scripts/acs_error_covariance.py # 2024 and 2019 (~10 min) + python scripts/acs_error_covariance.py --years 2024 # one release + python scripts/acs_error_covariance.py --no-holdout # skip the dense model comparison + +Data: Variance Replicate Estimate files (census.replicate.fetch_vre; cached +under $CENSUS_VRE_DIR, default .cache/vre) and cartographic boundary files +(census.boundaries; $CENSUS_GEO_DIR, default .cache/geo). Both are public +www2.census.gov downloads; no API key is used. Large files are never written +to the repository; the report lists their URLs and SHA-256. + +Variables (types): + count B01003_001 total population (tracts; county totals are + controlled to population estimates and have no sampling error), + B25003_001 occupied housing units, B17001_002 persons below poverty + proportion B17001_002 / B17001_001 poverty rate, B25003_002 / B25003_001 owner share + (evaluated on each replicate; an area with a zero denominator in any replicate is + excluded for that variable: the replicate ratio is undefined) + median household income, linearly interpolated from the B19001 replicate + distributions (census.medians); no published median table has VREs + +The study is info (PLAN_ROUND_16 section 5): it gates nothing. +""" + +import argparse +import math +import os +import platform +import subprocess +import sys +import time +import warnings +from datetime import datetime, timezone + +import numpy as np +import pandas as pd + +HERE = os.path.dirname(os.path.abspath(__file__)) +ROOT = os.path.dirname(HERE) +sys.path.insert(0, ROOT) + +import census # noqa: E402 +from census import replicate_covariance as rc # noqa: E402 +from census.boundaries import boundary_vintage, equal_area_crs, get_boundaries # noqa: E402 +from census.medians import _quantile_matrix, distribution_brackets # noqa: E402 +from census.replicate import VREDownloadError, fetch_vre, sumlev_for, vre_url # noqa: E402 +from census.spatial import spatial_weights # noqa: E402 +from census.variance import REPLICATE_COLUMNS, REPLICATE_COUNT # noqa: E402 + +OUT_MD = os.path.join(ROOT, "docs", "audits", "acs_sampling_covariance.md") +OUT_CSV = os.path.join(ROOT, "docs", "audits", "acs_sampling_covariance.csv") + +STATES = {"26": "Michigan", "36": "New York"} +SEED = 20261108 +N_BOOT = 200 +MAX_ORDER = 3 +WITHIN_COUNTY_CAP = 150_000 # far within-county tract pairs kept (random sample beyond this) +FAR_SAMPLE = 100_000 # random tract pairs drawn for the far, across-county class +CROSS_STATE_SAMPLE = 50_000 # MI x NY tract pairs (independent samples: empirical null) +TRACT_DIST_BINS = [0, 2, 5, 10, 20, 50, 100, np.inf] +COUNTY_DIST_BINS = [0, 50, 100, 200, 400, np.inf] + +# name -> (type, kind, spec); kind 'cell', 'ratio' (num, den) or 'median' (table) +VARIABLES = { + "population": ("count", "cell", "B01003_001"), + "households": ("count", "cell", "B25003_001"), + "poverty_count": ("count", "cell", "B17001_002"), + "poverty_rate": ("proportion", "ratio", ("B17001_002", "B17001_001")), + "owner_share": ("proportion", "ratio", ("B25003_002", "B25003_001")), + "median_hh_income": ("median", "median", "B19001"), +} +TABLES = ("B01003", "B25003", "B17001", "B19001") + + +def log(msg): + print(f"[{time.strftime('%H:%M:%S')}] {msg}", flush=True) + + +# --------------------------------------------------------------------------- +# Data +# --------------------------------------------------------------------------- + + +def _unavailable_reason(exc): + """Short, path-free reason for the report (the exception text names the local cache path).""" + if getattr(exc, "rejected", False): + sid = getattr(exc, "support_id", "") or "none shown" + return f"census.gov firewall 'Request Rejected' page (support ID {sid}), also on the single delayed retry" + return str(exc).split("\n")[0] + + +def _load_tables(year, level, state, files): + frames = {} + for t in TABLES: + try: + f = fetch_vre(t, year, level, state, check_moe=False) + except (VREDownloadError, ValueError) as exc: # skipped and reported; census.http never loops + log(f"{year} {level} {state} {t}: not available ({exc})") + frames.setdefault("_missing", []).append(t) + sumlev = sumlev_for(level) + files.append({"kind": "VRE", "year": year, "level": level, "state": state, "table": t, + "url": getattr(exc, "url", None) or vre_url(t, year, level, state), "sha256": None, + "unavailable": _unavailable_reason(exc), + "manual_path": f"$CENSUS_VRE_DIR/{year}/{sumlev}/" + + os.path.basename(vre_url(t, year, level, state))}) + continue + files.append({"kind": "VRE", "year": year, "level": level, "state": state, "table": t, + "url": f.attrs["source_url"], "sha256": f.attrs["sha256"], + "acquisition": f.attrs.get("acquisition", "download")}) + frames[t] = f + return frames + + +def _cell(frames, var, geoids): + t = var.split("_")[0] + f = frames[t] + f = f[f["variable"] == var].set_index("GEOID").reindex(geoids) + return f["estimate"].to_numpy(float), f[REPLICATE_COLUMNS].to_numpy(float), f["controlled"].fillna(False).to_numpy(bool) + + +def variable_values(frames, name, geoids, year): + """(estimate (n,), replicates (n, 80), valid mask, note) for one study variable.""" + vtype, kind, spec = VARIABLES[name] + if kind == "cell": + est, reps, ctrl = _cell(frames, spec, geoids) + valid = np.isfinite(est) & np.isfinite(reps).all(axis=1) & ~ctrl + note = f"{int(ctrl.sum())} controlled" if ctrl.any() else "" + elif kind == "ratio": + num, num_r, _ = _cell(frames, spec[0], geoids) + den, den_r, _ = _cell(frames, spec[1], geoids) + with np.errstate(divide="ignore", invalid="ignore"): + est = np.where(den > 0, num / den, np.nan) + # a zero denominator in any replicate leaves the ratio undefined there: the area is invalid for + # this variable (setting the replicate ratio to 0 would fabricate a huge deviation) + reps = np.where(den_r > 0, num_r / den_r, np.nan) + zero_den = np.isfinite(est) & (np.isfinite(den_r) & (den_r <= 0)).any(axis=1) + valid = np.isfinite(est) & np.isfinite(reps).all(axis=1) + note = (f"{int(zero_den.sum())} area(s) with a zero replicate denominator excluded" if zero_den.any() + else "") + else: + br = distribution_brackets(spec, year) + lo = np.array([b[1] for b in br]) + hi = np.array([b[2] for b in br]) + cells = [_cell(frames, b[0], geoids) for b in br] + counts = np.column_stack([c[0] for c in cells]) # (n, k) + rep_counts = np.stack([c[1] for c in cells], axis=2) # (n, 80, k) + n = counts.shape[0] + full = _quantile_matrix(lo, hi, counts, 0.5) + rq = _quantile_matrix(lo, hi, rep_counts.reshape(n * REPLICATE_COUNT, -1), 0.5) + est = full["value"] + reps = rq["value"].reshape(n, REPLICATE_COUNT) + coded = full["top"] | full["bottom"] + valid = full["valid"] & ~coded & rq["valid"].reshape(n, -1).all(axis=1) & np.isfinite(counts).all(axis=1) + note = f"{int((full['valid'] & coded).sum())} top/bottom-coded medians excluded" + reps = np.where(valid[:, None], reps, np.nan) + est = np.where(valid, est, np.nan) + with np.errstate(invalid="ignore"): + zero = valid & (np.nansum((reps - est[:, None]) ** 2, axis=1) == 0) + valid &= ~zero + return est, reps, valid, {"type": vtype, "n_zero_variance": int(zero.sum()), "note": note} + + +def load_geography(year, level, state, files): + by = boundary_vintage(year, level) + gdf = get_boundaries(level, by, state if level == "tract" else None) + if level == "county": + gdf = gdf[gdf["GEOID"].str[:2] == state].reset_index(drop=True) + files.append({"kind": "boundary", "year": by, "level": level, "state": state, "table": "", + "url": gdf.attrs["source_url"], "sha256": gdf.attrs["sha256"]}) + gdf = gdf[["GEOID", "geometry"]].reset_index(drop=True) + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + queen = spatial_weights(gdf, "queen", transform="b") + rook = spatial_weights(gdf, "rook", transform="b") + assert list(queen.id_order) == list(gdf["GEOID"]) + orders = rc.contiguity_orders(queen, MAX_ORDER) + rook1 = rc.contiguity_orders(rook, 1) + pts = gdf.to_crs(equal_area_crs(gdf)).geometry.centroid + xy = np.column_stack([pts.x.to_numpy(), pts.y.to_numpy()]) / 1000.0 # km + return {"geoids": list(gdf["GEOID"]), "orders": orders, "rook1": rook1, "xy": xy, + "county": gdf["GEOID"].str[:5].to_numpy(), "boundary_year": by, + "weights": {"queen": queen.census_report, "rook": rook.census_report}} + + +# --------------------------------------------------------------------------- +# Pairs +# --------------------------------------------------------------------------- + + +def build_pairs(geo, level, rng): + """Pair index arrays with order (0 = beyond MAX_ORDER), same-county flag and distance.""" + n = len(geo["geoids"]) + o = sparse_triu(geo["orders"]) + i, j, order = o + parts = [(i, j)] + if level == "tract": + county = geo["county"] + wi, wj = [], [] + for c in np.unique(county): + idx = np.flatnonzero(county == c) + a, b = np.triu_indices(idx.size, 1) + wi.append(idx[a]) + wj.append(idx[b]) + wi, wj = np.concatenate(wi), np.concatenate(wj) + far = rc.pair_orders(geo["orders"], wi, wj) == 0 + wi, wj = wi[far], wj[far] + if wi.size > WITHIN_COUNTY_CAP: + keep = rng.choice(wi.size, WITHIN_COUNTY_CAP, replace=False) + wi, wj = wi[keep], wj[keep] + parts.append((wi, wj)) + ri = rng.integers(0, n, FAR_SAMPLE) + rj = rng.integers(0, n, FAR_SAMPLE) + a, b = np.minimum(ri, rj), np.maximum(ri, rj) + ok = (a != b) & (county[a] != county[b]) + a, b = a[ok], b[ok] + ok = rc.pair_orders(geo["orders"], a, b) == 0 + parts.append((a[ok], b[ok])) + else: + a, b = np.triu_indices(n, 1) + ok = rc.pair_orders(geo["orders"], a, b) == 0 + parts.append((a[ok], b[ok])) + pi = np.concatenate([p[0] for p in parts]) + pj = np.concatenate([p[1] for p in parts]) + key = pd.Index(pi.astype(np.int64) * n + pj).duplicated() + pi, pj = pi[~key], pj[~key] + order = rc.pair_orders(geo["orders"], pi, pj) + same = geo["county"][pi] == geo["county"][pj] + dist = np.sqrt(np.sum((geo["xy"][pi] - geo["xy"][pj]) ** 2, axis=1)) + rook = rc.pair_orders(geo["rook1"], pi, pj) == 1 + return {"i": pi, "j": pj, "order": order, "same": same, "dist": dist, "rook": rook} + + +def sparse_triu(m): + from scipy import sparse + t = sparse.triu(m, k=1).tocoo() + return t.row.astype(np.int64), t.col.astype(np.int64), t.data.astype(int) + + +def label_sets(pairs, level): + o = pairs["order"] + order_lab = np.where(o > 0, np.char.add("order ", o.astype(str)), "beyond order 3").astype(object) + sets = {"order": order_lab} + if level == "tract": + cnt = np.where(pairs["same"], "within county", "across county") + sets["order_county"] = (order_lab + " / " + cnt).astype(object) + rook_lab = np.where(o == 1, np.where(pairs["rook"], "order 1 rook (shared edge)", "order 1 queen-only (corner)"), + "") + sets["rook"] = rook_lab.astype(object) + bins = TRACT_DIST_BINS if level == "tract" else COUNTY_DIST_BINS + cut = pd.cut(pairs["dist"], bins, right=False) + sets["distance"] = np.asarray([f"{iv.left:g}-{iv.right:g} km" for iv in cut], dtype=object) + return sets + + +# --------------------------------------------------------------------------- +# Analysis +# --------------------------------------------------------------------------- + + +def holdout_models(geo, idx, level): + """Dense class matrices for the valid areas ``idx``.""" + sub = geo["orders"][idx][:, idx].toarray().astype(np.int16) + models = {"order (1-3)": np.where(sub > 0, sub - 1, -1)} + if level == "tract": + c = geo["county"][idx] + same = c[:, None] == c[None, :] + models["order x county"] = np.where(sub > 0, (sub - 1) * 2 + same, -1) + models["order x county (+ far within county)"] = np.where(sub > 0, (sub - 1) * 2 + same, + np.where(same, 6, -1)) + for m in models.values(): + np.fill_diagonal(m, -1) + return models + + +def analyse(year, state, level, frames, geo, rng, do_holdout, out_rows, extras): + pairs = build_pairs(geo, level, rng) + sets = label_sets(pairs, level) + gid = geo["geoids"] + group = pd.factorize(geo["county"])[0] + for name, (vtype, _, _) in VARIABLES.items(): + needed = {VARIABLES[name][2].split("_")[0]} if VARIABLES[name][1] != "ratio" else { + c.split("_")[0] for c in VARIABLES[name][2]} + missing = sorted(needed & set(frames.get("_missing", []))) + if missing: + extras["skipped"].append(f"{year} {STATES[state]} {level} {name}: VRE file " + f"{', '.join(missing)} unavailable (see Data: unavailable files)") + continue + est, reps, valid, info = variable_values(frames, name, gid, year) + if name == "population" and level == "county": + extras["skipped"].append(f"{year} {STATES[state]} county population: {info['note']} (no sampling error)") + continue + n_valid = int(valid.sum()) + d = np.zeros_like(reps) + d[valid] = rc.replicate_deviations(est[valid], reps[valid]) + keep = valid[pairs["i"]] & valid[pairs["j"]] + pi, pj = pairs["i"][keep], pairs["j"][keep] + base = {"year": year, "state": STATES[state], "geography": level, "variable": name, "type": vtype} + for sname, labs in sets.items(): + labs = labs[keep] + sel = labs != "" + tab = rc.summarize_pair_correlations(d, pi[sel], pj[sel], labs[sel], groups=group, n_boot=N_BOOT, + seed=SEED) + for row in tab.to_dict(orient="records"): + out_rows.append({**base, "classification": sname, **row}) + # noise floor and split-half on the order classification + labs = sets["order"][keep] + far = labs == "beyond order 3" + perm = rc.permutation_null(d, pi[far], pj[far], n_perm=10, seed=SEED) + halves = pd.concat([rc.split_half_agreement(d, pi, pj, labs, split=s, seed=SEED) + for s in ("random", "odd_even", "first_last")]) + extras["perm"].append({**base, **perm}) + for row in halves.to_dict(orient="records"): + extras["split"].append({**base, **row}) + extras["n"].append({**base, "n_areas": len(gid), "n_valid": n_valid, **info}) + spec = rc.replicate_spectrum(d[valid]) + if level == "tract": + extras["directions"][(year, state, name)] = spec["null_directions"] + extras["spectrum"].append({**base, "n_null": spec["n_null"], "effective_replicates": spec["effective_replicates"], + "lowest_eigen": [float(x) for x in spec["eigen_ratio"][:4]], + "mp_lower_edge": spec["mp_lower_edge"], "note": spec["note"]}) + if vtype == "count" and level == "tract": + _, agg = rc.aggregation_variance_ratio(d[valid], geo["county"][valid], n_boot=N_BOOT, seed=SEED) + extras["agg"].append({**base, **agg}) + if do_holdout: + idx = np.flatnonzero(valid) + t0 = time.time() + models = holdout_models(geo, idx, level) + # Design near-null directions: detected on this set when it is large enough (tracts), + # else taken from the same release's tracts (a property of the replicate design). + dirs = (extras["directions"].get((year, state, name)) if level == "county" else None) + remove = "auto" if dirs is None else np.vstack([dirs, np.ones((1, REPLICATE_COUNT))]) + ho = rc.holdout_model_comparison(d[idx], models, split="random", seed=SEED, remove_directions=remove) + rank = rc.rank_report(d[idx]) + lw_full = rc.ledoit_wolf_intensity(d[idx]) + extras["holdout"].append({**base, **ho, "rank": rank, "lw_full": lw_full["intensity"], + "seconds": time.time() - t0}) + extras["dev"][(year, state, level, name)] = (d, valid) + log(f"{year} {STATES[state]} {level}: {len(gid)} areas, {pairs['i'].size} pairs") + + +def cross_state(year, level, extras): + """MI x NY pairs: independent samples, an empirical null for the pairwise correlation.""" + rng = np.random.default_rng(SEED + 1) + rows = [] + null = rc.null_correlation(REPLICATE_COUNT) + for name in VARIABLES: + a = extras["dev"].get((year, "26", level, name)) + b = extras["dev"].get((year, "36", level, name)) + if a is None or b is None: + continue + ia, ib = np.flatnonzero(a[1]), np.flatnonzero(b[1]) + if level == "tract": + pi = rng.choice(ia, CROSS_STATE_SAMPLE) + pj = rng.choice(ib, CROSS_STATE_SAMPLE) + else: + pi, pj = (g.ravel() for g in np.meshgrid(ia, ib, indexing="ij")) + d = np.vstack([a[0], b[0]]) + r = rc.pair_correlations(d, pi, pj + a[0].shape[0]) + tab = rc.summarize_pair_correlations(d, pi, pj + a[0].shape[0], np.full(pi.size, "MI x NY"), + n_boot=N_BOOT, seed=SEED) + rows.append({"year": year, "geography": level, "variable": name, "type": VARIABLES[name][0], + "n_pairs": int(r.size), "mean": float(np.mean(r)), "se": float(tab["se_boot"].iloc[0]), + "sd": float(np.std(r)), "q975_abs": float(np.quantile(np.abs(r), 0.95)), + "share_beyond_critical": float(np.mean(np.abs(r) > null["critical_abs_r"]))}) + return rows + + +# --------------------------------------------------------------------------- +# Report +# --------------------------------------------------------------------------- + + +def _f(x, nd=3): + if x is None or (isinstance(x, float) and not math.isfinite(x)): + return "-" + return f"{x:.{nd}f}" + + +def _ci(row): + return f"{_f(row['mean'])} [{_f(row['ci_low'])}, {_f(row['ci_high'])}]" + + +def _md_table(header, rows): + out = ["| " + " | ".join(header) + " |", "|" + "|".join([" :--- "] * len(header)) + "|"] + out += ["| " + " | ".join(str(c) for c in r) + " |" for r in rows] + return "\n".join(out) + + +def _git_commit(): + try: + return subprocess.run(["git", "describe", "--always", "--dirty"], cwd=ROOT, capture_output=True, text=True, + check=True).stdout.strip() + except Exception: # pragma: no cover + return "unknown" + + +def _versions(): + import geopandas + import libpysal + import scipy + return {"census-acs-toolkit": census.__version__, "python": platform.python_version(), + "numpy": np.__version__, "scipy": scipy.__version__, "pandas": pd.__version__, + "geopandas": geopandas.__version__, "libpysal": libpysal.__version__} + + +INTERPRETATION = """\ +## Findings + +Numbers below refer to the tables in this report (2020-2024 release unless +stated). r is the replicate correlation of two areas' sampling errors. + +{findings} + +## C2 conclusion + +{conclusion} +""" + + +def write_report(df, extras, files, years, elapsed, findings, conclusion): + os.makedirs(os.path.dirname(OUT_MD), exist_ok=True) + df.to_csv(OUT_CSV, index=False, float_format="%.5g") + null = rc.null_correlation(REPLICATE_COUNT) + null40 = rc.null_correlation(REPLICATE_COUNT // 2) + L = [] + L.append("# ACS sampling-error covariance between areas (VRE replicates)\n") + L.append(f"Generated {datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M UTC')} by " + f"`scripts/acs_error_covariance.py` (commit {_git_commit()}, {elapsed / 60:.1f} min). " + "PLAN_ROUND_16 workstream C (C1-C2). Status: **info** (gates nothing). " + f"Per-class numbers: `docs/audits/{os.path.basename(OUT_CSV)}` (written by the script; git-ignored like " + "the other audit CSVs).\n") + L.append("## Question\n") + L.append("Do the sampling errors of ACS 5-year estimates covary between areas (neighbouring tracts or " + "counties, tracts in the same county), and can 80 replicates measure that covariance well enough " + "to build it into a measurement-error correction (workstream A1) or a LISA null model " + "(workstream B)?\n") + L.append("## Methods\n") + L.append( + "* **Covariance.** For estimates X (area i) and Y (area j) of one release, " + "Cov = 4/80 sum_r (X_r - X)(Y_r - Y) over the 80 successive-difference replicates of the Census " + "Variance Replicate Estimate (VRE) tables, the cross-product form of the VRE variance formula " + "(census.replicate.VRE_DOCUMENTATION, formula 1). The 4/80 factor cancels in the correlation r, " + "which is the cosine of the two replicate-deviation vectors. Implementation: " + "`census/replicate_covariance.py`.\n" + "* **Variables.** Counts: total population B01003_001 (tracts only; county populations are " + "controlled to the population estimates and have no sampling error), occupied housing units " + "B25003_001, persons below poverty B17001_002. Proportions: poverty rate B17001_002/B17001_001 and " + "owner share B25003_002/B25003_001, recomputed on every replicate. Median: household income linearly " + "interpolated from the B19001 " + "replicate distributions (census.medians; no published median table has VRE files - checked against " + "the 2019 and 2024 VRE table lists); areas whose median is top- or bottom-coded are excluded. Areas " + "with a zero estimate denominator, a zero denominator in any replicate (the replicate ratio is " + "undefined; the SE convention of VRE documentation Example 7, ratio 0, would fabricate a deviation of " + "minus the estimate), controlled estimates or zero replicate variance are excluded " + "(their correlation is undefined).\n" + "* **Pairs.** Contiguity from queen weights (libpysal via census.spatial.spatial_weights) on the " + "Census cartographic boundary files of the release's vintage; orders 1-3 are graph distances in the " + "queen contiguity graph (order 1 split into rook = shared edge vs queen-only = corner). Tracts: all " + f"pairs of order 1-3; tract pairs in the same county beyond order 3 (random sample of at most " + f"{WITHIN_COUNTY_CAP:,}); and a random sample of {FAR_SAMPLE:,} draws of tract pairs in different " + "counties beyond order 3. Counties: all pairs in the state. Distance = centroid distance in an " + "equal-area projection (EPSG:5070).\n" + "* **Structured summaries, not pairwise matrices.** The replicate covariance of n areas has rank " + "<= 80, and one pairwise r from 80 replicates has null SD 1/sqrt(80). So only class summaries are " + "reported: mean and median r per contiguity order, within vs across county, distance band. Their " + f"uncertainty: a bootstrap over the 80 replicate columns ({N_BOOT} draws; replicate noise; " + "approximate because SDR deviations are not exactly iid across replicates) and a delete-one-county " + "jackknife (drops every pair touching the county; area heterogeneity plus replicate noise). " + "Intervals are mean +/- 1.96 x the larger of the two SEs.\n" + "* **Noise floor.** Analytic: if one deviation vector is spherically symmetric in R^80 and " + f"independent of the other, (1 + r)/2 ~ Beta(39.5, 39.5): SD {null['sd']:.3f}, two-sided 5% critical " + f"|r| {null['critical_abs_r']:.3f} (40 replicates: SD {null40['sd']:.3f}, critical " + f"{null40['critical_abs_r']:.3f}). Empirical: (a) replicate-index derangements applied to the " + "second area of each far pair; (b) Michigan x New York area pairs, whose samples are independent, " + "so their correlations show the real null spread of SDR replicate correlations.\n" + "* **Reliability.** Split-half agreement: per-pair r computed on two disjoint halves of the " + "replicates (random, odd/even, 1-40 vs 41-80); the correlation across pairs of the two halves' r is " + "the reliability of a 40-replicate pairwise r, stepped up to 80 replicates by Spearman-Brown. " + "Aggregation ratio (counts, tracts): Var(sum of a county's tracts) / sum of the tracts' variances " + "= 1 + sum of within-county covariances / sum of variances; < 1 means negatively correlated errors " + "(the signature of weighting controls), and it pools every within-county pair in one number.\n" + "* **Replicate design spectrum.** The eigenvalues of the replicate-space Gram matrix of standardized " + "deviations (80 x 80, over thousands of tracts) are compared with the Marchenko-Pastur lower edge " + "for independent replicates (`replicate_spectrum`). Directions far below it are near-null for every " + "area: a property of the replicate-factor design, not of geography. They lower the effective number " + "of replicates, hence widen the null spread of r.\n" + "* **Structured model and shrinkage, scored out of sample.** Replicates are split in half (random, " + "seeded); on one half fit (i) one correlation per contiguity order 1-3 (0 beyond), (ii) for tracts " + "one per order x within/across county (0 beyond order 3), (iii) as (ii) plus one for far pairs " + "within a county, and (iv) Ledoit-Wolf shrinkage of the replicate correlation toward the identity; " + "score the Gaussian log-likelihood of the other half's standardized replicate deviations (scaled so " + "E[x x^T] = Sigma) relative to independence; average both directions. Each half is first projected " + "orthogonally to the design near-null directions and the constant vector: otherwise the held-out " + "half's component along those directions is minus the training half's for every area at once, a " + "cross-half dependence that Ledoit-Wolf exploits (it then 'gains' 30-90 nats per replicate even on " + "variables with no neighbour correlation; with the projection it loses, as it does on independent " + "synthetic data). A gain > 0 means the fitted structure predicts held-out replicates better than " + "independent errors. A fitted structure that is not positive definite is projected to the nearest " + "correlation matrix with eigenvalues >= 1e-3 and flagged.\n") + L.append("## Data\n") + L.append(f"Releases {', '.join(f'{y - 4}-{y}' for y in years)}; states Michigan (26) and New York (36). " + "Files (cached, not committed):\n") + seen = set() + rows = [] + for f in files: + if f["url"] in seen or f.get("unavailable"): + continue + seen.add(f["url"]) + name = f"`{f['url'].rsplit('/', 1)[-1]}`" + (" (manual download)" if f.get("acquisition") == "manual download" + else "") + rows.append([f["kind"], f["year"], f["level"], name, f"`{f['sha256']}`"]) + L.append(_md_table(["kind", "year", "level", "file", "SHA-256"], rows) + "\n") + missing = [f for f in files if f.get("unavailable")] + if missing: + L.append("**Unavailable files** (skipped; the variables that need them are not analysed for that state " + "and release, everything else is):\n") + L.append(_md_table(["year", "level", "state", "table", "URL", "reason"], + [[f["year"], f["level"], STATES.get(f["state"], f["state"]), f["table"], f["url"], + f["unavailable"]] for f in missing]) + "\n") + L.append("A rejection of one specific URL can persist while every other file in the same directory " + "downloads normally, so it is not retried further. To include such a file, download it in a " + "browser and save it as " + ", ".join(f"`{f['manual_path']}`" for f in missing) + + " (default `$CENSUS_VRE_DIR` = `.cache/vre`); census.replicate then records its SHA-256 with " + "source 'manual download' and the report marks it.\n") + L.append("Versions: " + ", ".join(f"{k} {v}" for k, v in _versions().items()) + "\n") + L.append("Areas analysed (after exclusions):\n") + nrows = [[r["year"], r["state"], r["geography"], r["variable"], r["type"], r["n_areas"], r["n_valid"], + r["n_zero_variance"], r["note"]] for r in extras["n"]] + L.append(_md_table(["year", "state", "geography", "variable", "type", "areas", "analysed", + "zero variance", "note"], nrows) + "\n") + if extras["skipped"]: + L.append("Skipped: " + "; ".join(extras["skipped"]) + ".\n") + + def class_table(year, geo, sname, classes): + sub = df[(df["year"] == year) & (df["geography"] == geo) & (df["classification"] == sname)] + if not len(sub): + return f"_Not analysed for {geo}s in this release (see Data: skipped)._" + hdr = ["state", "variable"] + classes + rows = [] + for (st, var), g in sub.groupby(["state", "variable"], sort=False): + cells = [] + for c in classes: + r = g[g["class"] == c] + cells.append(_ci(r.iloc[0]) + f" (n={int(r.iloc[0]['n_pairs'])})" if len(r) else "-") + rows.append([st, var] + cells) + return _md_table(hdr, rows) + + for year in years: + L.append(f"## Results {year - 4}-{year}\n") + L.append("Mean pairwise r [95% interval] (number of pairs) by queen contiguity order.\n") + for geo in ("tract", "county"): + L.append(f"### {'Tracts' if geo == 'tract' else 'Counties'}: contiguity order\n") + L.append(class_table(year, geo, "order", ["order 1", "order 2", "order 3", "beyond order 3"]) + "\n") + L.append("### Tracts: within vs across county (weighting controls)\n") + L.append(class_table(year, "tract", "order_county", + ["order 1 / within county", "order 1 / across county", "order 2 / within county", + "order 2 / across county", "beyond order 3 / within county", + "beyond order 3 / across county"]) + "\n") + L.append("### Rook (shared edge) vs queen-only (corner) first-order neighbours\n") + for geo in ("tract", "county"): + L.append(f"{'Tracts' if geo == 'tract' else 'Counties'}:\n") + L.append(class_table(year, geo, "rook", ["order 1 rook (shared edge)", "order 1 queen-only (corner)"]) + + "\n") + L.append("### By centroid distance\n") + for geo, bins in (("tract", TRACT_DIST_BINS), ("county", COUNTY_DIST_BINS)): + labs = [f"{a:g}-{b:g} km" for a, b in zip(bins[:-1], bins[1:])] + L.append(f"{'Tracts' if geo == 'tract' else 'Counties'}:\n") + L.append(class_table(year, geo, "distance", labs) + "\n") + agg = [a for a in extras["agg"] if a["year"] == year] + if agg: + L.append("### Aggregation ratio: Var(county sum of tracts) / sum of tract variances\n") + L.append(_md_table( + ["state", "variable", "counties", "tracts", "pooled ratio (SE)", "median county ratio", + "implied mean within-county r (SE)"], + [[a["state"], a["variable"], a["n_groups"], a["n_areas"], + f"{_f(a['pooled_ratio'])} ({_f(a['pooled_ratio_se_boot'])})", _f(a["median_group_ratio"]), + f"{_f(a['pooled_implied_mean_corr'], 4)} ({_f(a['pooled_implied_mean_corr_se_boot'], 4)})"] + for a in agg]) + "\n") + L.append("### Noise floor and spread\n") + L.append(f"Analytic null (80 replicates): SD {null['sd']:.3f}, 95% |r| {null['critical_abs_r']:.3f}.\n") + cs = [c for c in extras["cross"] if c["year"] == year] + if cs: + L.append("Michigan x New York pairs (independent samples):\n") + L.append(_md_table(["geography", "variable", "pairs", "mean r (SE)", "SD of r", "95% |r|", + "share |r| > analytic critical"], + [[c["geography"], c["variable"], c["n_pairs"], f"{_f(c['mean'], 4)} ({_f(c['se'], 4)})", + _f(c["sd"]), _f(c["q975_abs"]), _f(c["share_beyond_critical"])] for c in cs]) + "\n") + pm = [p for p in extras["perm"] if p["year"] == year] + L.append("Derangement null on far pairs (same state, beyond order 3):\n") + L.append(_md_table(["state", "geography", "variable", "pairs", "mean", "SD", "95% |r|"], + [[p["state"], p["geography"], p["variable"], p["n_pairs"], _f(p["mean"], 4), _f(p["sd"]), + _f(p["critical_abs_r"])] for p in pm]) + "\n") + spec = [x for x in extras["spectrum"] if x["year"] == year] + if spec: + L.append("Replicate design spectrum (lowest eigenvalues relative to the mean; Marchenko-Pastur lower " + "edge for independent replicates in brackets):\n") + L.append(_md_table(["state", "geography", "variable", "lowest 4 eigenvalues (MP edge)", "near-null", + "effective replicates", "null SD 1/sqrt(eff.)"], + [[x["state"], x["geography"], x["variable"], + " ".join(_f(v) for v in x["lowest_eigen"]) + f" ({_f(x['mp_lower_edge'])})", + x["n_null"] if not x["note"] else "not tested (too few areas)", + x["effective_replicates"] if not x["note"] else "-", + _f(1 / math.sqrt(x["effective_replicates"])) if not x["note"] else "-"] + for x in spec]) + "\n") + sub = df[(df["year"] == year) & (df["classification"] == "order")] + cross_sd = {(c["geography"], c["variable"]): c["sd"] for c in cs} + L.append("Spread of pairwise r per class: SD, excess SD over the analytic null (sqrt(SD^2 - 1/80)) and " + "over the empirical Michigan x New York null (sqrt(SD^2 - SD_cross^2)), and the share of pairs " + "with |r| above the analytic 5% critical value (0.05 expected under no correlation; about 0.06 " + "in the independent cross-state pairs):\n") + + def _excess(r): + c = cross_sd.get((r["geography"], r["variable"])) + return math.sqrt(max(0.0, r["sd"] ** 2 - c ** 2)) if c is not None else math.nan + + L.append(_md_table(["state", "geography", "variable", "class", "SD of r", "excess SD (analytic)", + "excess SD (cross-state)", "share |r| > crit"], + [[r["state"], r["geography"], r["variable"], r["class"], _f(r["sd"]), _f(r["excess_sd"]), + _f(_excess(r)), _f(r["share_beyond_critical"])] + for r in sub.to_dict(orient="records")]) + "\n") + sp = pd.DataFrame([s for s in extras["split"] if s["year"] == year]) + if len(sp): + L.append("### Split-half reliability of individual pairwise correlations\n") + L.append("Per class: half-A and half-B class means (random split), and the reliability of a single " + "pair's r (correlation across pairs between halves; stepped up to 80 replicates), for each " + "split.\n") + rows = [] + for (st, geo, var, cl), g in sp.groupby(["state", "geography", "variable", "class"], sort=False): + rnd = g[g["split"] == "random"].iloc[0] + rel = " / ".join(_f(g[g["split"] == s]["pair_reliability_full"].iloc[0], 2) + for s in ("random", "odd_even", "first_last")) + rows.append([st, geo, var, cl, rnd["n_pairs"], _f(rnd["mean_half_a"]), _f(rnd["mean_half_b"]), rel]) + L.append(_md_table(["state", "geography", "variable", "class", "pairs", "mean (half A)", + "mean (half B)", "reliability (random / odd-even / first-last)"], rows) + "\n") + ho = [h for h in extras["holdout"] if h["year"] == year] + if ho: + L.append("### Held-out replicate log-likelihood gain over independence (nats per replicate)\n") + names = list(dict.fromkeys(k for h in ho for k in h["gain_nats_per_replicate"])) + rows = [] + for h in ho: + g = h["gain_nats_per_replicate"] + rho = h["fitted_rho"]["order (1-3)"] + rho1 = np.mean([p[0] for p in rho]) if rho and len(rho[0]) else math.nan + rows.append([h["state"], h["geography"], h["variable"], h["n_areas"]] + + [(_f(g[k], 2) + (" (projected)" if h["projected_to_pd"].get(k) else "")) if k in g else "-" + for k in names] + + [_f(rho1), _f(np.mean(h["ledoit_wolf_intensity"])), _f(h["lw_full"]), + f"{h['rank']['rank']} / {h['rank']['n_areas']}"]) + L.append(_md_table(["state", "geography", "variable", "areas"] + names + + ["fitted order-1 r (halves)", "LW intensity (40 reps)", "LW intensity (80 reps)", + "rank / areas"], rows) + "\n") + if len(years) > 1: + a, b = years[0], years[-1] + L.append(f"## Stability across releases ({b - 4}-{b} vs {a - 4}-{a})\n") + L.append("Mean r per class in each release (different tract vintages: 2010 tracts in 2015-2019, 2020 " + "tracts in 2020-2024; so pair sets differ). Difference with a combined SE.\n") + sub = df[df["classification"].isin(["order", "order_county"])] + rows = [] + for (st, geo, var, cl), g in sub.groupby(["state", "geography", "variable", "class"], sort=False): + ra, rb = g[g["year"] == a], g[g["year"] == b] + if not (len(ra) and len(rb)): + continue + ra, rb = ra.iloc[0], rb.iloc[0] + diff = ra["mean"] - rb["mean"] + se = math.hypot(ra["se"], rb["se"]) + rows.append([st, geo, var, cl, _ci(rb), _ci(ra), f"{_f(diff)} ({_f(se)})"]) + L.append(_md_table(["state", "geography", "variable", "class", f"{b}", f"{a}", "difference (SE)"], rows) + + "\n") + L.append(INTERPRETATION.format(findings=findings, conclusion=conclusion)) + L.append(LIMITATIONS) + with open(OUT_MD, "w", encoding="utf-8") as fh: + fh.write("\n".join(L).rstrip() + "\n") + + +LIMITATIONS = """\ +## Limitations + +* Two states, two releases. Michigan and New York contain dense urban tracts (Detroit, New York City) and + sparse rural ones; other states' weighting areas and sample sizes differ. +* Counties stand in for ACS weighting areas. Census controls weights to population and housing-unit + estimates within weighting areas that are counties or groups of small counties, so "within county" is an + approximation of "within weighting area"; for small Michigan counties grouped into one weighting area the + negative covariance would appear "across county". +* The replicate bootstrap treats the 80 SDR replicate deviations as exchangeable draws. They are + constructed from a Hadamard design and are close to, but not exactly, iid; the delete-one-county jackknife + and the split-half comparisons are reported alongside as checks, and every interval uses the larger SE. +* Tract-pair classes beyond order 3 are random samples (sizes in the CSV), so their means carry sampling + error from that draw as well (included in both SEs). +* The median is a linear interpolation from B19001 brackets, not Census's published B19013 (computed from + finer internal distributions; no VRE file exists for it). Its replicate deviations are therefore those of + the interpolated statistic. +* Correlation is reported, not covariance in units: covariance in units scales with each area's + variance, which the published MOEs already give; only the correlation structure is new information. +* Distance bins use centroid distance; large rural tracts make "0-2 km" mostly urban pairs. +""" + + +def main(argv=None): + p = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + p.add_argument("--years", type=int, nargs="+", default=[2024, 2019]) + p.add_argument("--no-holdout", action="store_true", help="skip the dense held-out model comparison") + p.add_argument("--holdout-years", type=int, nargs="*", default=[2024], + help="releases for the dense held-out model comparison (default 2024)") + p.add_argument("--tract-years", type=int, nargs="*", default=[2024, 2019], + help="releases analysed at tract level (counties: every --years release)") + p.add_argument("--results", help="pickle of computed results: loaded if it exists (re-render the report " + "only), else written after the computation") + args = p.parse_args(argv) + t0 = time.time() + if args.results and os.path.exists(args.results): + import pickle + with open(args.results, "rb") as fh: + df, extras, files, elapsed = pickle.load(fh) + findings, conclusion = interpret(df, extras, args.years) + write_report(df, extras, files, args.years, elapsed, findings, conclusion) + log(f"re-rendered {OUT_MD} from {args.results}") + return 0 + rng = np.random.default_rng(SEED) + files, rows = [], [] + extras = {"perm": [], "split": [], "n": [], "agg": [], "holdout": [], "dev": {}, "skipped": [], "cross": [], + "spectrum": [], "directions": {}} + for year in args.years: + for level in ("tract", "county"): + if level == "tract" and year not in args.tract_years: + continue + for st in STATES: + frames = _load_tables(year, level, st, files) + geo = load_geography(year, level, st, files) + analyse(year, st, level, frames, geo, rng, + not args.no_holdout and year in args.holdout_years, rows, extras) + extras["cross"] += cross_state(year, level, extras) + extras["dev"].clear() + extras["directions"].clear() + df = pd.DataFrame(rows) + if args.results: + import pickle + with open(args.results, "wb") as fh: + pickle.dump((df, extras, files, time.time() - t0), fh) + findings, conclusion = interpret(df, extras, args.years) + write_report(df, extras, files, args.years, time.time() - t0, findings, conclusion) + log(f"wrote {OUT_MD} and {OUT_CSV}") + return 0 + + +def interpret(df, extras, years): + """Narrative text; see FINDINGS / CONCLUSION (written for the recorded file hashes).""" + return FINDINGS, CONCLUSION + + +# Written for the run recorded in docs/audits/acs_sampling_covariance.md (file hashes listed there); the +# tables are regenerated, this text is not, so re-read it against them after a rerun on other files. +FINDINGS = """\ +1. **Noise floor.** One pairwise r from 80 replicates is noise-dominated. Analytic null SD 0.112 (95% |r| + 0.219). Real independent pairs (Michigan x New York tracts) have mean r within 0.001 of 0 but SD + 0.115-0.116 and 95% |r| 0.226-0.236; 6% of them exceed the analytic critical value. The reason is the + replicate design: in every variable and both states the same 3 directions of replicate space are + near-null (eigenvalues 0.02-0.11 of the mean vs the Marchenko-Pastur edge 0.69-0.77), so a + correlation has about 77 effective replicates (null SD 1/sqrt(77) = 0.114). Derangements of replicate + indices destroy this structure and give the textbook 0.112, so they understate the real floor + slightly. Practical floor: |r| below ~0.23 is indistinguishable from 0 for a single pair. +2. **Uncontrolled characteristics: no detectable neighbour correlation.** Poverty rate, owner share and + interpolated median household income have mean tract r within +/-0.003 of 0 at every contiguity + order, within and across counties, and at every distance, with 95% intervals of about +/-0.002-0.003 + (e.g. order 1: Michigan poverty rate -0.000 [-0.003, 0.002], New York owner share -0.001 [-0.002, + 0.001], Michigan median income 0.000 [-0.002, 0.003]). The poverty count is the same with a slight + negative tilt among neighbours in the same county (Michigan order 1 -0.004 [-0.007, -0.002]). + Counties: the class means for these variables lie in -0.015..+0.025 with intervals of about + +/-0.02-0.03, none clearly different from the cross-state null (which itself has small positive means, + +0.006 to +0.018, from the offset described in 4). The per-pair spread beyond the cross-state null is ~0 and + the split-half reliability of an individual pair's r is 0.02-0.07, the same as for far pairs: the + individual pairwise correlations are replicate noise. +3. **Controlled totals: negative correlation among neighbours in the same county.** Total population + (B01003, whose county total is controlled to the population estimates) and occupied housing units + (B25003_001, tied to the housing-unit controls) show clearly negative first-order correlations inside + counties: population order 1 within county Michigan -0.081 [-0.106, -0.056], New York -0.043 + [-0.065, -0.020]; households -0.028 [-0.039, -0.018] and -0.016 [-0.024, -0.008]. They decay fast + with order (population order 2: -0.018 / -0.010; order 3: -0.007 / -0.004; far within county: -0.001) + and vanish across county lines at every order (order 1 across county: +0.003 [-0.007, 0.014] and + +0.002 [-0.006, 0.009]). Rook (shared-edge) pairs are more negative than corner-only pairs (Michigan + population -0.080 vs -0.029). The aggregation ratio confirms the control mechanism without any + pairwise noise: Var(sum of a county's tracts) / sum of the tract variances is 0.000 for population in + Michigan and 0.005 in New York (the tract errors sum to exactly zero within a county), 0.22 / 0.26 for + households and 0.92 / 0.88 for the poverty count. Here the structure is real at the pair level too: + split-half reliability of an individual neighbour pair's r is 0.6-0.67 for population and 0.2-0.34 for + households (excess SD 0.13-0.16 and 0.06-0.08), i.e. strongly heterogeneous, with some neighbour pairs + far more negative than the mean (a few tracts sharing a county's controlled total). These are + 2020-2024 numbers; in 2015-2019 population behaves the same but households do not (6). +4. **A design-wide positive offset, not spatial covariance, at the county level.** County households + (and New York county owner share in 2015-2019, +0.02-0.04) show mean r of about +0.03 at every order + and distance - and + equally between Michigan and New York counties (+0.029-0.032), whose samples are independent. The + replicate estimates of these counties sit systematically above the full-sample estimate (mean + deviation about 0.18 of its RMS), so the uncentred 4/80 cross-products of any two such counties are + positive (0.18^2 = 0.03). This is part of what the ACS covariance formula returns, but it is not a + neighbourhood effect, and with 80 replicates it cannot be separated from a true common component. +5. **Structured models out of sample.** One correlation per contiguity order, fitted on 40 replicates, + predicts the other 40 better than independence only where (3) found structure: population +33 (MI) and + +17 (NY) nats per replicate, households +4.4 / +2.2; for every uncontrolled variable and for counties + the gain is within +/-0.2 of 0 (no improvement). Splitting by county adds a little for households + (+5.1) and, for population in Michigan, cannot be represented as a positive-definite correlation + matrix (projected, -165): the within-county constraint is an exact linear restriction (the covariance + is singular), not a small correlation. Adding a single far-within-county correlation makes every + controlled-total fit non-positive-definite. Ledoit-Wolf shrinkage toward the identity has intensity + 0.87 (40 replicates) to 0.93 (80) for tracts - it keeps only 7-13% of the sample correlation - and, + scored on held-out replicates after removing the design directions, it is worse than independence + for every variable (-25 to -50 nats for tracts). Without that projection it appeared to gain 30-90 + nats, entirely from the cross-half dependence created by the near-null design directions. +6. **Stability across releases (tracts and counties, 2015-2019 vs 2020-2024).** One 2015-2019 file is + missing: Michigan tract total population (`2019/data/5-year/140/B01003_26.csv.zip`). census.gov's + firewall answered that one URL with its 'Request Rejected' page (support ID 13979921158196382640) on + the first request and on the single delayed retry, while every other 2019 tract file, including New + York's B01003_36 from the same directory, downloaded normally; the rejection is URL-specific, not + rate limiting (see Data: unavailable files). So the 2015-2019 tract comparison has no Michigan + population row; everything else was analysed. Results: (a) **Uncontrolled variables are stable.** + Poverty count, poverty rate, owner share and median income stay at 0 to within +/-0.005 at every order + in both states and releases; of 108 tract class comparisons for them, 3 differ by more than 2 combined + SEs (about 5 expected by chance; largest |z| 2.5, all differences <= 0.004). (b) **Population (New + York) is stable**: order 1 -0.039 [-0.060, -0.018] in 2015-2019 vs -0.040 [-0.060, -0.020] (within + county -0.042 vs -0.043), the same fast decay, a county aggregation ratio of 0.000 again (tract errors + sum to zero within a county), and split-half pair reliability 0.60 (2020-2024: 0.61). (c) + **Households change sign.** Occupied housing units (B25003_001) in 2015-2019 have *positive* mean r + between neighbouring tracts in the same county - Michigan order 1 +0.025 [0.016, 0.033] (within county + +0.028), New York +0.012 [0.003, 0.020] - against -0.025 and -0.015 in 2020-2024 (differences 8.4 and + 4.7 combined SEs), still zero across county lines (+0.001 / -0.004) and beyond order 3. Their + aggregation ratio is 0.90 (Michigan) and 1.00 (New York) instead of 0.22 / 0.26: in 2015-2019 the tract + household errors are not constrained to (nearly) cancel within a county. The pair reliability (0.37 / + 0.28) shows the pattern is real, not replicate noise. The study cannot say why from the replicates + alone; a change between the releases in how occupied-unit estimates relate to the weighting controls + is the obvious candidate, not verified here. (d) County class means agree within about 1.5 combined + SEs for every variable and order except New York owner share beyond order 3 (+0.036 vs +0.012, + difference -0.024, SE 0.011) - the county-wide offset in (4), not a neighbour effect. +""" + +CONCLUSION = """\ +**The covariance structure is estimable only as a few structured summaries, and for the variables most +analyses map it is indistinguishable from zero.** + +* For characteristics that are not weighting controls (rates, shares, medians and most counts), the mean + sampling-error correlation between neighbouring tracts is 0 to within about +/-0.003 (95%), at orders + 1-3, within and across counties, in Michigan and New York; between neighbouring counties it is 0 to + within about +/-0.03. Individual pairwise correlations are replicate noise (reliability ~0.05). There + is nothing to estimate beyond "approximately independent", and an unrestricted or shrunk replicate + covariance would only add noise (Ledoit-Wolf loses out of sample). +* For controlled totals (population, households, and to a lesser degree counts tied to them) the + structure is real and estimable: negative, confined to first- and second-order neighbours inside the + same county (order 1: -0.08 to -0.04 for population, -0.03 to -0.015 for households in 2020-2024; + population equally in 2015-2019, households there +0.01 to +0.03 instead, see Findings 6), and summarised + far more precisely by the within-county aggregation ratio (population: tract errors sum to zero within + a county). It is a linear constraint, not a smooth correlation function: a one-parameter-per-order + model captures it out of sample, a full within-county correlation model is not positive definite. +* Using it. In a measurement-error correction (A1) or LISA null model (B), sampling errors of rates, + shares and medians can be treated as independent across areas, with that assumption cited to this + study. For controlled counts the correct treatment is the replicate covariance itself (4/80 D D^T, + rank 80 - usable for sums and contrasts, as census.replicate already does), or a structured model with + one negative correlation per contiguity order within county; a generic positive spatial correlation + (e.g. SAR errors) has the wrong sign for them. +* **For the LISA parameter grid (-0.2..0.5):** the data place ACS sampling-error neighbour correlation at + about 0 (|rho| <= 0.003) for rates, shares and medians, and at -0.02 to -0.08 (95% bounds -0.11 to + -0.01) for controlled counts in tracts; the only positive neighbour correlations beyond the design-wide + county offset of about +0.03 are 2015-2019 tract households, +0.01 to +0.03 (95% bounds up to +0.033) + within county. The release matters for controlled counts. Grid values of +0.1..0.5 are therefore + stress tests outside the observed range, not plausible ACS conditions for these variables; the + realistic end of the grid is -0.1..0.05. Correlated-error scenarios with large positive rho should be + stated as hypothetical (for example for variables or survey designs not studied here). +* Limits: two states, two releases at tract level (2015-2019 without Michigan population: firewall + rejection of that one file), 80 replicates (about 77 effective), and counties as a proxy for weighting + areas. The 95% |r| of 0.23 for a single pair means no individual pair can be + flagged; only class means are evidence. +""" + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/fetch_acs_errata.py b/scripts/fetch_acs_errata.py index 80841f0..eafe799 100644 --- a/scripts/fetch_acs_errata.py +++ b/scripts/fetch_acs_errata.py @@ -4,8 +4,8 @@ Walks every page of the two listings (tab 'all'), fetches each note page and parses it with census.errata.parse_note. Writes the records and a sidecar (data/acs_errata.sidecar.json) with the listing URLs, the retrieval date, the -record count and the SHA-256 of the data file. Requests are spaced (census.gov -rejects rapid scripted requests). +record count and the SHA-256 of the data file. Requests are spaced by --delay +as a courtesy, on top of census.http's pacing and retries. Usage: python scripts/fetch_acs_errata.py [--delay 2] [--html-cache DIR] --html-cache keeps the raw pages (for re-parsing without re-fetching). @@ -26,12 +26,13 @@ sys.path.insert(0, ROOT) from census import errata # noqa: E402 +from census.http import polite_get # noqa: E402 LISTINGS = { "errata": errata.ERRATA_LISTING_URL, "user_note": errata.USER_NOTES_LISTING_URL, } -USER_AGENT = "acs-batch-downloader/errata (research toolkit; python-requests)" +USER_AGENT = "census-research-engine/errata (research toolkit; python-requests)" MAX_PAGES = 40 @@ -50,7 +51,8 @@ def get(self, url: str) -> str: with open(path, encoding="utf-8") as f: return f.read() time.sleep(self.delay) - resp = self.session.get(url, timeout=60) + resp = polite_get(url, user_agent=USER_AGENT, error=requests.RequestException, session=self.session, + timeout=60) resp.raise_for_status() resp.encoding = "utf-8" text = resp.text diff --git a/scripts/fetch_tract_relationships.py b/scripts/fetch_tract_relationships.py index 97fc105..8b70dee 100644 --- a/scripts/fetch_tract_relationships.py +++ b/scripts/fetch_tract_relationships.py @@ -42,9 +42,10 @@ from datetime import datetime, timezone from typing import Dict, List, Optional, Tuple -import requests - ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.insert(0, ROOT) + +from census.http import polite_get # noqa: E402 OUT_DIR = os.path.join(ROOT, "data", "geo_crosswalks") NATIONAL_URL = ( @@ -65,7 +66,7 @@ LAND_TOLERANCE = 0.10 # Matches whose land area differs by more than this are listed in the sidecar. LAND_REPORT = 0.01 -USER_AGENT = "acs-batch-downloader/fetch_tract_relationships (research toolkit)" +USER_AGENT = "census-research-engine/fetch_tract_relationships (research toolkit)" # Output column -> source column in the national file. NATIONAL_COLUMNS: List[Tuple[str, str]] = [ @@ -92,7 +93,7 @@ def download(url: str, local: Optional[str]) -> Tuple[bytes, Dict[str, Optional[ if local: with open(local, "rb") as f: return f.read(), {"retrieved_from": "local copy of " + url, "last_modified": None} - resp = requests.get(url, headers={"User-Agent": USER_AGENT}, timeout=300) + resp = polite_get(url, user_agent=USER_AGENT, timeout=300) resp.raise_for_status() ctype = resp.headers.get("Content-Type", "") if "html" in ctype.lower(): diff --git a/scripts/inference_validation.py b/scripts/inference_validation.py index d9652b8..44b52a2 100644 --- a/scripts/inference_validation.py +++ b/scripts/inference_validation.py @@ -24,6 +24,11 @@ --jobs output is identical to the serial output at the same seed except the "seconds" column (wall time per study). +The Round 16 spatial regression validation studies (spatial_error_attenuation, +spatial_error_attenuation_correlated, spatial_dgp_recovery, common_factor_lr, +common_factor_lr_acs; info rows only) take 5-25 min each and run only when named with --studies +(docs/audits/spatial_regression_validation.md). + Report versus gate (PLAN_ROUND_12 decision P12-3): by default the command is a report generator and always exits 0. With --gate it exits 1 when any gated row has status FAIL or a required study (REQUIRED_STUDIES) produced no gated @@ -68,8 +73,10 @@ ROOT = os.path.dirname(HERE) sys.path.insert(0, ROOT) +from census.lisa_validation import LISA_STUDIES # noqa: E402 from census.simulation import (BAND, CV_PROFILES, DESIGN_STUDIES, FAST, FULL, NOMINAL, SPATIAL_STUDIES, # noqa: E402 STUDIES, group_config, run_study, run_study_job) +from census.simulation_spreg import SPREG_VALIDATION_STUDIES # noqa: E402 OUT_MD = os.path.join(ROOT, "docs", "audits", "inference_validation.md") @@ -260,7 +267,8 @@ def main() -> int: if unknown: ap.error(f"unknown studies: {', '.join(unknown)} (choose from {', '.join(STUDIES)})") main_studies = None if args.studies is None else [ - s for s in args.studies if s not in SPATIAL_STUDIES and s not in DESIGN_STUDIES] + s for s in args.studies if s not in SPATIAL_STUDIES and s not in DESIGN_STUDIES and s not in LISA_STUDIES + and s not in SPREG_VALIDATION_STUDIES] groups = [] if main_studies is None or main_studies: groups.append(("main", main_studies)) @@ -271,6 +279,14 @@ def main() -> int: else [s for s in args.studies if s in DESIGN_STUDIES]) if designs: # Round 10 multilevel and DiD studies (own configuration) groups.append(("designs", designs)) + lisa = [] if args.studies is None else [s for s in args.studies if s in LISA_STUDIES] + if lisa: # Round 16 LISA validation studies: on demand only (info rows) + groups.append(("lisa", lisa)) + # Round 16 workstream A (info only): 10-20 min each in the full configuration, + # so they run only when named with --studies (docs/audits/CHECK_MAP.md) + spreg16 = [s for s in SPREG_VALIDATION_STUDIES if args.studies is not None and s in args.studies] + if spreg16: + groups.append(("spreg_validation", spreg16)) frames = _run_groups(groups, args.fast, jobs, lambda m: print(m, flush=True)) frames = [f for f in frames if f is not None and len(f)] df = _pd.concat(frames, ignore_index=True) if frames else _pd.DataFrame( diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..725490d --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,12 @@ +# SPDX-License-Identifier: AGPL-3.0-or-later +"""Offline test runs: no real waits in census.http (fake sessions answer instantly). + +Live runs (CENSUS_LIVE_TESTS=1) keep the polite defaults: pacing, backoff and +the delayed firewall-page retry. Values already set in the environment win. +""" + +import os + +if not os.environ.get("CENSUS_LIVE_TESTS"): + for _name in ("CENSUS_WWW2_MIN_INTERVAL", "CENSUS_WWW2_BACKOFF_BASE", "CENSUS_WWW2_REJECTED_RETRY_DELAY"): + os.environ.setdefault(_name, "0") diff --git a/tests/test_http.py b/tests/test_http.py new file mode 100644 index 0000000..0e6cdf6 --- /dev/null +++ b/tests/test_http.py @@ -0,0 +1,272 @@ +# SPDX-License-Identifier: AGPL-3.0-or-later +"""census.http: polite census.gov downloads (retries, pacing, firewall page, manual files). + +All offline: fake sessions, a recorded sleep, and a mock clock; nothing waits. +""" + +import hashlib +import json +import os +import shutil +import tempfile +import unittest +import zipfile +from unittest import mock + +import requests + +from census import boundaries as bd +from census import http as H +from census import replicate as rp + +FIXTURES = os.path.join(os.path.dirname(__file__), "fixtures", "vre") +F2023 = os.path.join(FIXTURES, "B19001_040_2023_subset.csv.zip") + +# The page census.gov's firewall serves (HTTP 200, text/html, 247 bytes) for the 2019 Michigan tract +# B01003 VRE file, with the support ID replaced. +REJECTED_PAGE = ( + b"Request RejectedThe requested URL was rejected. " + b"Please consult with your administrator.

Your support ID is: 1234567890123456789

" + b"[Go Back]" +) + + +class FakeResponse: + def __init__(self, content=b"ok", status=200, headers=None): + self.content = content + self.status_code = status + self.headers = {"Content-Type": "application/zip", **(headers or {})} + + +class FakeSession: + """Answers each GET with the next item; an exception item is raised.""" + + def __init__(self, items): + self.items = list(items) + self.calls = [] + + def get(self, url, headers=None, timeout=None, **kwargs): + self.calls.append({"url": url, "headers": dict(headers or {}), "timeout": timeout, **kwargs}) + item = self.items.pop(0) + if isinstance(item, BaseException): + raise item + return item + + +class HttpTestCase(unittest.TestCase): + """Records sleeps instead of sleeping; jitter fixed at 1; default env values unless a test sets them.""" + + env = {"CENSUS_WWW2_MIN_INTERVAL": "0", "CENSUS_WWW2_BACKOFF_BASE": "2", + "CENSUS_WWW2_ATTEMPTS": "4", "CENSUS_WWW2_REJECTED_RETRY_DELAY": "0"} + + def setUp(self): + self.sleeps = [] + self.now = [1000.0] + patches = [ + mock.patch.dict(os.environ, self.env), + mock.patch.object(H, "_sleep", self._sleep), + mock.patch.object(H, "_monotonic", lambda: self.now[0]), + mock.patch.object(H, "_jitter", lambda: 1.0), + mock.patch.dict(H._last_request, clear=True), + ] + for p in patches: + p.start() + self.addCleanup(p.stop) + self.tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.tmp, True) + + def _sleep(self, secs): + self.sleeps.append(secs) + self.now[0] += secs + + +class TestBackoff(HttpTestCase): + def test_exponential_sequence_then_success(self): + s = FakeSession([FakeResponse(status=503), FakeResponse(status=502), FakeResponse(status=500), + FakeResponse(b"data")]) + resp = H.polite_get("https://www2.census.gov/x", user_agent="ua/1", session=s) + self.assertEqual(resp.content, b"data") + self.assertEqual(self.sleeps, [2, 4, 8]) + self.assertEqual(len(s.calls), 4) + + def test_cap_and_jitter(self): + with mock.patch.dict(os.environ, {"CENSUS_WWW2_ATTEMPTS": "8"}): + s = FakeSession([FakeResponse(status=503)] * 7 + [FakeResponse()]) + H.polite_get("https://www2.census.gov/x", user_agent="ua/1", session=s) + self.assertEqual(self.sleeps, [2, 4, 8, 16, 32, 60, 60]) + with mock.patch.object(H, "_jitter", lambda: 0.5): + self.assertEqual(H.backoff_delay(0), 1.0) + + def test_exhausted_status_returned_to_caller(self): + s = FakeSession([FakeResponse(status=503)] * 4) + self.assertEqual(H.polite_get("https://www2.census.gov/x", user_agent="ua/1", session=s).status_code, 503) + self.assertEqual(len(s.calls), 4) + self.assertEqual(len(self.sleeps), 3) + # The module's own error text is kept. + s = FakeSession([FakeResponse(status=503)] * 4) + with self.assertRaisesRegex(rp.VREDownloadError, "HTTP 503"): + rp._download("https://www2.census.gov/x", session=s) + + def test_not_retried(self): + for status in (404, 403, 400): + s = FakeSession([FakeResponse(status=status)]) + self.assertEqual(H.polite_get("https://www2.census.gov/x", user_agent="ua", session=s).status_code, + status) + self.assertEqual(self.sleeps, []) + + def test_connection_errors_and_timeouts(self): + s = FakeSession([requests.ConnectionError("reset"), requests.Timeout("slow"), FakeResponse(b"d")]) + self.assertEqual(H.polite_get("https://www2.census.gov/x", user_agent="ua", session=s).content, b"d") + self.assertEqual(self.sleeps, [2, 4]) + s = FakeSession([requests.ConnectionError("down")] * 4) + with self.assertRaisesRegex(rp.VREDownloadError, "after 4 attempt"): + rp._download("https://www2.census.gov/x", session=s) + self.assertEqual(len(s.calls), 4) + + def test_honest_user_agent(self): + s = FakeSession([FakeResponse()]) + rp._download("https://www2.census.gov/x", session=s) + ua = s.calls[0]["headers"]["User-Agent"] + self.assertEqual(ua, rp.USER_AGENT) + self.assertNotIn("Mozilla", ua) + + +class TestRetryAfter(HttpTestCase): + def test_seconds_header_is_honoured(self): + s = FakeSession([FakeResponse(status=429, headers={"Retry-After": "7"}), FakeResponse()]) + H.polite_get("https://www2.census.gov/x", user_agent="ua", session=s) + self.assertEqual(self.sleeps, [7.0]) + + def test_parse(self): + self.assertEqual(H.retry_after_seconds("12"), 12.0) + self.assertEqual(H.retry_after_seconds("100000"), H.RETRY_AFTER_CAP) + self.assertIsNone(H.retry_after_seconds(None)) + self.assertIsNone(H.retry_after_seconds("soon")) + self.assertEqual(H.retry_after_seconds("Wed, 21 Oct 2015 07:28:00 GMT"), 0.0) # in the past + + +class TestRejectedPage(HttpTestCase): + env = {**HttpTestCase.env, "CENSUS_WWW2_REJECTED_RETRY_DELAY": "30"} + + def test_support_id(self): + self.assertEqual(H.rejection_support_id(REJECTED_PAGE), "1234567890123456789") + self.assertEqual(H.rejection_support_id(b"Request Rejected"), "") + self.assertIsNone(H.rejection_support_id(b"PK\x03\x04 data")) + + def test_one_delayed_retry_then_clear_error(self): + url = rp.vre_url("B01003", 2019, "tract", "26") + page = FakeResponse(REJECTED_PAGE, headers={"Content-Type": "text/html"}) + s = FakeSession([page, page, FakeResponse(b"never reached")]) + with self.assertRaises(rp.VREDownloadError) as ctx: + rp.fetch_vre("B01003", 2019, "tract", "26", session=s, cache_dir=self.tmp, check_moe=False) + exc = ctx.exception + path = os.path.join(self.tmp, "140", "B01003_26.csv.zip") + self.assertEqual(len(s.calls), 2) # never loops + self.assertEqual(self.sleeps, [30.0]) + self.assertEqual(exc.support_id, "1234567890123456789") + self.assertEqual(exc.cache_path, path) + self.assertTrue(exc.rejected) + msg = str(exc) + for part in ("1234567890123456789", url, path, "browser", "persistent", "manual download"): + self.assertIn(part, msg) + self.assertFalse(os.path.exists(path)) + + def test_retry_can_succeed(self): + with open(F2023, "rb") as f: + raw = f.read() + page = FakeResponse(REJECTED_PAGE, headers={"Content-Type": "text/html"}) + s = FakeSession([page, FakeResponse(raw)]) + frame = rp.fetch_vre("B19001", 2023, "state", session=s, cache_dir=self.tmp) + self.assertEqual(len(frame), 85) + self.assertEqual(frame.attrs["acquisition"], "download") + + def test_other_modules_raise_their_own_error(self): + page = FakeResponse(REJECTED_PAGE, headers={"Content-Type": "text/html"}) + with self.assertRaises(bd.BoundaryError) as ctx: + bd._download("https://www2.census.gov/geo/tiger/x.zip", session=FakeSession([page, page]), + cache_path="/c/x.zip") + self.assertIn("/c/x.zip", str(ctx.exception)) + + +class TestManualFile(HttpTestCase): + def test_vre_manual_file_is_adopted(self): + with open(F2023, "rb") as f: + raw = f.read() + url = rp.vre_url("B19001", 2023, "state") + path = os.path.join(self.tmp, "040", "B19001.csv.zip") + os.makedirs(os.path.dirname(path)) + with open(path, "wb") as f: + f.write(raw) + s = FakeSession([]) # any request would fail + frame = rp.fetch_vre("B19001", 2023, "state", session=s, cache_dir=self.tmp) + self.assertEqual(s.calls, []) + self.assertEqual(len(frame), 85) + self.assertEqual(frame.attrs["acquisition"], H.MANUAL_SOURCE) + self.assertEqual(frame.attrs["sha256"], hashlib.sha256(raw).hexdigest()) + with open(path + ".json", encoding="utf-8") as f: + meta = json.load(f) + self.assertEqual(meta["source"], "manual download") + self.assertEqual(meta["source_url"], url) + self.assertEqual(meta["sha256"], hashlib.sha256(raw).hexdigest()) + # Later reads use the sidecar like any download. + frame2 = rp.fetch_vre("B19001", 2023, "state", session=s, cache_dir=self.tmp) + self.assertEqual(frame2.attrs["acquisition"], H.MANUAL_SOURCE) + + def test_saved_html_page_is_refused(self): + path = os.path.join(self.tmp, "040", "B19001.csv.zip") + os.makedirs(os.path.dirname(path)) + with open(path, "wb") as f: + f.write(REJECTED_PAGE) + with self.assertRaisesRegex(rp.VREDownloadError, "Manually placed file"): + rp.fetch_vre("B19001", 2023, "state", session=FakeSession([]), cache_dir=self.tmp) + self.assertFalse(os.path.exists(path + ".json")) + + def test_boundary_manual_zip(self): + path = os.path.join(self.tmp, "cb.zip") + with zipfile.ZipFile(path, "w") as zf: + zf.writestr("a.txt", "x") + meta = bd._read_cached(path, "https://www2.census.gov/geo/tiger/cb.zip") + self.assertEqual(meta["source"], H.MANUAL_SOURCE) + bad = os.path.join(self.tmp, "bad.zip") + with open(bad, "wb") as f: + f.write(b"") + with self.assertRaises(bd.BoundaryError): + bd._read_cached(bad, "https://www2.census.gov/geo/tiger/bad.zip") + + def test_no_adoption_without_url_or_with_sidecar(self): + path = os.path.join(self.tmp, "f.bin") + with open(path, "wb") as f: + f.write(b"x") + self.assertIsNone(rp._read_cached(path)) + self.assertFalse(os.path.exists(path + ".json")) + with open(path + ".json", "w") as f: + json.dump({"sha256": "other"}, f) + self.assertIsNone(H.adopt_manual_file(path, "https://www2.census.gov/f.bin")) + + +class TestPacing(HttpTestCase): + env = {**HttpTestCase.env, "CENSUS_WWW2_MIN_INTERVAL": "1"} + + def test_min_interval_per_host(self): + s = FakeSession([FakeResponse()] * 4) + H.polite_get("https://www2.census.gov/a", user_agent="ua", session=s) + self.assertEqual(self.sleeps, []) + self.now[0] += 0.25 + H.polite_get("https://www2.census.gov/b", user_agent="ua", session=s) + self.assertEqual(self.sleeps, [0.75]) + H.polite_get("https://www.census.gov/c", user_agent="ua", session=s) # other host: no wait + self.assertEqual(self.sleeps, [0.75]) + self.now[0] += 5 + H.polite_get("https://www2.census.gov/d", user_agent="ua", session=s) + self.assertEqual(self.sleeps, [0.75]) + + def test_configurable(self): + with mock.patch.dict(os.environ, {"CENSUS_WWW2_MIN_INTERVAL": "3"}): + s = FakeSession([FakeResponse()] * 2) + H.polite_get("https://www2.census.gov/a", user_agent="ua", session=s) + H.polite_get("https://www2.census.gov/a", user_agent="ua", session=s) + self.assertEqual(self.sleeps, [3.0]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_lisa_validation.py b/tests/test_lisa_validation.py new file mode 100644 index 0000000..873bd91 --- /dev/null +++ b/tests/test_lisa_validation.py @@ -0,0 +1,175 @@ +# SPDX-License-Identifier: AGPL-3.0-or-later +"""PLAN_ROUND_16 workstream B: building blocks and registration of the LISA validation studies.""" + +import importlib.util +import math +import os +import unittest + +import numpy as np +import pandas as pd +from scipy import stats as sps + +from census import lisa_validation as lv +from census.simulation import FAST, STUDIES, SPATIAL_STUDIES, group_config, lattice_weights +from census.variance import REPLICATE_FACTOR + +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + + +def _script(): + spec = importlib.util.spec_from_file_location("inference_validation_lv", os.path.join(ROOT, "scripts", + "inference_validation.py")) + iv = importlib.util.module_from_spec(spec) + spec.loader.exec_module(iv) + return iv + + +class TestErrorCorrelation(unittest.TestCase): + W = lattice_weights(lv.LV_SIDE).full()[0] + A = lv.rook_adjacency(lv.LV_SIDE) + + def test_rook_adjacency_matches_lattice_weights(self): + np.testing.assert_array_equal(self.A, (self.W > 0).astype(float)) + + def test_neighbour_correlation_hits_each_target(self): + for r1 in lv.LV_NEIGHBOUR_CORRS: + st = lv.error_correlation(self.W, self.A, r1) + self.assertAlmostEqual(st["r1"], r1, places=6) + np.testing.assert_allclose(np.diag(st["R"]), 1.0) + np.testing.assert_allclose(st["L"] @ st["L"].T, st["R"], atol=1e-10) + self.assertEqual(lv.error_correlation(self.W, self.A, 0.0)["lam"], 0.0) + + def test_negative_neighbour_correlation_is_a_checkerboard(self): + st = lv.error_correlation(self.W, self.A, -0.2) + self.assertLess(st["lam"], 0) + self.assertGreater(st["r2"], 0) # second-order neighbours positively correlated + + def test_replicates_follow_acs_scaling_and_the_error_correlation(self): + st = lv.error_correlation(self.W, self.A, 0.5) + rng = np.random.default_rng(3) + se = rng.uniform(2, 9, self.W.shape[0]) + dev = lv.correlated_replicates(se, st["L"], rng.standard_normal((len(se), 80))) + np.testing.assert_allclose(np.sqrt(REPLICATE_FACTOR * (dev ** 2).sum(axis=1)), se, rtol=1e-12) + # replicate covariance 4/80 sum d d' reproduces the neighbour correlation on average + C = REPLICATE_FACTOR * dev @ dev.T + R = C / np.outer(se, se) + self.assertAlmostEqual(lv.mean_neighbour_correlation(R, self.A), 0.5, delta=0.05) + + +class TestWong(unittest.TestCase): + # a 1-D chain 0-1-2-3 + NB = [[1], [0, 2], [1, 3], [2]] + + def test_join_rule_is_p_at_least_cl_as_in_the_worked_example(self): + # Wong (2021) sec. 5.1: Louisiana joins Mississippi's hot spot at CL 0.648 (t-test p = 0.65) + y = np.array([35.5, 35.5 - sps.norm.isf(0.65 / 2) * math.sqrt(2), 0.0, 0.0]) + se = np.ones(4) + lab, seeds, seed_of = lv.wong_hot_cold_spots(y, se, self.NB, high=35.0, low=-100.0, confidence=0.648) + self.assertEqual(list(lab[:2]), ["hot", "hot"]) + self.assertEqual(list(seeds), [True, False, False, False]) + self.assertEqual(list(seed_of), [0, 0, -1, -1]) + lab, _, _ = lv.wong_hot_cold_spots(y, se, self.NB, high=35.0, low=-100.0, confidence=0.66) + self.assertEqual(list(lab[:2]), ["hot", ""]) # a higher CL needs a larger p-value to join + + def test_growth_compares_with_the_seed_not_the_member(self): + y = np.array([10.0, 9.7, 9.4, 8.4]) # 2 is close to 1 (p 0.67) but not to the seed 0 (p 0.40) + lab, _, _ = lv.wong_hot_cold_spots(y, np.full(4, 0.5), self.NB, high=9.9, low=-1.0, confidence=0.5) + self.assertEqual(list(lab), ["hot", "hot", "", ""]) + + def test_cold_spots_and_no_seeds(self): + y = np.array([1.0, 1.1, 5.0, 5.0]) + lab, _, seed_of = lv.wong_hot_cold_spots(y, np.full(4, 0.5), self.NB, high=9.0, low=2.0, confidence=0.5) + self.assertEqual(list(lab), ["cold", "cold", "", ""]) + self.assertEqual(list(seed_of), [0, 1, -1, -1]) # both are seeds + lab, seeds, _ = lv.wong_hot_cold_spots(y, np.full(4, 0.5), self.NB, high=9.0, low=0.0, confidence=0.5) + self.assertFalse(seeds.any() or (lab != "").any()) + + +class TestWongMetrics(unittest.TestCase): + def test_scored_against_latent_elevation(self): + theta = np.array([12.0, 11.0, 4.0, 0.0, 5.0]) + labels = np.array(["hot", "hot", "hot", "", ""]) + seeds = np.array([True, False, False, False, False]) + seed_of = np.array([0, 0, 0, -1, -1]) + m = lv.wong_metrics(labels, seeds, seed_of, theta, high=10.0, low=-5.0, truth=np.zeros(5, bool)) + self.assertEqual(m["seed_precision"], 1.0) + self.assertEqual(m["exceed_recall"], 1.0) # both areas with theta > 10 flagged hot + self.assertEqual(m["growth_per_seed"], 2.0) + self.assertEqual(m["member_beyond_share"], 0.5) # area 1 is beyond 10, area 2 is not + self.assertEqual(m["member_near_share"], 0.5) # |11 - 12| <= LV_WONG_NEAR, |4 - 12| is not + self.assertNotIn("fpr", m) + self.assertNotIn("power", m) + + +class TestMetrics(unittest.TestCase): + NB = [[1], [0, 2], [1, 3], [2]] + + def test_null_map(self): + flag = np.array([True, True, False, True]) + lab = np.array(["HH", "HH", "LL", "LH"]) + m = lv.map_metrics(flag, lab, np.zeros(4, bool), "HH", ("HH", "LL"), self.NB) + self.assertAlmostEqual(m["fpr"], 0.75) + self.assertEqual(m["fdr"], 1.0) + self.assertEqual(m["false_cluster_map_k2"], 1.0) # areas 0-1 (HH) form a component of size 2 + self.assertEqual(m["false_cluster_map_k4"], 0.0) + self.assertTrue(math.isnan(m["power"])) + + def test_cluster_map(self): + truth = np.array([False, True, True, False]) + flag = np.array([False, True, True, True]) + lab = np.array(["HH", "HH", "LL", "HH"]) + m = lv.map_metrics(flag, lab, truth, "HH", ("HH", "LL"), self.NB, halo=np.array([True, False, False, True])) + self.assertAlmostEqual(m["fpr"], 0.5) + self.assertAlmostEqual(m["fdr"], 1 / 3) + self.assertAlmostEqual(m["power"], 0.5) # only area 1 is flagged with the correct label + self.assertAlmostEqual(m["jaccard"], 1 / 3) # {1} vs {1, 2} plus false area 3: |{1}| / |{1, 2, 3}| + self.assertEqual(m["false_cluster_map_k2"], 0.0) # the null area 3 is a single-area component + self.assertEqual(m["halo_share"], 1.0) + + def test_misspecified_weights_keep_the_rook_degrees(self): + ws = lv.analysis_weights(6, [f"{i:05d}" for i in range(36)], seed=1) + rook, mis = ws["rook"], ws["misspecified"] + self.assertEqual(sorted(rook.cardinalities.values()), sorted(mis.cardinalities.values())) + for i, nb in mis.neighbors.items(): + for j in nb: + self.assertIn(i, mis.neighbors[j]) + self.assertNotEqual(rook.neighbors, mis.neighbors) + self.assertEqual(set(ws), {"rook", "queen", f"knn{lv.LV_KNN}", "misspecified"}) + + +class TestRegistration(unittest.TestCase): + """Info studies run on demand only (--studies NAME): not in the default report, never required.""" + + def test_registered_on_demand_only(self): + iv = _script() + for name in lv.LISA_STUDIES: + self.assertIn(name, STUDIES) + self.assertNotIn(name, SPATIAL_STUDIES) + self.assertNotIn(name, iv.REQUIRED_STUDIES) + self.assertNotIn(name, iv.DEFAULT_DESIGN_STUDIES) + self.assertGreater(lv.LISA_REPS_FULL[name], 0) + cfg = group_config("lisa", True) + self.assertEqual({k for k, v in cfg.reps.items() if v > 0}, set(lv.LISA_STUDIES)) + + def test_tiny_runs_are_info_only_and_seeded(self): + # a 6 x 6 lattice keeps the dense 144-area algebra out of the unit tests + a = pd.DataFrame(lv.study_lisa_validation_clusters(FAST, 1, side=6)) + b = pd.DataFrame(lv.study_lisa_validation_clusters(FAST, 1, side=6)) + self.assertEqual(a.to_csv(index=False), b.to_csv(index=False)) + self.assertEqual(set(a["status"]), {"info"}) + self.assertEqual(set(a["lv_method"]), {"lisa", "lisa_stable", "stable_only", "lisa_eblup", "wong_cl0.5", + "wong_cl0.75"}) + self.assertEqual(set(a["scenario"]), {"no_sampling_error", "cluster_independent", "cluster_correlated"}) + self.assertEqual(set(a.loc[a["lv_method"] == "lisa", "weights"]), + {"rook", "queen", f"knn{lv.LV_KNN}", "misspecified"}) + iv = _script() + self.assertTrue(iv.gate(a.assign(study="x"), ())[0]) + null = pd.DataFrame(lv.study_lisa_validation_null(FAST, 1, side=6)) + self.assertEqual(set(null["scenario"]), {"no_sampling_error", "independent_homogeneous", + "heterogeneous_independent", "correlated"}) + self.assertTrue(null.loc[null["metric"] == "power", "value"].empty) # no true clusters + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_replicate_covariance.py b/tests/test_replicate_covariance.py new file mode 100644 index 0000000..ed4223b --- /dev/null +++ b/tests/test_replicate_covariance.py @@ -0,0 +1,269 @@ +# SPDX-License-Identifier: AGPL-3.0-or-later +"""Tests for census.replicate_covariance (PLAN_ROUND_16.md workstream C). + +Synthetic replicates with a known covariance: under the 4/80 SDR scaling the +deviation vectors d_r = X_r - X are drawn iid N(0, Sigma / 4), so +E[4/80 sum_r d_r d_r^T] = Sigma. +""" + +import math + +import numpy as np +import pytest +from scipy import sparse + +from census import replicate_covariance as rc +from census.variance import REPLICATE_COUNT, replicate_variance + + +def _grid_adjacency(side): + n = side * side + rows, cols = [], [] + for r in range(side): + for c in range(side): + k = r * side + c + if c + 1 < side: + rows += [k, k + 1] + cols += [k + 1, k] + if r + 1 < side: + rows += [k, k + side] + cols += [k + side, k] + return sparse.csr_matrix((np.ones(len(rows)), (rows, cols)), shape=(n, n)) + + +def _draw(sigma, rng, n_rep=REPLICATE_COUNT): + """(n, R) deviations with E[4/R D D^T] = sigma, plus estimates and replicates.""" + chol = np.linalg.cholesky(sigma) + d = (chol @ rng.standard_normal((sigma.shape[0], n_rep))) / 2.0 + est = rng.uniform(100, 200, size=sigma.shape[0]) + return d, est, est[:, None] + d + + +def _structured_sigma(side, rho1, rho2, sd=None): + a = _grid_adjacency(side) + orders = rc.contiguity_orders(a, max_order=2).toarray() + r = np.eye(side * side) + rho1 * (orders == 1) + rho2 * (orders == 2) + assert np.linalg.eigvalsh(r).min() > 0 + s = np.ones(side * side) if sd is None else sd + return r * np.outer(s, s), orders + + +def test_covariance_diagonal_is_replicate_variance_and_rank_limited(): + rng = np.random.default_rng(1) + est = rng.uniform(10, 50, 120) + reps = est[:, None] + rng.normal(size=(120, REPLICATE_COUNT)) + cov = rc.replicate_covariance(est, reps) + np.testing.assert_allclose(np.diag(cov), replicate_variance(est, reps)) + rank = rc.rank_report(reps - est[:, None]) + assert rank["rank"] == REPLICATE_COUNT == rank["max_rank"] + assert np.linalg.matrix_rank(cov) == REPLICATE_COUNT + corr = rc.replicate_correlation(est, reps) + np.testing.assert_allclose(np.diag(corr), 1.0) + + +def test_covariance_is_unbiased_for_known_sigma(): + rng = np.random.default_rng(2) + sigma = np.array([[4.0, 1.2, -0.8], [1.2, 1.0, 0.0], [-0.8, 0.0, 2.25]]) + acc = np.zeros_like(sigma) + n_sim = 2000 + for _ in range(n_sim): + _, est, reps = _draw(sigma, rng) + acc += rc.replicate_covariance(est, reps) + mean = acc / n_sim + # SE of each mean entry ~ sqrt((s_ii s_jj + s_ij^2) / 80 / n_sim) <= 0.01 + np.testing.assert_allclose(mean, sigma, atol=0.05) + + +def test_zero_variance_area_gives_nan_correlation(): + est = np.array([1.0, 2.0, 3.0]) + reps = np.tile(est[:, None], (1, REPLICATE_COUNT)).astype(float) + reps[0] += np.linspace(-1, 1, REPLICATE_COUNT) + reps[1] += np.cos(np.arange(REPLICATE_COUNT)) + corr = rc.replicate_correlation(est, reps) + assert np.isnan(corr[2]).all() and np.isnan(corr[:, 2]).all() + assert np.isfinite(corr[0, 1]) + d = rc.replicate_deviations(est, reps) + assert np.isnan(rc.pair_correlations(d, [0], [2]))[0] + with pytest.raises(ValueError): + rc.replicate_deviations(est, reps[:2]) + with pytest.raises(ValueError): + rc.replicate_deviations([np.nan, 1, 2], reps) + + +def test_pair_correlations_match_matrix(): + rng = np.random.default_rng(3) + sigma, _ = _structured_sigma(5, 0.2, 0.05) + d, est, reps = _draw(sigma, rng) + full = rc.replicate_correlation(est, reps) + i, j = np.triu_indices(25, 1) + np.testing.assert_allclose(rc.pair_correlations(d, i, j), full[i, j], atol=1e-12) + + +def test_null_noise_floor_matches_simulation(): + null = rc.null_correlation(80, alpha=0.05) + assert null["sd"] == pytest.approx(1 / math.sqrt(80)) + assert null["critical_abs_r"] == pytest.approx(0.2194, abs=0.002) + rng = np.random.default_rng(4) + d = rng.normal(size=(4000, REPLICATE_COUNT)) * rng.uniform(0.5, 5, size=(4000, 1)) + i, j = np.arange(0, 4000, 2), np.arange(1, 4000, 2) + r = rc.pair_correlations(d, i, j) + assert np.std(r) == pytest.approx(null["sd"], rel=0.06) + assert np.mean(np.abs(r) > null["critical_abs_r"]) == pytest.approx(0.05, abs=0.015) + perm = rc.permutation_null(d, i, j, n_perm=10, seed=1) + assert perm["sd"] == pytest.approx(null["sd"], rel=0.05) + assert perm["critical_abs_r"] == pytest.approx(null["critical_abs_r"], abs=0.015) + + +def test_permutation_null_destroys_real_correlation(): + rng = np.random.default_rng(5) + base = rng.normal(size=(1000, REPLICATE_COUNT)) + d = np.vstack([base, base + 0.3 * rng.normal(size=base.shape)]) + i, j = np.arange(1000), np.arange(1000, 2000) + assert np.mean(rc.pair_correlations(d, i, j)) > 0.9 + perm = rc.permutation_null(d, i, j, n_perm=5) + assert abs(perm["mean"]) < 0.01 + + +def test_contiguity_orders_on_grid(): + a = _grid_adjacency(4) + orders = rc.contiguity_orders(a, max_order=3).toarray() + # node 0 (corner): 1 and 4 are order 1; 2, 5, 8 order 2; 3, 6, 9, 12 order 3 + assert orders[0, 1] == orders[0, 4] == 1 + assert orders[0, 2] == orders[0, 5] == orders[0, 8] == 2 + assert {orders[0, k] for k in (3, 6, 9, 12)} == {3} + assert orders[0, 15] == 0 and orders[0, 0] == 0 + assert (orders == orders.T).all() + np.testing.assert_array_equal(rc.pair_orders(rc.contiguity_orders(a, 3), [0, 0, 0], [1, 5, 15]), [1, 2, 0]) + + +def test_class_summaries_recover_known_structure_and_se_is_calibrated(): + sigma, orders = _structured_sigma(12, 0.25, 0.08, sd=np.linspace(1, 20, 144)) + i, j = np.nonzero(np.triu(orders > 0)) + labels = orders[i, j] + groups = (np.arange(144) // 12) // 3 # four blocks of three grid rows + means, ses = {1: [], 2: []}, {1: [], 2: []} + rng = np.random.default_rng(6) + for rep in range(40): + d, _, _ = _draw(sigma, rng) + tab = rc.summarize_pair_correlations(d, i, j, labels, groups=groups, n_boot=60, seed=rep) + for k in (1, 2): + row = tab[tab["class"] == k].iloc[0] + means[k].append(row["mean"]) + ses[k].append(row["se_boot"]) + for k, truth in ((1, 0.25), (2, 0.08)): + assert np.mean(means[k]) == pytest.approx(truth, abs=0.01) + # the replicate bootstrap SE matches the Monte Carlo spread of the class mean + assert np.mean(ses[k]) == pytest.approx(np.std(means[k], ddof=1), rel=0.35) + assert {"se_jackknife", "excess_sd", "share_beyond_critical", "ci_low", "ci_high"} <= set(tab.columns) + + +def test_split_half_reliability_separates_noise_from_signal(): + rng = np.random.default_rng(7) + n_pairs = 600 + # pairs with true correlation +0.7 or -0.7 (signal) vs 0 (noise only) + rho = np.where(np.arange(n_pairs) % 2 == 0, 0.7, -0.7) + x = rng.normal(size=(n_pairs, REPLICATE_COUNT)) + y = rho[:, None] * x + np.sqrt(1 - rho[:, None] ** 2) * rng.normal(size=x.shape) + d = np.vstack([x, y]) + i, j = np.arange(n_pairs), np.arange(n_pairs, 2 * n_pairs) + signal = rc.split_half_agreement(d, i, j, split="odd_even").iloc[0] + assert signal["pair_reliability_full"] > 0.9 + noise = rc.split_half_agreement(rng.normal(size=d.shape), i, j, split="random", seed=1).iloc[0] + assert abs(noise["pair_reliability_half"]) < 0.12 + with pytest.raises(ValueError): + rc.split_half_agreement(d, i, j, split="bad") + + +def test_aggregation_ratio_detects_controlled_totals(): + rng = np.random.default_rng(8) + m, k = 10, 30 + groups = np.repeat(np.arange(k), m) + free = rng.normal(size=(m * k, REPLICATE_COUNT)) + frame, summary = rc.aggregation_variance_ratio(free, groups, n_boot=50) + assert summary["pooled_ratio"] == pytest.approx(1.0, abs=0.1) + assert abs(summary["pooled_implied_mean_corr"]) < 0.02 + # group totals fixed (deviations sum to zero within each group): ratio 0, mean corr -1/(m-1) + ctrl = free - np.repeat(free.reshape(k, m, -1).mean(axis=1), m, axis=0) + frame, summary = rc.aggregation_variance_ratio(ctrl, groups, n_boot=50) + assert summary["pooled_ratio"] == pytest.approx(0.0, abs=1e-12) + assert summary["pooled_implied_mean_corr"] == pytest.approx(-1 / (m - 1), abs=0.01) + assert len(frame) == k and frame["ratio"].max() < 1e-12 + + +def test_ledoit_wolf_matches_direct_formula(): + rng = np.random.default_rng(9) + d = rng.normal(size=(15, REPLICATE_COUNT)) * rng.uniform(1, 3, size=(15, 1)) + out = rc.ledoit_wolf_intensity(d) + x = 2.0 * d.T + z = x / np.sqrt(np.mean(x * x, axis=0)) + t, n = z.shape + s = z.T @ z / t + delta2 = np.sum((s - np.eye(n)) ** 2) / n + beta2 = sum(np.sum((np.outer(zr, zr) - s) ** 2) for zr in z) / t ** 2 / n + assert out["intensity"] == pytest.approx(min(1.0, min(beta2, delta2) / delta2)) + assert out["intensity"] > 0.6 # independent areas: shrink nearly all the way + sigma, _ = _structured_sigma(5, 0.35, 0.1) + dd, _, _ = _draw(sigma, rng) + assert rc.ledoit_wolf_intensity(dd)["intensity"] < out["intensity"] + + +def test_lowrank_loglik_matches_dense(): + rng = np.random.default_rng(10) + z_tr = rng.normal(size=(40, 30)) + z_te = rng.normal(size=(40, 30)) + a = 0.7 + r = a * np.eye(30) + (1 - a) * z_tr.T @ z_tr / 40 + dense, _ = rc._loglik_dense(r, z_te) + assert rc._loglik_lowrank(a, z_tr, z_te) == pytest.approx(dense, rel=1e-9) + + +def test_holdout_comparison_prefers_true_structure(): + rng = np.random.default_rng(11) + sigma, orders = _structured_sigma(10, 0.3, 0.1) + d, _, _ = _draw(sigma, rng) + cls = np.where(orders == 1, 0, np.where(orders == 2, 1, -1)) + np.fill_diagonal(cls, -1) + out = rc.holdout_model_comparison(d, {"order12": cls}) + assert out["gain_nats_per_replicate"]["order12"] > 0.5 + rho1 = np.mean([p[0] for p in out["fitted_rho"]["order12"]]) + assert rho1 == pytest.approx(0.3, abs=0.05) + # no structure: the structured model gains nothing material + d0 = rng.normal(size=(100, REPLICATE_COUNT)) + out0 = rc.holdout_model_comparison(d0, {"order12": cls}) + assert out0["gain_nats_per_replicate"]["order12"] < 0.2 + assert out0["gain_nats_per_replicate"]["order12"] < out["gain_nats_per_replicate"]["order12"] + + +def _constrained(n, k, rng): + """iid deviations made orthogonal to k random replicate-space directions (a design constraint).""" + v = np.linalg.qr(rng.normal(size=(REPLICATE_COUNT, k)))[0].T # (k, R) orthonormal + g = rng.normal(size=(n, REPLICATE_COUNT)) + return g - (g @ v.T) @ v, v + + +def test_replicate_spectrum_detects_design_null_directions(): + rng = np.random.default_rng(12) + d, v = _constrained(1500, 3, rng) + spec = rc.replicate_spectrum(d) + assert spec["n_null"] == 3 and spec["effective_replicates"] == 77 + cosines = np.linalg.svd(spec["null_directions"] @ v.T, compute_uv=False) + assert cosines.min() > 0.999 + assert rc.replicate_spectrum(rng.normal(size=(1500, REPLICATE_COUNT)))["n_null"] == 0 + small = rc.replicate_spectrum(d[:100]) + assert small["n_null"] == 0 and "not detected" in small["note"] + # fewer effective replicates widen the null spread of a pairwise r to ~1/sqrt(R - k) + r = rc.pair_correlations(d, np.arange(0, 1500, 2), np.arange(1, 1500, 2)) + assert np.std(r) == pytest.approx(1 / math.sqrt(77), rel=0.08) + + +def test_holdout_removes_cross_half_dependence_from_design_directions(): + rng = np.random.default_rng(13) + d, v = _constrained(1200, 3, rng) + leaky = rc.holdout_model_comparison(d, {}, remove_directions=None) + fixed = rc.holdout_model_comparison(d, {}) + explicit = rc.holdout_model_comparison(d, {}, remove_directions=v) + # independent areas: only the cross-half leak lets Ledoit-Wolf beat independence + assert leaky["gain_nats_per_replicate"]["ledoit_wolf"] > 0 + assert fixed["gain_nats_per_replicate"]["ledoit_wolf"] < 0 + assert explicit["gain_nats_per_replicate"]["ledoit_wolf"] < 0 + assert fixed["directions_removed"] == 4 and explicit["directions_removed"] == 3 diff --git a/tests/test_replicate_live.py b/tests/test_replicate_live.py index 5502d94..04cf4f6 100644 --- a/tests/test_replicate_live.py +++ b/tests/test_replicate_live.py @@ -4,8 +4,8 @@ Skipped unless CENSUS_LIVE_TESTS=1. Downloads real Variance Replicate Estimate files from www2.census.gov into a temporary cache and checks that replicate MOEs reproduce the published MOEs (validation check F3) and that the frozen -fixture still matches the live file. Requests are spaced out, because -census.gov's firewall rejects rapid scripted requests. +fixture still matches the live file. Requests are spaced out as a courtesy +(census.http also paces, retries, and handles the firewall page). CENSUS_LIVE_TESTS=1 python -m pytest -q tests/test_replicate_live.py """ diff --git a/tests/test_review_fixes_r16.py b/tests/test_review_fixes_r16.py new file mode 100644 index 0000000..e542ca3 --- /dev/null +++ b/tests/test_review_fixes_r16.py @@ -0,0 +1,518 @@ +# SPDX-License-Identifier: AGPL-3.0-or-later +"""Regression tests for the Round 16 code-review findings (offline). + +1. The A1 studies' diagonal-Sigma rows call the shipped correct_outcome_error + estimator (census.spatial.error_model_known_variances), not me_sem_fit. +2. correct_outcome_error refuses frames above CORRECTION_MAX_N areas. +3. Manually placed / downloaded / cached PUMS and block relationship files are + validated; invalid ones get no sidecar and are moved aside (do not stick). +4. SE == 0 is refused; a singular V at the optimum raises RuntimeError. +5. Lag refits with |rho| >= 1 are failed refits; a main fit with |rho| >= 1 + reports undefined effects. +6. census.http paces per host without sleeping under the lock. +7. W eigenvalues are cached (eigvalsh for symmetric-pattern W); spreg impacts + are off in our fits; results unchanged. +8. diagnostics['effects_multipliers'] account for islands like the effects table. +9. Retryable streamed responses are closed before the retry. +10. scripts/acs_error_covariance: a zero ratio denominator in a replicate + invalidates the area for that variable. +""" + +import importlib.util +import io +import os +import shutil +import sys +import tempfile +import threading +import unittest +import warnings +import zipfile +from unittest import mock + +import numpy as np +import pandas as pd + +from census import http as H + +try: + import libpysal + import spreg # noqa: F401 + HAVE_PYSAL = True +except ImportError: # pragma: no cover + HAVE_PYSAL = False + +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +HTML_PAGE = b"ErrorNot found" + + +def setUpModule(): + import census.simulation # noqa: F401 (registers census.simulation_spreg) + + +def _a1(kind="queen", lam=0.3, cv=1.0): + from tests.test_spatial_lambda_correction import _a1_draw + return _a1_draw(kind, lam, cv) + + +def _frame(g, x, y, s): + from tests.test_spatial_lambda_correction import _frame as f + return f(g, x, y, s) + + +# --------------------------------------------------------------------------- +# 1, 2, 4: correct_outcome_error +# --------------------------------------------------------------------------- + + +@unittest.skipUnless(HAVE_PYSAL, "PySAL stack not installed") +class TestStudyUsesShippedEstimator(unittest.TestCase): + def test_diagonal_rows_call_the_tool_function(self): + from census import simulation_spreg as S, spatial + cfg, seed, g, w, W, _ = _a1("rook", 0.6) + calls = [] + real = spatial.error_model_known_variances + + def spy(*a, **k): + calls.append(a[4] if len(a) > 4 else k.get("method")) + return real(*a, **k) + + with mock.patch.object(spatial, "error_model_known_variances", spy), \ + mock.patch.object(S, "me_sem_fit", side_effect=AssertionError("diagonal rows must not use me_sem_fit")): + rows = S._attenuation_cell(cfg, 1, seed, "rook", 0.6, 1.0, None) + self.assertEqual(sorted(calls), ["ML", "REML"]) + self.assertTrue(all(r["fit_failures"] == 0 for r in rows)) + + def test_oracle_still_uses_dense_sigma(self): + from census import simulation_spreg as S + cfg, _, g, w, W, _ = _a1("rook", 0.0) + real = S.me_sem_fit + seen = [] + + def spy(y, X, W_, Sig, **k): + seen.append(np.count_nonzero(Sig - np.diag(np.diag(Sig))) > 0) + return real(y, X, W_, Sig, **k) + + with mock.patch.object(S, "me_sem_fit", spy): + S._attenuation_cell(cfg, 1, 7, "rook", 0.0, 1.0, 0.6) + self.assertEqual(seen, [True]) # only the oracle, with a dense Sigma + + def test_cell_equals_tool(self): + from census import simulation_spreg as S + from census.spatial import spatial_regression + cfg, seed, g, w, W, draws = _a1("knn", 0.3) + x, y, s = draws[0] + rows = S._attenuation_cell(cfg, 1, seed, "knn", 0.3, 1.0, None) + tab = spatial_regression("y ~ x", _frame(g, x, y, s), w, model="error", correct_outcome_error=True, + se_col="y_se").coefficients.set_index("term") + reml = next(r for r in rows if "REML" in r["estimator"] and r["term"] == "lambda") + beta = next(r for r in rows if "REML" in r["estimator"] and r["term"] == "x") + self.assertEqual(reml["mean_estimate"], tab.loc["lambda", "estimate"]) + self.assertEqual(beta["mean_estimate"], tab.loc["x", "estimate"]) + + +@unittest.skipUnless(HAVE_PYSAL, "PySAL stack not installed") +class TestCorrectionLimits(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.cfg, _, cls.g, cls.w, cls.W, draws = _a1("queen", 0.3) + cls.x, cls.y, cls.s = draws[0] + + def fit(self, fr, **kw): + from census.spatial import spatial_regression + return spatial_regression("y ~ x", fr, self.w, model="error", correct_outcome_error=True, se_col="y_se", **kw) + + def test_size_limit(self): + from census import spatial + self.assertEqual(spatial.CORRECTION_MAX_N, 2000) + self.assertIn("n=2025", spatial.CORRECTION_COST_NOTE) + with mock.patch.object(spatial, "CORRECTION_MAX_N", 100), \ + mock.patch.object(spatial, "error_model_known_variances", + side_effect=AssertionError("must refuse before fitting")): + with self.assertRaisesRegex(ValueError, r"limited to 100 areas \(got 144\).*county level"): + self.fit(_frame(self.g, self.x, self.y, self.s)) + + def test_zero_se_refused(self): + fr = _frame(self.g, self.x, self.y, self.s) + fr.loc[[3, 7], "y_se"] = 0.0 + with self.assertRaisesRegex(ValueError, r"is 0 in 2 area\(s\)"): + self.fit(fr) + + def test_singular_optimum_is_runtime_error(self): + from census import sae_spatial + from scipy import linalg as sla + + def singular(*a, **k): + raise sla.LinAlgError("2-th leading minor not positive definite") + + with mock.patch.object(sae_spatial, "_fit", singular): + with self.assertRaisesRegex(RuntimeError, "singular at the optimum.*not substituted"): + self.fit(_frame(self.g, self.x, self.y, self.s)) + + +# --------------------------------------------------------------------------- +# 5, 7, 8: lag model +# --------------------------------------------------------------------------- + + +def _lag_with_reps(seed=3, side=7, island=False): + from census.simulation import lattice_weights, make_replicates, replicate_long, sar_inverse + rng = np.random.default_rng(seed) + w = lattice_weights(side) + if island: + nb = dict(w.neighbors) + nb["island"] = [] + w = libpysal.weights.W(nb, id_order=list(w.id_order) + ["island"], silence_warnings=True) + w.transform = "r" + n = w.n + ids = list(w.id_order) + x = rng.uniform(0, 10, n) + y = sar_inverse(w.full()[0], 0.4) @ (5 + 2 * x + rng.normal(0, 2, n)) + reps = replicate_long(ids, ["y"], y[:, None], make_replicates(y[:, None], np.ones((n, 1)), rng)) + return w, pd.DataFrame({"GEOID": ids, "y": y, "x": x}), reps + + +@unittest.skipUnless(HAVE_PYSAL, "PySAL stack not installed") +class TestLagRhoBounds(unittest.TestCase): + def _patched(self, bad_calls): + from census import spatial + real = spatial._spreg_fit + count = [0] + + def fit(*a, **k): + r, b, se, sp = real(*a, **k) + i = count[0] + count[0] += 1 + if i in bad_calls: + b = b.copy() + b[-1] = 1.05 + sp = 1.05 + return r, b, se, sp + return mock.patch.object(spatial, "_spreg_fit", fit) + + def test_explosive_refits_are_failures(self): + from census.spatial import spatial_regression + w, df, reps = _lag_with_reps() + with self._patched({2, 5, 9}): # call 0 is the main fit + with self.assertRaisesRegex(RuntimeError, r"3 of 80 replicate refits failed"): + spatial_regression("y ~ x", df, w, model="lag", replicates=reps, target="conservative") + with self._patched({2, 5, 9}): + res = spatial_regression("y ~ x", df, w, model="lag", replicates=reps, target="conservative", + max_failed_replicates=5) + self.assertEqual(res.diagnostics["replicate_failures"], 3) + self.assertEqual(res.diagnostics["replicate_refits_rho_out_of_bounds"], 3) + self.assertIn("replicates_incomplete", res.diagnostics) + self.assertTrue(any("|rho| >= 1" in n for n in res.notes)) + # no NaN silently zeroed: every effect has a finite sampling SE from the 77 usable refits + self.assertTrue(np.isfinite(res.effects["se_sampling"]).all()) + self.assertTrue((res.effects["se_total"] > res.effects["se_model"]).all()) + + def test_explosive_main_fit(self): + from census.spatial import spatial_regression + w, df, _ = _lag_with_reps() + with self._patched({0}): + res = spatial_regression("y ~ x", df, w, model="lag") + self.assertTrue(res.diagnostics["effects_undefined"]) + self.assertTrue(res.notes[0].startswith("rho = 1.0500 is outside (-1, 1)")) + self.assertTrue(res.effects["estimate"].isna().all()) + self.assertTrue(res.effects["ci_lower"].isna().all()) + + +@unittest.skipUnless(HAVE_PYSAL, "PySAL stack not installed") +class TestEigenvaluesAndImpacts(unittest.TestCase): + def test_eigvalsh_equals_eigvals(self): + from census.simulation_spreg import lattice_w + from census.spatial import _eigenvalues_dense + for kind in ("rook", "queen", "knn"): + W = lattice_w(6, kind).full()[0] + ref = np.sort_complex(np.linalg.eigvals(W)) + got = np.sort_complex(np.asarray(_eigenvalues_dense(W), complex)) + np.testing.assert_allclose(got, ref, atol=1e-10, err_msg=kind) + w, _, _ = _lag_with_reps(island=True) + W = w.full()[0] + np.testing.assert_allclose(np.sort(np.real(_eigenvalues_dense(W))), np.sort(np.real(np.linalg.eigvals(W))), + atol=1e-10) + self.assertTrue(np.isrealobj(_eigenvalues_dense(W))) # symmetric pattern: eigvalsh path + + def test_cached_on_weights(self): + from census import spatial + w, _, _ = _lag_with_reps() + with mock.patch.object(spatial, "_eigenvalues_dense", wraps=spatial._eigenvalues_dense) as dense: + a = spatial.weights_eigenvalues(w) + b = spatial.weights_eigenvalues(w) + self.assertEqual(dense.call_count, 1) + self.assertIs(a, b) + w.transform = "b" # different matrix: recomputed + spatial.weights_eigenvalues(w) + self.assertEqual(dense.call_count, 2) + w.transform = "r" + + def test_results_unchanged(self): + from census import spatial + w, df, reps = _lag_with_reps(island=True) + reps = reps[reps["GEOID"].isin(df["GEOID"])] + new = {m: spatial.spatial_regression("y ~ x", df, w, model="lag", method=m, replicates=reps) + for m in ("gm", "ml")} + old_eig = lambda w_: np.linalg.eigvals(np.asarray(w_.full()[0], float)) # noqa: E731 + with mock.patch.object(spatial, "weights_eigenvalues", old_eig): + for m in ("gm", "ml"): + old = spatial.spatial_regression("y ~ x", df, w, model="lag", method=m, replicates=reps, + spat_impacts="simple") + for attr in ("coefficients", "effects"): + a, b = getattr(new[m], attr), getattr(old, attr) + num = a.select_dtypes("number").columns + np.testing.assert_allclose(a[num].to_numpy(float), b[num].to_numpy(float), rtol=1e-9, + atol=1e-12, err_msg=f"{m} {attr}") + + def test_spreg_impacts_off(self): + from census import spatial + w, df, _ = _lag_with_reps() + y, X = df[["y"]].to_numpy(), df[["x"]].to_numpy() + for m in ("gm", "ml"): + r, *_ = spatial._spreg_fit(y, X, w, "lag", m, ["x"], "y", "full", {}) + self.assertIsNone(getattr(r, "sp_multipliers", None), m) + + +@unittest.skipUnless(HAVE_PYSAL, "PySAL stack not installed") +class TestIslandMultipliers(unittest.TestCase): + def test_diagnostics_match_effects_table(self): + from census.spatial import spatial_regression + w, df, _ = _lag_with_reps(island=True) + res = spatial_regression("y ~ x", df, w, model="lag") + beta = res.coefficients.set_index("term").loc["x", "estimate"] + eff = res.effects.set_index("effect")["estimate"] + m = res.diagnostics["effects_multipliers"] + for e in ("direct", "indirect", "total"): + self.assertAlmostEqual(m[e] * beta, eff[e], places=10, msg=e) + rho = res.coefficients.set_index("term").loc["rho", "estimate"] + s = 1 - 1 / w.n + self.assertAlmostEqual(m["total"], (1 - s) + s / (1 - rho), places=12) + + +# --------------------------------------------------------------------------- +# 6, 9: census.http +# --------------------------------------------------------------------------- + + +class TestPacingThreads(unittest.TestCase): + def setUp(self): + for p in (mock.patch.dict(os.environ, {"CENSUS_WWW2_MIN_INTERVAL": "1"}), + mock.patch.dict(H._last_request, clear=True), + mock.patch.object(H, "_monotonic", lambda: 1000.0)): + p.start() + self.addCleanup(p.stop) + + def test_sleep_outside_the_lock(self): + release = threading.Event() + sleeping = [] + lock = threading.Lock() + + def blocking_sleep(secs): + with lock: + sleeping.append((threading.current_thread().name, secs)) + release.wait(10) + + H._last_request["www2.census.gov"] = 1000.0 # a request just went out + with mock.patch.object(H, "_sleep", blocking_sleep): + ts = [threading.Thread(target=H._pace, args=("www2.census.gov",), name=f"t{i}") for i in range(2)] + other = threading.Thread(target=H._pace, args=("www.census.gov",), name="other") + for t in ts: + t.start() + other.start() + other.join(5) + self.assertFalse(other.is_alive(), "a wait for one host blocked another host") + for _ in range(500): # both same-host threads reach their (blocking) sleep concurrently + with lock: + if len(sleeping) == 2: + break + threading.Event().wait(0.01) + release.set() + for t in ts: + t.join(5) + self.assertEqual(sorted(s for _, s in sleeping), [1.0, 2.0]) # successive slots, per host + self.assertEqual(H._last_request["www2.census.gov"], 1002.0) + self.assertEqual(H._last_request["www.census.gov"], 1000.0) + + +class _Resp: + def __init__(self, status, content=b"PK", headers=None): + self.status_code, self.content = status, content + self.headers = {"Content-Type": "application/zip", **(headers or {})} + self.closed = False + + def close(self): + self.closed = True + + +class TestStreamedRetryClosed(unittest.TestCase): + def test_closed_before_retry(self): + first, second = _Resp(503), _Resp(200) + page = _Resp(200, b"Request RejectedYour support ID is: 12", + {"Content-Type": "text/html"}) + session = mock.Mock() + session.get.side_effect = [first, page, second] + with mock.patch.dict(os.environ, {"CENSUS_WWW2_MIN_INTERVAL": "0", "CENSUS_WWW2_REJECTED_RETRY_DELAY": "0"}), \ + mock.patch.object(H, "_sleep", lambda s: None), mock.patch.object(H, "_jitter", lambda: 1.0): + out = H.polite_get("https://www2.census.gov/f.zip", user_agent="ua", session=session, stream=True) + self.assertIs(out, second) + self.assertTrue(first.closed) + self.assertTrue(page.closed) + self.assertFalse(second.closed) + + +# --------------------------------------------------------------------------- +# 3: manually placed / cached / downloaded files +# --------------------------------------------------------------------------- + + +def _zip(member, text="a,b\n1,2\n"): + raw = io.BytesIO() + with zipfile.ZipFile(raw, "w") as z: + z.writestr(member, text) + return raw.getvalue() + + +class _Session: + def __init__(self, items): + self.items, self.calls = list(items), 0 + + def get(self, url, **kw): + self.calls += 1 + return _Resp(200, self.items.pop(0)) + + +class TestManualFiles(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.tmp, True) + for p in (mock.patch.dict(os.environ, {"CENSUS_WWW2_MIN_INTERVAL": "0"}), + mock.patch.object(H, "_sleep", lambda s: None)): + p.start() + self.addCleanup(p.stop) + + def test_payload_problem(self): + self.assertIsNone(H.payload_problem(_zip("psam_p56.csv"), ".csv")) + self.assertIn("no .csv member", H.payload_problem(_zip("readme.txt"), ".csv")) + self.assertIn("not a zip", H.payload_problem(b"a,b\n1,2\n", ".csv")) + self.assertIn("HTML", H.payload_problem(HTML_PAGE, ".csv")) + self.assertIn("HTML", H.payload_problem(b"\xef\xbb\xbf ", None)) + self.assertIn("firewall", H.payload_problem(b"Request Rejected", None)) + self.assertIn("empty", H.payload_problem(b"", None)) + self.assertIsNone(H.payload_problem(b"NAME,TYPE\nAGEP,N\n", None)) + + def test_pums_manual_html_refused_and_moved_aside(self): + from census import pums as P + url = P.pums_url("56", 2024, "1-year", "person") + path = os.path.join(self.tmp, os.path.basename(url)) + with open(path, "wb") as f: + f.write(HTML_PAGE) + with self.assertRaisesRegex(P.PumsError, r"not usable: it is an HTML.*moved aside to .*\.invalid"): + P.cached_download(url, cache_dir=self.tmp, session=_Session([])) + self.assertFalse(os.path.exists(path + ".json")) + self.assertFalse(os.path.exists(path)) + self.assertTrue(os.path.exists(path + ".invalid")) + # does not stick: the next call downloads + good = _zip("psam_p56.csv") + s = _Session([good]) + p2, meta = P.cached_download(url, cache_dir=self.tmp, session=s) + self.assertEqual((s.calls, p2), (1, path)) + self.assertNotIn("source", meta) + + def test_pums_manual_zip_without_csv_refused(self): + from census import pums as P + url = P.pums_url("56", 2024, "1-year", "person") + path = os.path.join(self.tmp, os.path.basename(url)) + with open(path, "wb") as f: + f.write(_zip("notes.txt")) + with self.assertRaisesRegex(P.PumsError, "no .csv member"): + P.read_cache_entry(path, url) + self.assertFalse(os.path.exists(path + ".json")) + + def test_pums_bad_download_not_cached(self): + from census import pums as P + url = P.pums_url("56", 2024, "1-year", "person") + path = os.path.join(self.tmp, os.path.basename(url)) + with self.assertRaisesRegex(P.PumsError, "not the data file.*Nothing was cached"): + P.cached_download(url, cache_dir=self.tmp, session=_Session([HTML_PAGE])) + self.assertFalse(os.path.exists(path)) + self.assertFalse(os.path.exists(path + ".json")) + + def test_pums_invalid_cached_file_is_replaced(self): + from census import pums as P + url = P.pums_url("56", 2024, "1-year", "person") + path = os.path.join(self.tmp, os.path.basename(url)) + P.write_cache_entry(path, HTML_PAGE, url) # e.g. cached before validation existed + s = _Session([_zip("psam_p56.csv")]) + with warnings.catch_warnings(record=True) as rec: + warnings.simplefilter("always") + P.cached_download(url, cache_dir=self.tmp, session=s) + self.assertEqual(s.calls, 1) + self.assertTrue(any("moved aside" in str(r.message) for r in rec)) + self.assertTrue(zipfile.is_zipfile(path)) + self.assertTrue(os.path.exists(path + ".invalid")) + + def test_block_relationship_files(self): + from census import block_weights as BW + with mock.patch.dict(os.environ, {"CENSUS_BLOCK_DIR": self.tmp}): + url = BW.REL_URL.format(state="11") + path = os.path.join(self.tmp, os.path.basename(url)) + with open(path, "wb") as f: + f.write(HTML_PAGE) + with self.assertRaisesRegex(BW.BlockDataError, r"Manually placed file .* not usable.*moved aside"): + BW.load_block_relationships("11", session=_Session([])) + self.assertFalse(os.path.exists(path + ".json")) + self.assertTrue(os.path.exists(path + ".invalid")) + with self.assertRaisesRegex(BW.BlockDataError, r"no \.txt member.*Nothing was cached"): + BW.load_block_relationships("11", session=_Session([_zip("x.csv")])) + self.assertFalse(os.path.exists(path)) + + def test_boundary_manual_file_moved_aside(self): + from census import boundaries as bd + bad = os.path.join(self.tmp, "bad.zip") + with open(bad, "wb") as f: + f.write(b"") + with self.assertRaisesRegex(bd.BoundaryError, "moved aside"): + bd._read_cached(bad, "https://www2.census.gov/geo/tiger/bad.zip") + self.assertFalse(os.path.exists(bad + ".json")) + self.assertTrue(os.path.exists(bad + ".invalid")) + + +# --------------------------------------------------------------------------- +# 10: scripts/acs_error_covariance.py +# --------------------------------------------------------------------------- + + +def _load_script(): + spec = importlib.util.spec_from_file_location("acs_error_covariance", + os.path.join(ROOT, "scripts", "acs_error_covariance.py")) + mod = importlib.util.module_from_spec(spec) + sys.modules.setdefault("acs_error_covariance", mod) + spec.loader.exec_module(mod) + return mod + + +class TestCovarianceZeroDenominator(unittest.TestCase): + def test_zero_replicate_denominator_invalidates_area(self): + from census.variance import REPLICATE_COLUMNS + mod = _load_script() + rng = np.random.default_rng(0) + geoids = ["a", "b", "c"] + rows = [] + for var, base in (("B17001_002", 20.0), ("B17001_001", 100.0)): + for g in geoids: + reps = base + rng.normal(0, 2, 80) + if var == "B17001_001" and g == "b": + reps[4] = 0.0 # replicate 5: no persons in the poverty universe + rows.append({"GEOID": g, "variable": var, "estimate": base, "controlled": False, + **dict(zip(REPLICATE_COLUMNS, reps))}) + frames = {"B17001": pd.DataFrame(rows)} + est, reps, valid, info = mod.variable_values(frames, "poverty_rate", geoids, 2024) + self.assertEqual(valid.tolist(), [True, False, True]) + self.assertTrue(np.isnan(reps[1]).all()) + self.assertIn("1 area(s) with a zero replicate denominator excluded", info["note"]) + # never a fabricated replicate ratio of 0 for a valid area + self.assertTrue((reps[valid] > 0).all()) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_spatial_lambda_correction.py b/tests/test_spatial_lambda_correction.py new file mode 100644 index 0000000..ef66862 --- /dev/null +++ b/tests/test_spatial_lambda_correction.py @@ -0,0 +1,253 @@ +# SPDX-License-Identifier: AGPL-3.0-or-later +"""spatial_regression(model='error', correct_outcome_error=True): lambda corrected for outcome sampling error. + +PLAN_ROUND_16 A1 (owner-approved option). The tool path must be the A1 +study's estimator: the spatial Fay-Herriot REML (or ML) fit with diagonal +Sigma = diag(SE^2), census.spatial.error_model_known_variances, which the +study's diagonal rows call directly (tests/test_review_fixes_r16.py); its +coverage is in docs/audits/spatial_regression_validation.md. Also checked +against the study-only me_sem_fit on the study's own seeded draws, plus +option handling, warnings, boundary and convergence. +""" + +import json +import unittest +from unittest.mock import patch + +import numpy as np + +try: + import libpysal # noqa: F401 + import spreg # noqa: F401 + HAVE_PYSAL = True +except ImportError: # pragma: no cover + HAVE_PYSAL = False + + +def setUpModule(): + # census.simulation registers (imports) census.simulation_spreg; import it first + import census.simulation # noqa: F401 + + +def _a1_draw(kind="rook", lam=0.6, cv_scale=1.0, reps=1, seed=None): + """The first ``reps`` draws of the A1 'spatial_error_attenuation' cell (same generator order and seed).""" + from census.simulation import FULL, _geoids, draw_cv, sar_inverse + from census.simulation_spreg import (SPREG16_CV_SCALES, SPREG16_LAMBDAS, SPREG16_SIDE, SPREG16_WEIGHTS, + lattice_w) + cfg = FULL + m = SPREG16_SIDE * SPREG16_SIDE + g = _geoids(m) + w = lattice_w(SPREG16_SIDE, kind, g) + W = w.full()[0] + if seed is None: + seed = (cfg.seed + 1600 + 100 * SPREG16_WEIGHTS.index(kind) + 10 * SPREG16_LAMBDAS.index(lam) + + SPREG16_CV_SCALES.index(cv_scale)) + rng = np.random.default_rng(seed) + Binv = sar_inverse(W, lam) + mean_y = 10.0 + cfg.beta * cfg.x_mean + out = [] + for _ in range(reps): + x = cfg.x_mean + cfg.x_sd * rng.standard_normal(m) + ystar = 10.0 + cfg.beta * x + Binv @ (cfg.sigma_u * rng.standard_normal(m)) + s = draw_cv(m, "tract", rng) / 100 * mean_y * cv_scale + y = ystar + s * rng.standard_normal(m) + out.append((x, y, s)) + return cfg, seed, g, w, W, out + + +def _frame(g, x, y, s, outcome="y"): + from census.simulation import acs_frame + f = acs_frame(g, {"y": y, "x": x}, {"y": s}) + return f.rename(columns={"y": outcome, "y_se": f"{outcome}_se"}) + + +@unittest.skipUnless(HAVE_PYSAL, "PySAL stack not installed") +class TestReproducesStudyEstimator(unittest.TestCase): + def test_equals_me_sem_fit_on_study_draws(self): + """The tool's lambda, its SE, beta and its SE equal me_sem_fit (diagonal SE^2) on A1's seeded draws.""" + from census.simulation_spreg import me_sem_fit + from census.spatial import spatial_regression + for kind, lam in (("rook", 0.6), ("queen", 0.3), ("knn", 0.0)): + _, _, g, w, W, draws = _a1_draw(kind, lam) + x, y, s = draws[0] + for meth in ("REML", "ML"): + f = me_sem_fit(y, x[:, None], W, np.diag(s ** 2), method=meth) + res = spatial_regression("y ~ x", _frame(g, x, y, s), w, model="error", correct_outcome_error=True, + se_col="y_se", method=None if meth == "REML" else "ml") + tab = res.coefficients.set_index("term") + msg = f"{kind} {meth}" + self.assertTrue(f["converged"], msg) + self.assertAlmostEqual(tab.loc["lambda", "estimate"], f["lambda"], places=6, msg=msg) + self.assertAlmostEqual(tab.loc["lambda", "se_model"], f["se_lambda"], places=6, msg=msg) + np.testing.assert_allclose(tab.loc[["Intercept", "x"], "estimate"], f["beta"], rtol=1e-7, err_msg=msg) + np.testing.assert_allclose(tab.loc[["Intercept", "x"], "se_model"], f["se_beta"], rtol=1e-6, + err_msg=msg) + self.assertAlmostEqual(res.diagnostics["outcome_error_correction"]["sigma2"] / f["A"], 1.0, places=6) + # the validated interval: Wald, z at the 90% level (the study's lam_cov / b_cov) + z = 1.6448536269514722 + self.assertAlmostEqual(tab.loc["lambda", "ci_lower"], f["lambda"] - z * f["se_lambda"], places=6) + self.assertAlmostEqual(tab.loc["x", "ci_upper"], f["beta"][1] + z * f["se_beta"][1], places=6) + + def test_equals_study_cell_row(self): + """One repetition of the A1 study cell reports the same lambda estimates as the tool on that draw.""" + from census.simulation_spreg import _attenuation_cell + from census.spatial import spatial_regression + cfg, seed, g, w, W, draws = _a1_draw("rook", 0.6) + rows = _attenuation_cell(cfg, 1, seed, "rook", 0.6, 1.0, None) + lam_rows = {r["estimator"]: r for r in rows if r["term"] == "lambda"} + x, y, s = draws[0] + fr = _frame(g, x, y, s) + reml = spatial_regression("y ~ x", fr, w, model="error", correct_outcome_error=True, se_col="y_se") + ml = spatial_regression("y ~ x", fr, w, model="error", correct_outcome_error=True, se_col="y_se", method="ml") + naive = spatial_regression("y ~ x", fr, w, model="error") + get = lambda r: float(r.coefficients.set_index("term").loc["lambda", "estimate"]) # noqa: E731 + self.assertAlmostEqual(get(reml), lam_rows[next(k for k in lam_rows if "REML" in k)]["mean_estimate"], places=6) + self.assertAlmostEqual(get(ml), lam_rows[next(k for k in lam_rows if " ML rho" in k)]["mean_estimate"], + places=6) + self.assertAlmostEqual(get(naive), lam_rows["uncorrected ML_Error [tool default]"]["mean_estimate"], places=6) + self.assertAlmostEqual(reml.diagnostics["outcome_error_correction"]["lambda_uncorrected"], get(naive), places=8) + + def test_small_coverage_check(self): + """Fast-size coverage smoke check on the study's first draws (rook, lambda 0.6, CV x 1). + + Not an acceptance band: the full study (300 reps) is the evidence. This + only guards against a tool path that diverges from it. + """ + from census.spatial import spatial_regression + reps = 20 + _, _, g, w, W, draws = _a1_draw("rook", 0.6, reps=reps) + cov, est, naive = [], [], [] + for x, y, s in draws: + fr = _frame(g, x, y, s) + tab = spatial_regression("y ~ x", fr, w, model="error", correct_outcome_error=True, + se_col="y_se").coefficients.set_index("term") + est.append(tab.loc["lambda", "estimate"]) + cov.append(tab.loc["lambda", "ci_lower"] <= 0.6 <= tab.loc["lambda", "ci_upper"]) + naive.append(spatial_regression("y ~ x", fr, w, model="error").coefficients.set_index("term") + .loc["lambda", "estimate"]) + self.assertLess(abs(np.mean(est) - 0.6), 0.1) # study: bias -0.017 (SFH REML) + self.assertLess(np.mean(naive), np.mean(est) - 0.1) # study: naive bias -0.274 + self.assertGreaterEqual(np.mean(cov), 0.7) # study: 0.863 (300 reps) + + +@unittest.skipUnless(HAVE_PYSAL, "PySAL stack not installed") +class TestOption(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.cfg, _, cls.g, cls.w, cls.W, draws = _a1_draw("queen", 0.3) + cls.x, cls.y, cls.s = draws[0] + + def fit(self, outcome="y", **kw): + from census.spatial import spatial_regression + kw.setdefault("se_col", f"{outcome}_se") + return spatial_regression(f"{outcome} ~ x", _frame(self.g, self.x, self.y, self.s, outcome), self.w, + model="error", correct_outcome_error=True, **kw) + + def test_result_contract(self): + res = self.fit() + d = res.to_dict() + json.dumps(d) + self.assertEqual(res.model_type, "spatial_error") + self.assertEqual(list(res.coefficients["term"]), ["Intercept", "x", "lambda"]) + self.assertEqual(res.uncertainty_sources, ["model", "sampling_model"]) + self.assertIn("REML", res.method) + self.assertEqual(d["interval"]["basis_column"], "se_model") + self.assertEqual(d["interval"]["undefined_terms"], []) + diag = res.diagnostics + self.assertTrue(diag["converged"]) + self.assertFalse(diag["boundary"]) + self.assertFalse(diag["controlled_total_warning"]) + self.assertIn("lm_tests", diag) + oc = diag["outcome_error_correction"] + self.assertEqual(oc["variance_method"], "REML") + self.assertGreater(oc["lambda"], oc["lambda_uncorrected"]) # attenuation removed + text = " ".join(res.notes) + self.assertIn("independent across areas", text) + self.assertIn("known (= SE^2", text) + self.assertIn("acs_sampling_covariance.md", text) + self.assertIn("spatial_regression_validation.md", text) + self.assertIn("diffusion of shocks", text) + self.assertIn("not a spillover", text) + self.assertIn("positively correlated sampling errors", text) + self.assertIn("+0.19 to +0.22", text) + self.assertNotIn("WARNING", text) + self.assertTrue(res.provenance["outcome_error_correction"]) + + def test_default_unchanged(self): + from census import spatial + fr = _frame(self.g, self.x, self.y, self.s) + a = spatial.spatial_regression("y ~ x", fr, self.w, model="error") + b = spatial.spatial_regression("y ~ x", fr, self.w, model="error", correct_outcome_error=False) + self.assertEqual(a.method, "spreg ML spatial error") + self.assertEqual(a.to_dict()["coefficients"], b.to_dict()["coefficients"]) + self.assertIn(spatial.ERROR_CAVEAT, a.notes) + self.assertIn("correct_outcome_error=True", spatial.ERROR_CAVEAT) + self.assertIn("understates the true lambda", spatial.ERROR_CAVEAT) + self.assertNotIn("outcome_error_correction", a.diagnostics) + + def test_invalid_options(self): + from census.spatial import spatial_regression + fr = _frame(self.g, self.x, self.y, self.s) + with self.assertRaisesRegex(ValueError, "needs se_col"): + self.fit(se_col=None) + with self.assertRaisesRegex(ValueError, "model='error' only"): + spatial_regression("y ~ x", fr, self.w, model="lag", correct_outcome_error=True, se_col="y_se") + with self.assertRaisesRegex(ValueError, "method must be one of"): + self.fit(method="gm") + with self.assertRaisesRegex(ValueError, "only with correct_outcome_error"): + spatial_regression("y ~ x", fr, self.w, model="error", se_col="y_se") + with self.assertRaisesRegex(ValueError, "target='population'"): + self.fit(target="conservative") + with self.assertRaisesRegex(KeyError, "nope"): + self.fit(se_col="nope") + with self.assertRaisesRegex(ValueError, "plain outcome column"): + spatial_regression("np.log(y) ~ x", fr, self.w, model="error", correct_outcome_error=True, se_col="y_se") + bad = fr.copy() + bad.loc[3, "y_se"] = -1.0 + with self.assertRaisesRegex(ValueError, "finite and >= 0"): + spatial_regression("y ~ x", bad, self.w, model="error", correct_outcome_error=True, se_col="y_se") + + def test_controlled_total_warning(self): + from census.spatial import controlled_total_reason + for name in ("B01003_001E", "B01003_001", "B11001_001E", "B25003_001", "population", "Households"): + self.assertIsNotNone(controlled_total_reason(name), name) + for name in ("poverty_rate", "B25003_002E", "owner_share", "median_hh_income", "y"): + self.assertIsNone(controlled_total_reason(name), name) + res = self.fit(outcome="B01003_001E") + self.assertTrue(res.diagnostics["controlled_total_warning"]) + self.assertTrue(res.notes[0].startswith("WARNING: the outcome looks like a controlled total")) + self.assertIn("B01003_001", res.diagnostics["outcome_error_correction"]["controlled_total_reason"]) + flagged = self.fit(controlled_total=True) + self.assertTrue(flagged.diagnostics["controlled_total_warning"]) + self.assertIn("flagged by the caller", flagged.notes[0]) + off = self.fit(outcome="B01003_001E", controlled_total=False) + self.assertFalse(off.diagnostics["controlled_total_warning"]) + # the estimate itself does not depend on the warning + self.assertEqual(off.to_dict()["coefficients"], res.to_dict()["coefficients"]) + + def test_non_convergence_is_an_error(self): + from census import sae_spatial + real = sae_spatial._fit + + def unconverged(*a, **k): + out = real(*a, **k) + out["converged"] = False + return out + with patch.object(sae_spatial, "_fit", unconverged): + with self.assertRaisesRegex(RuntimeError, "did not converge.*not substituted"): + self.fit() + + def test_boundary_flag(self): + # sampling variances far above the outcome's variation: sigma^2 -> 0, lambda not identified + from census.spatial import spatial_regression + fr = _frame(self.g, self.x, self.y, self.s * 40) + res = spatial_regression("y ~ x", fr, self.w, model="error", correct_outcome_error=True, se_col="y_se") + self.assertTrue(res.diagnostics["boundary"]) + self.assertTrue(any("sigma^2 estimated at 0" in r + for r in res.diagnostics["outcome_error_correction"]["boundary_reasons"])) + self.assertIn("lambda", res.to_dict()["interval"]["undefined_terms"]) + self.assertTrue(any(n.startswith("boundary fit:") for n in res.notes)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_spatial_regression_validation.py b/tests/test_spatial_regression_validation.py new file mode 100644 index 0000000..9dda27d --- /dev/null +++ b/tests/test_spatial_regression_validation.py @@ -0,0 +1,339 @@ +# SPDX-License-Identifier: AGPL-3.0-or-later +"""PLAN_ROUND_16 workstream A: lag-model effects (A2) and the study estimators (A1, A4). + +Effects are checked against an independent computation from +S(W) = (I - rho W)^-1 beta (direct = mean diagonal, total = mean row sum, +indirect = total - direct) and against spreg's own multipliers; the study +likelihoods against spreg's ML_Error / ML_Lag(slx_lags=1) and the +measurement-error SEM against census.sae_spatial (spatial Fay-Herriot, ML). +""" + +import unittest +import warnings + +import numpy as np +import pandas as pd + +try: + import libpysal + import spreg + HAVE_PYSAL = True +except ImportError: # pragma: no cover + HAVE_PYSAL = False + + +def _lag_data(seed=2, side=8, rho=0.5, island=False): + from census.simulation import lattice_weights, sar_inverse + rng = np.random.default_rng(seed) + w = lattice_weights(side) + if island: + nb = dict(w.neighbors) + nb["island"] = [] + w = libpysal.weights.W(nb, id_order=list(w.id_order) + ["island"], silence_warnings=True) + w.transform = "r" + n = w.n + W = w.full()[0] + x, z = rng.normal(size=n), rng.normal(size=n) + y = sar_inverse(W, rho) @ (1 + 2 * x - z + rng.normal(size=n)) + return w, W, pd.DataFrame({"GEOID": w.id_order, "y": y, "x": x, "z": z}) + + +@unittest.skipUnless(HAVE_PYSAL, "PySAL stack not installed") +class TestLagEffects(unittest.TestCase): + def _independent(self, res, W): + tab = res.coefficients.set_index("term")["estimate"] + n = W.shape[0] + S = np.linalg.inv(np.eye(n) - tab["rho"] * W) + out = {} + for v in ("x", "z"): + direct, total = tab[v] * np.trace(S) / n, tab[v] * S.sum() / n + out[v] = {"direct": direct, "total": total, "indirect": total - direct} + return out + + def test_effects_match_independent_computation(self): + from census.spatial import spatial_regression + for island in (False, True): + w, W, df = _lag_data(island=island) + for method in ("gm", "ml"): + res = spatial_regression("y ~ x + z", df, w, model="lag", method=method) + ref = self._independent(res, W) + eff = res.effects.set_index(["variable", "effect"])["estimate"] + for v in ("x", "z"): + for e in ("direct", "indirect", "total"): + self.assertAlmostEqual(eff[(v, e)], ref[v][e], places=10, msg=f"{method} {v} {e} {island}") + + def test_effects_match_spreg_multipliers(self): + from census.spatial import spatial_regression + w, W, df = _lag_data() + res = spatial_regression("y ~ x + z", df, w, model="lag", method="ml") + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + r = spreg.ML_Lag(df[["y"]].to_numpy(), df[["x", "z"]].to_numpy(), w, spat_impacts="full") + adi, aii, ati = r.sp_multipliers["full"] + beta = res.coefficients.set_index("term")["estimate"] + eff = res.effects.set_index(["variable", "effect"])["estimate"] + for v in ("x", "z"): + self.assertAlmostEqual(eff[(v, "direct")], beta[v] * float(adi), places=6) + self.assertAlmostEqual(eff[(v, "indirect")], beta[v] * float(aii), places=6) + self.assertAlmostEqual(eff[(v, "total")], beta[v] * float(ati), places=6) + + def test_delta_method_se(self): + # (1) the analytic gradient equals a numerical gradient of the effects (exact check of the + # delta method); (2) where rho is well determined (n = 400) the delta-method SEs agree with + # Monte Carlo draws of (beta, rho) from N(estimate, vm). With an imprecise rho the effects + # are visibly nonlinear in rho (1 / (1 - rho)) and the two differ; coverage of the + # delta-method intervals is measured by the 'spatial_dgp_recovery' study. + from census.spatial import _lag_effect_values, _silence_spreg_output, lag_effects, weights_eigenvalues + _silence_spreg_output() + for side, check_mc in ((8, False), (20, True)): + w, W, df = _lag_data(side=side, rho=0.4) + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + r = spreg.GM_Lag(df[["y"]].to_numpy(), df[["x", "z"]].to_numpy(), w=w) + params, vm = np.asarray(r.betas, float).ravel(), np.asarray(r.vm, float) + eig = weights_eigenvalues(w) + out = lag_effects(params, vm, ["x", "z"], eig) + h = 1e-6 + J = np.column_stack([(_lag_effect_values(params + h * e, eig, 2, np.asarray(1.0)) + - _lag_effect_values(params - h * e, eig, 2, np.asarray(1.0))) / (2 * h) + for e in np.eye(len(params))]) + np.testing.assert_allclose(out["se"], np.sqrt(np.diag(J @ vm @ J.T)), rtol=1e-6) + if check_mc: + draws = np.random.default_rng(0).multivariate_normal(params, vm, size=20000) + sim = np.vstack([_lag_effect_values(d, eig, 2, np.asarray(1.0)) for d in draws]) + np.testing.assert_allclose(out["se"], sim.std(axis=0), rtol=0.05) + + def test_effects_first_and_labels(self): + from census.spatial import LAG_EFFECTS_NOTE, spatial_regression + w, W, df = _lag_data() + res = spatial_regression("y ~ x + z", df, w) + d = res.to_dict() + keys = list(d) + self.assertLess(keys.index("effects"), keys.index("coefficients")) + self.assertEqual(res.notes[0], LAG_EFFECTS_NOTE) + self.assertIn("simulation 'spatial_dgp_recovery'", res.notes[1]) + self.assertIn("not a specification test", " ".join(res.notes)) + self.assertTrue(res.summary().startswith("EFFECTS FIRST")) + interp = res.coefficients.set_index("term")["interpretation"] + self.assertIn("NOT the marginal effect", interp["x"]) + self.assertIn("not a marginal effect", res.provenance["coefficient_meaning"]) + self.assertEqual(list(res.effects.columns[:3]), ["term", "variable", "effect"]) + self.assertTrue(np.isfinite(res.effects["se_model"]).all()) + self.assertTrue((res.effects["ci_lower"] < res.effects["ci_upper"]).all()) + cv = res.diagnostics["coefficient_vcov"] + self.assertEqual(cv["terms"], ["Intercept", "x", "z", "rho"]) + np.testing.assert_allclose(np.sqrt(np.diag(cv["matrix"])), res.coefficients["se_model"]) + err = spatial_regression("y ~ x + z", df, w, model="error") + self.assertIsNone(err.effects) + self.assertNotIn("effects", err.to_dict()) + self.assertNotIn("interpretation", err.coefficients.columns) + + def test_replicate_effects_and_size_limit(self): + from census import spatial + from census.simulation import make_replicates, replicate_long + w, W, df = _lag_data(side=7) + rng = np.random.default_rng(1) + se = np.full(w.n, 0.3) + reps = replicate_long(w.id_order, ["y"], df[["y"]].to_numpy(), make_replicates(df[["y"]].to_numpy(), + se[:, None], rng)) + res = spatial.spatial_regression("y ~ x + z", df, w, replicates=reps) + self.assertTrue(np.isfinite(res.effects["se_sampling"]).all()) + self.assertTrue((res.effects["se_sampling"] > 0).all()) + old = spatial.EFFECTS_MAX_N + try: + spatial.EFFECTS_MAX_N = 10 + res = spatial.spatial_regression("y ~ x + z", df, w) + finally: + spatial.EFFECTS_MAX_N = old + self.assertIsNone(res.effects) + self.assertIn("Effects were NOT computed", res.notes[0]) + + def test_multipliers_outside_unit_interval(self): + from census.spatial import lag_multipliers + self.assertTrue(np.isnan(lag_multipliers(np.array([1.0, -0.5]), 1.0)["direct"])) + m = lag_multipliers(np.array([1.0, 0.0, -1.0]), 0.0) + self.assertEqual((m["direct"], m["total"], m["indirect"]), (1.0, 1.0, 0.0)) + + def test_effects_table(self): + from census.report_tables import effects_table, regression_table + from census.spatial import spatial_regression + w, W, df = _lag_data() + res = spatial_regression("y ~ x + z", df, w) + t = effects_table(res) + self.assertEqual(list(t.display.columns), ["Variable", "Direct", "Indirect", "Total"]) + self.assertEqual(t.display["Variable"].tolist(), ["x", "", "z", ""]) + self.assertIn("not marginal effects", t.notes[0]) + self.assertIn("[", effects_table(res, uncertainty="ci").display.iloc[1, 1]) + self.assertIn("not marginal effects", " ".join(regression_table({"lag": res}).notes)) + with self.assertRaises(ValueError): + effects_table(spatial_regression("y ~ x + z", df, w, model="error")) + + +@unittest.skipUnless(HAVE_PYSAL, "PySAL stack not installed") +class TestStudyEstimators(unittest.TestCase): + @classmethod + def setUpClass(cls): + from census.simulation import sar_inverse + from census.simulation_spreg import lattice_w + cls.data = {} + rng = np.random.default_rng(7) + for kind in ("rook", "queen", "knn"): + w = lattice_w(10, kind) + W = w.full()[0] + x = 50 + 24 * rng.standard_normal(w.n) + y = 10 + 0.5 * x + sar_inverse(W, 0.5) @ (8 * rng.standard_normal(w.n)) + 4 * rng.standard_normal(w.n) + cls.data[kind] = (w, W, np.linalg.eigvals(W), x, y) + + def test_lattice_weights(self): + from census.simulation_spreg import lattice_w + card = {k: np.mean(list(lattice_w(10, k).cardinalities.values())) for k in ("rook", "queen", "knn")} + self.assertAlmostEqual(card["rook"], 3.6) + self.assertAlmostEqual(card["queen"], (4 * 3 + 32 * 5 + 64 * 8) / 100) + self.assertEqual(card["knn"], 6) + np.testing.assert_array_equal(lattice_w(10, "knn").full()[0], lattice_w(10, "knn").full()[0]) + + def test_likelihoods_match_spreg(self): + from census.simulation_spreg import sdm_ml, sem_ml + for kind, (w, W, eig, x, y) in self.data.items(): + X = np.column_stack([np.ones(w.n), x]) + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + e = spreg.ML_Error(y[:, None], x[:, None], w) + lag = spreg.ML_Lag(y[:, None], x[:, None], w, slx_lags=1) + sem, sdm = sem_ml(y, X, W, eig), sdm_ml(y, X, W, eig) + self.assertAlmostEqual(sem["loglik"], float(np.ravel(e.logll)[0]), places=5, msg=kind) + self.assertAlmostEqual(sdm["loglik"], float(lag.logll), places=5, msg=kind) + self.assertAlmostEqual(sdm["rho"], float(lag.rho), places=5, msg=kind) + np.testing.assert_allclose(sdm["coef"], np.ravel(lag.betas)[:-1], rtol=1e-4) + + def test_common_factor_nesting(self): + # SEM = SDM with rho = lambda and theta = -lambda beta: the SDM likelihood at the + # restricted SEM estimates equals the SEM likelihood (same data, same Jacobian), so LR >= 0 + from census.simulation_spreg import _logdet_terms, common_factor_lr + for kind, (w, W, eig, x, y) in self.data.items(): + X = np.column_stack([np.ones(w.n), x]) + lr, sdm, sem = common_factor_lr(y, X, W, eig) + lam, (a, b) = sem["lambda"], sem["beta"] + e = y - lam * W @ y - (a * (1 - lam) + b * x - lam * b * (W @ x)) + n = w.n + s2 = float(e @ e) / n + ll = -0.5 * n * (np.log(2 * np.pi * s2) + 1) + _logdet_terms(eig, lam) + self.assertAlmostEqual(ll, sem["loglik"], places=6, msg=kind) + self.assertGreaterEqual(sdm["loglik"], sem["loglik"] - 1e-9) + self.assertGreaterEqual(lr, 0.0) + + def test_me_sem_matches_spatial_fay_herriot(self): + from census import sae_spatial + from census.simulation_spreg import me_sem_fit + w, W, eig, x, y = self.data["rook"] + D = np.random.default_rng(3).uniform(5, 30, w.n) + f = me_sem_fit(y, x[:, None], W, np.diag(D), method="ML") + g = sae_spatial._fit(y, np.column_stack([np.ones(w.n), x]), D, W, "ML") + self.assertTrue(f["converged"]) + self.assertAlmostEqual(f["lambda"], g["rho"], places=4) + self.assertAlmostEqual(f["A"] / g["A"], 1.0, places=3) + np.testing.assert_allclose(f["beta"], g["beta"], rtol=1e-4) + self.assertTrue(np.isfinite(f["se_lambda"])) + + def test_me_sem_reml_matches_spatial_fay_herriot(self): + # REML is spatial_fay_herriot's default; rho_se there uses the same REML information + from census import sae_spatial + from census.simulation_spreg import me_sem_fit + w, W, eig, x, y = self.data["queen"] + D = np.random.default_rng(4).uniform(5, 30, w.n) + f = me_sem_fit(y, x[:, None], W, np.diag(D)) + frame = pd.DataFrame({"GEOID": w.id_order, "y": y, "x": x, "y_se": np.sqrt(D)}) + res = sae_spatial.spatial_fay_herriot("y ~ x", frame, "y_se", W) + self.assertAlmostEqual(f["lambda"], res.provenance["rho"], places=4) + self.assertAlmostEqual(f["se_lambda"], res.provenance["rho_se"], places=4) + self.assertAlmostEqual(f["A"] / res.sigma2_u, 1.0, places=3) + + def test_residual_bootstrap(self): + from census.simulation_spreg import common_factor_lr, lr_residual_bootstrap + w, W, eig, x, y = self.data["rook"] + X = np.column_stack([np.ones(w.n), x]) + lr, _, sem = common_factor_lr(y, X, W, eig) + a = lr_residual_bootstrap(y, X, W, eig, lr, sem, np.random.default_rng(0), n_boot=20) + b = lr_residual_bootstrap(y, X, W, eig, lr, sem, np.random.default_rng(0), n_boot=20) + self.assertEqual(a, b) + self.assertFalse(lr_residual_bootstrap(y, X, W, eig, 0.0, sem, np.random.default_rng(1), n_boot=20)) + + def test_me_sem_without_error_is_sem(self): + from census.simulation_spreg import me_sem_fit, sem_ml + w, W, eig, x, y = self.data["queen"] + f = me_sem_fit(y, x[:, None], W, np.zeros((w.n, w.n)), method="ML") + s = sem_ml(y, np.column_stack([np.ones(w.n), x]), W, eig) + self.assertAlmostEqual(f["lambda"], s["lambda"], places=4) + self.assertAlmostEqual(f["loglik"], s["loglik"], places=4) + + def test_true_effects(self): + from census.simulation_spreg import true_effects + from census.spatial import lag_multipliers + w, W, eig, x, y = self.data["rook"] + self.assertEqual(true_effects(W, 0.0, 0.5), {"direct": 0.5, "indirect": 0.0, "total": 0.5}) + m = lag_multipliers(eig, 0.6) + t = true_effects(W, 0.6, 0.5) + self.assertAlmostEqual(t["direct"], 0.5 * m["direct"], places=10) + self.assertAlmostEqual(t["total"], 0.5 * m["total"], places=10) + sdm = true_effects(W, 0.4, 0.5, 0.3) + self.assertAlmostEqual(sdm["total"], 0.8 / 0.6, places=10) + # SEM written as SDM (theta = -rho beta): total effect beta + self.assertAlmostEqual(true_effects(W, 0.4, 0.5, -0.2)["total"], 0.5, places=10) + + def test_rs_additivity(self): + # Koley & Bera (2024): RS_rho,gamma = RS_rho + RS*_gamma = RS*_rho + RS_gamma (in spreg 1.9.1 the + # robust tests are defined by these differences, so this checks the wiring, not the theory) + from census.spatial import _silence_spreg_output + _silence_spreg_output() + w, W, eig, x, y = self.data["rook"] + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + lm = spreg.LMtests(spreg.OLS(y[:, None], x[:, None]), w, + tests=["lml", "lmwx", "lmspdurbin", "rlmwx", "rlmdurlag"]) + joint = lm.lmspdurbin[0] + self.assertAlmostEqual(joint, lm.lml[0] + lm.rlmwx[0], places=8) + self.assertAlmostEqual(joint, lm.rlmdurlag[0] + lm.lmwx[0], places=8) + + def test_sar_correlation(self): + from census.simulation_spreg import neighbour_correlation, sar_correlation + w, W, *_ = self.data["rook"] + R = sar_correlation(W, 0.6) + np.testing.assert_allclose(np.diag(R), 1.0) + self.assertTrue(np.all(np.linalg.eigvalsh(R) > 0)) + self.assertGreater(neighbour_correlation(R, W), neighbour_correlation(sar_correlation(W, 0.3), W)) + + +@unittest.skipUnless(HAVE_PYSAL, "PySAL stack not installed") +class TestStudies(unittest.TestCase): + def test_registered_on_demand_only(self): + from census.simulation import FULL, STUDIES, group_config, spatial_config + from census.simulation_spreg import SPREG16_REPS_FULL, SPREG_VALIDATION_STUDIES + for name in SPREG_VALIDATION_STUDIES: + self.assertIn(name, STUDIES) + self.assertEqual(FULL.reps.get(name, 0), 0) + self.assertEqual(spatial_config(fast=False).reps.get(name, 0), 0) + self.assertEqual({k: v for k, v in group_config("spreg_validation", False).reps.items() if v}, + SPREG16_REPS_FULL) + + def test_fast_studies_info_and_order_independent(self): + from census.simulation import run_study + from census.simulation_spreg import spreg_validation_config + cfg = spreg_validation_config(fast=True) + cfg.reps.update({"spatial_dgp_recovery": 1, "common_factor_lr_acs": 1}) + names = ["common_factor_lr_acs", "spatial_dgp_recovery"] + a = run_study(cfg, studies=names) + b = run_study(cfg, studies=names[::-1]) + self.assertTrue((a["status"] == "info").all()) + cols = ["method", "target", "coverage", "bias"] + pd.testing.assert_frame_equal( + a[a["study"] == "spatial_dgp_recovery"][cols].reset_index(drop=True), + b[b["study"] == "spatial_dgp_recovery"][cols].reset_index(drop=True)) + lag = a[(a["study"] == "spatial_dgp_recovery") & (a["model"] == "lag")] + self.assertTrue(set(lag["term"]) >= {"x (direct)", "x (indirect)", "x (total)", "rho", "x"}) + self.assertEqual(set(a[a["study"] == "common_factor_lr_acs"]["critical"]), + {"chi2", "bootstrap_residual", "bootstrap_me"}) + self.assertIn("juhl", set(a["dgp"].dropna())) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_validation_gate.py b/tests/test_validation_gate.py index 3d13b8a..5569334 100644 --- a/tests/test_validation_gate.py +++ b/tests/test_validation_gate.py @@ -218,7 +218,7 @@ def test_tiny_smoke_runs_are_info_only(self): self.assertEqual(set(lisa["scenario"]), {"null_random_se", "null_clustered_se", "hot_spot"}) self.assertTrue(lisa["power"].notna().any()) # the tool-default rows are labelled: GMM lag, ML error; two-sided 9999-permutation LISA - dflt = reg[reg["tool_default"].fillna(False).astype(bool)] + dflt = reg[reg["tool_default"].eq(True)] self.assertEqual(set(zip(dflt["model"], dflt["estimator"])), {("lag", "gm"), ("error", "ml")}) self.assertTrue(dflt["method"].str.contains(r"\[tool default\]").all()) ld = lisa[lisa["reading"] == "default"] diff --git a/tests/test_workflows_geo.py b/tests/test_workflows_geo.py index a819053..04cc5cb 100644 --- a/tests/test_workflows_geo.py +++ b/tests/test_workflows_geo.py @@ -64,10 +64,31 @@ def test_spatial(self): self.guided(res) self.assertIn("GMM", res["method"]) self.assertIn("the default for the lag model", " ".join(res["notes"])) + # PLAN_ROUND_16 A2: effects come before the coefficients, and beta is labelled as not the marginal effect + keys = list(res) + self.assertLess(keys.index("effects"), keys.index("coefficients")) + self.assertEqual({r["effect"] for r in res["effects"]}, {"direct", "indirect", "total"}) + self.assertIn("NOT its marginal effect", res["notes"][0]) + self.assertIn("effects table first", res["next_step"]) res = g.spatial_regression(self.path, "rate ~ x", "tract", 2023, method="ml") self.guided(res) self.assertIn("ML lag intervals for rho covered 0.48", " ".join(res["notes"])) self.assertFalse(g.spatial_regression(self.path, "rate ~ x", "tract", 2023, method="ols")["success"]) + # PLAN_ROUND_16 A1 option: lambda corrected for independent outcome sampling error + res = g.spatial_regression(self.path, "rate ~ x", "tract", 2023, model="error", correct_outcome_error=True, + se_col="rate_se") + self.guided(res) + self.assertIn("known outcome sampling variances", res["method"]) + self.assertIn("independent across areas", " ".join(res["notes"])) + self.assertIn("corrected lambda", res["next_step"]) + self.assertIn("lambda_uncorrected", res["diagnostics"]["outcome_error_correction"]) + self.assertFalse(res["diagnostics"]["controlled_total_warning"]) + flagged = g.spatial_regression(self.path, "rate ~ x", "tract", 2023, model="error", + correct_outcome_error=True, se_col="rate_se", controlled_total=True) + self.assertTrue(flagged["diagnostics"]["controlled_total_warning"]) + fail = g.spatial_regression(self.path, "rate ~ x", "tract", 2023, model="error", correct_outcome_error=True) + self.assertEqual(fail["error_code"], "INVALID_INPUT") + self.assertIn("needs se_col", json.dumps(fail)) self.guided(g.spatial_small_area_estimates(self.path, "rate ~ x", "rate_se", "tract", 2023, output_path=os.path.join(self.dir, "sfh.csv"))) self.assertFalse(g.spatial_regression(self.path, "rate ~ nope", "tract", 2023)["success"])