diff --git a/CHANGELOG.md b/CHANGELOG.md index 87e0259..07cf15c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,30 @@ each tagged release (`v`) is archived on Zenodo with its own DOI (see [Releasing](#releasing)). Plans and acceptance criteria for each round are in `PLAN_ROUND_.md`. +## 0.15.1 (patch): B01003 total from B01001 for firewall-blocked VRE files + +- **`fetch_vre` substitute for the total-population VRE file.** census.gov's firewall + rejects the 2015-2019 Michigan tract B01003 file for every client (support ID + 13979921158196382640). When a B01003 file is rejected or returns HTTP 404, `fetch_vre` + now reads line 1 ("Total:") of the B01001 file for the same release, geography and + state, and relabels it as B01003_001. + - Only this pair is substituted (`VRE_TOTAL_FALLBACKS`). On the 2015-2019 New York + tract files the two lines are identical in every cell. A live test + (`CENSUS_LIVE_TESTS=1`) re-checks this. + - The substitution is never silent. It issues a warning and is recorded in + `attrs["substitution"]`: requested URL, reason, the B01001 URL and its SHA-256, and + the verification. Manifest replicate records and methods text carry it. + - Other B01003 lines, other tables or failures, `allow_substitute=False`, or an + unavailable B01001 file still raise the original error. A new `lines=` argument + keeps only the requested table lines. +- **Michigan 2015-2019 population covariance completed** + (`docs/audits/acs_sampling_covariance.md`), using the B01001 substitute. Results match + 2020-2024 and New York, so the conclusions are unchanged. + - Order 1 within county is -0.075 [-0.103, -0.046] (2020-2024: -0.081). + - Across county it is -0.005 [-0.014, 0.004]. + - The county aggregation ratio is 0.000. + - Split-half pair reliability is 0.64. + ## 0.15.0 (Round 16): spatial methods validated under ACS sampling error - Project renamed **Census Research Engine** (github.com/cafeTechne/Census-Research-Engine); diff --git a/CITATION.cff b/CITATION.cff index 1088182..d9f72dd 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -12,7 +12,7 @@ authors: - alias: cafeTechne license: AGPL-3.0-or-later repository-code: "https://github.com/cafeTechne/Census-Research-Engine" -version: "0.15.0" +version: "0.15.1" doi: "10.5281/zenodo.23925301" # doi: Zenodo concept DOI, added after the first archived release (see CHANGELOG.md "Releasing"). date-released: "2026-11-09" diff --git a/README.md b/README.md index 9111b2b..d8fbae6 100644 --- a/README.md +++ b/README.md @@ -51,7 +51,7 @@ Alternatively, set `BLS_USER_AGENT` to an identifying User-Agent that includes a | `CENSUS_PRICE_PARITY_DIR` | Alternate Regional Price Parity directory (default `data/price_parities/`); same treatment | | `BLS_USER_AGENT` | Identifying User-Agent for `scripts/update_price_indexes.py` | | `CENSUS_VRE_DIR` | Cache for downloaded Census Variance Replicate Estimate files (default `.cache/vre/`) | -| `CENSUS_WWW2_MIN_INTERVAL` / `CENSUS_WWW2_ATTEMPTS` / `CENSUS_WWW2_BACKOFF_BASE` / `CENSUS_WWW2_REJECTED_RETRY_DELAY` | census.gov file downloads (`census/http.py`): seconds between requests to one host (default 1), attempts on HTTP 429/5xx or connection errors (4), backoff base in seconds (2; doubled per retry, capped at 60, honours `Retry-After`), and the wait before the single retry of a firewall "Request Rejected" page (30). A rejection that persists names the cache path where a browser-downloaded copy of the file is picked up | +| `CENSUS_WWW2_MIN_INTERVAL` / `CENSUS_WWW2_ATTEMPTS` / `CENSUS_WWW2_BACKOFF_BASE` / `CENSUS_WWW2_REJECTED_RETRY_DELAY` | census.gov file downloads (`census/http.py`): seconds between requests to one host (default 1), attempts on HTTP 429/5xx or connection errors (4), backoff base in seconds (2; doubled per retry, capped at 60, honours `Retry-After`), and the wait before the single retry of a firewall "Request Rejected" page (30). A rejection that persists names the cache path where a browser-downloaded copy of the file is picked up; for a rejected (or 404) B01003 VRE file, `fetch_vre` instead reads the identical B01001 line 1 and records the substitution (`attrs["substitution"]`, with a warning) | | `CENSUS_LIVE_TESTS=1` | Enable live API contract tests and the live validation battery | ## Quick start diff --git a/census/__init__.py b/census/__init__.py index 760bf59..81e2dcd 100644 --- a/census/__init__.py +++ b/census/__init__.py @@ -1,7 +1,7 @@ # SPDX-License-Identifier: AGPL-3.0-or-later """Headless Census ACS data client and processing library.""" -__version__ = "0.15.0" +__version__ = "0.15.1" from census.api import ( DATASETS, diff --git a/census/manifest.py b/census/manifest.py index 5c9a6d7..3c8ab37 100644 --- a/census/manifest.py +++ b/census/manifest.py @@ -1587,6 +1587,8 @@ def _census_vre_replicates(m: Mapping[str, Any], frame: pd.DataFrame, codes_by_y files.append({"table": table, "release": year, "summary_level": f.attrs.get("summary_level"), "state_code": f.attrs.get("state_code"), "url": f.attrs.get("source_url"), "sha256": f.attrs.get("sha256"), "moe_check": f.attrs.get("moe_check")}) + if f.attrs.get("substitution"): # B01003 total read from B01001 (census.replicate.fetch_vre) + files[-1]["substitution"] = f.attrs["substitution"] frames.append(f[f["GEOID"].astype(str).isin(want)]) rows = pd.concat(frames, ignore_index=True) if frames else pd.DataFrame() for n in names: @@ -2005,6 +2007,11 @@ def methods_section(manifest: Mapping[str, Any], results: Optional[Mapping[str, "SHA-256 of each file in run.json" + (f"; canonical SHA-256 of the replicates used `{rep_info['sha256']}`" if rep_info.get("sha256") else "") + ")") + subs = [f["substitution"] for f in files if f.get("substitution")] + if subs: + rep_source += "; " + "; ".join( + f"{', '.join(s['lines'])} read from {', '.join(s['lines'].values())} of `{s['source_url']}` because " + f"`{s['requested_url']}` was unavailable ({s['reason']}); {s['verification']}" for s in subs) rep_short = " and replicate-based standard errors from Census VRE files" elif results is None and census_vre_source(m): rep_source = "the 80 replicate estimates of the Census VRE files fetched for the manifest's tables" diff --git a/census/pipeline.py b/census/pipeline.py index a01f4ea..9cd764f 100644 --- a/census/pipeline.py +++ b/census/pipeline.py @@ -103,7 +103,7 @@ from census.stats import add_reliability_columns, clean_census_sentinels, parse_variable_code from census.tables import load_variable_metadata -PACKAGE_VERSION = "0.15.0" +PACKAGE_VERSION = "0.15.1" CONCORDANCE_MODES = ("fail", "strict", "code") MAX_ISSUES_IN_RESULT = 100 diff --git a/census/replicate.py b/census/replicate.py index 3598c31..f0c77c3 100644 --- a/census/replicate.py +++ b/census/replicate.py @@ -685,7 +685,9 @@ def _download( raw = resp.content or b"" content_type = str(resp.headers.get("Content-Type", "")) if resp.status_code == 404: - raise VREDownloadError(f"Not published (HTTP 404): {url}") + exc = VREDownloadError(f"Not published (HTTP 404): {url}") + exc.url, exc.status_code = url, 404 # type: ignore[attr-defined] + raise exc if resp.status_code != 200: raise VREDownloadError(f"HTTP {resp.status_code} for {url}") return raw, content_type @@ -915,6 +917,54 @@ def num(col: str) -> np.ndarray: return out.reset_index(drop=True) +# Substitute files for VRE files census.gov does not serve. Only the verified +# pair is listed: B01003 (Total Population) line 1 and B01001 (Sex by Age) +# line 1 ("Total:") are the same total population. Verified on 2026-11-09 for +# the 2015-2019 New York tract files (B01003_36 and B01001_36, 4,918 tracts): +# ESTIMATE, MOE and Var_Rep1-80 identical, 0 differing cells; the live test in +# tests/test_replicate_live.py re-checks it. Needed because www2.census.gov's +# firewall rejects .../2019/data/5-year/140/B01003_26.csv.zip for every client +# (support ID 13979921158196382640) while B01001_26 downloads normally. +VRE_TOTAL_FALLBACKS: Dict[Tuple[str, int], Tuple[str, int]] = {("B01003", 1): ("B01001", 1)} +VRE_TOTAL_FALLBACK_TITLES: Dict[Tuple[str, int], str] = {("B01003", 1): "Total"} +VRE_TOTAL_FALLBACK_VERIFICATION = ( + "B01003 line 1 (Total population) equals B01001 line 1 (Total:) cell for cell: verified 2026-11-09 on the " + "2015-2019 New York tract VRE files (4,918 tracts; ESTIMATE, MOE and Var_Rep1-80 identical, 0 differing " + "cells); re-checked by tests/test_replicate_live.py (CENSUS_LIVE_TESTS=1)." +) + + +def _fallback_reason(exc: BaseException) -> Optional[str]: + """Why a download failed when the failure allows a substitute file (firewall rejection or HTTP 404); else None.""" + if getattr(exc, "rejected", False): + sid = getattr(exc, "support_id", "") or "none shown" + return f"census.gov firewall 'Request Rejected' page (support ID {sid})" + if getattr(exc, "status_code", None) == 404: + return "HTTP 404 (not published)" + return None + + +def _load_vre_file( + table: str, + year: int, + sumlev: str, + state: Optional[str], + session: Any, + cache_dir: Optional[str], + refresh: bool, +) -> Tuple[bytes, Dict[str, Any], str, str]: + """(raw bytes, sidecar metadata, URL, cache path) of one VRE file, from the cache or downloaded.""" + url = vre_url(table, year, sumlev, state) + path = os.path.join(cache_dir or vre_cache_dir(year), sumlev, url.rsplit("/", 1)[-1]) + raw = None if refresh else _read_cached(path, url, _decompress) + if raw is None: + raw, ctype = _download(url, session=session, cache_path=path) + _decompress(raw) # validate before caching + return raw, _write_cached(path, raw, url, ctype), url, path + with open(_sidecar_path(path), encoding="utf-8") as f: + return raw, json.load(f), url, path + + def fetch_vre( table: str, year: int, @@ -925,12 +975,15 @@ def fetch_vre( cache_dir: Optional[str] = None, refresh: bool = False, check_moe: bool = True, + lines: Optional[Sequence[int]] = None, + allow_substitute: bool = True, ) -> pd.DataFrame: """Download (or read from cache) one VRE file and return its replicate frame. Tract and block-group files are per state, so ``state_code`` is required for them; for state-nested national files (state, county, county subdivision, - place, congressional district) ``state_code`` filters the rows. + place, congressional district) ``state_code`` filters the rows. ``lines`` + (ORDER numbers) keeps only those table lines. Files are cached under ``$CENSUS_VRE_DIR`` (default ``.cache/vre``) as ``//`` with a ``.json`` sidecar holding the source URL @@ -938,6 +991,18 @@ def fetch_vre( ``client`` (a census.api client) is accepted for interface symmetry; only its ``session`` attribute is used, if present. + Substitute file (``VRE_TOTAL_FALLBACKS``): when the B01003 file cannot be + fetched because census.gov rejects it (firewall page) or answers HTTP 404, + and only line 1 is needed, the B01001 file of the same release, geography + and state is fetched and its line 1 ("Total:", verified identical cell for + cell) is returned relabelled as B01003_001 with title "Total". This is never + silent: a warning is issued, ``attrs["substitution"]`` records the + requested and the substituted file (URLs, SHA-256 of the file read, reason, + verification statement) and ``attrs["notes"]`` says so; ``source_url`` and + ``sha256`` name the file actually read. Other lines, other tables, other + failures, or ``allow_substitute=False`` raise the original error, as does + an unavailable substitute. + With ``check_moe`` the published-MOE self-check runs on load and a warning is issued when any uncontrolled row disagrees beyond rounding. """ @@ -949,20 +1014,55 @@ def fetch_vre( state = str(state_code).zfill(2) if state_code not in (None, "") else None if state is not None and sumlev not in STATE_NESTED_SUMLEVS: raise ValueError(f"Summary level {sumlev} is not nested in states; omit state_code.") - url = vre_url(table, year, sumlev, state) - path = os.path.join(cache_dir or vre_cache_dir(year), sumlev, url.rsplit("/", 1)[-1]) - raw = None if refresh else _read_cached(path, url, _decompress) - meta: Dict[str, Any] - if raw is None: - if session is None and client is not None: - session = getattr(client, "session", None) - raw, ctype = _download(url, session=session, cache_path=path) - _decompress(raw) # validate before caching - meta = _write_cached(path, raw, url, ctype) - else: - with open(_sidecar_path(path), encoding="utf-8") as f: - meta = json.load(f) + wanted = None if lines is None else sorted({int(x) for x in lines}) + if session is None and client is not None: + session = getattr(client, "session", None) + substitution: Optional[Dict[str, Any]] = None + try: + raw, meta, url, path = _load_vre_file(table, year, sumlev, state, session, cache_dir, refresh) + except VREDownloadError as exc: + reason = _fallback_reason(exc) + sub_lines = sorted(o for (t, o) in VRE_TOTAL_FALLBACKS if t == table) + if not allow_substitute or reason is None or not sub_lines: + raise + need = wanted if wanted is not None else sub_lines + other = [o for o in need if o not in sub_lines] + if other or not need: + raise VREDownloadError( + f"{exc}\nNo verified substitute file exists for {table} line(s) {other or need} (substitutes " + f"exist only for {[f'{table}_{o:03d}' for o in sub_lines]})." + ) from exc + src_table = VRE_TOTAL_FALLBACKS[(table, need[0])][0] + try: + raw, meta, url, path = _load_vre_file(src_table, year, sumlev, state, session, cache_dir, refresh) + except VREDownloadError: + raise exc from None # the substitute is unavailable too: report the original failure + substitution = { + "requested_table": table, + "requested_url": getattr(exc, "url", None) or vre_url(table, year, sumlev, state), + "reason": reason, + "support_id": getattr(exc, "support_id", None), + "http_status": getattr(exc, "status_code", None), + "source_table": src_table, + "lines": {f"{table}_{o:03d}": "{}_{:03d}".format(*VRE_TOTAL_FALLBACKS[(table, o)]) for o in need}, + "source_url": url, + "source_sha256": meta.get("sha256"), + "verification": VRE_TOTAL_FALLBACK_VERIFICATION, + } + wanted = need frame = read_vre_csv(raw) + if substitution is not None: + parts = [] + for target, source in substitution["lines"].items(): + part = frame[frame["variable"] == source].copy() + part["variable"] = target + part["title"] = VRE_TOTAL_FALLBACK_TITLES[(table, int(target.rsplit("_", 1)[1]))] + parts.append(part) + attrs = dict(frame.attrs) + frame = pd.concat(parts, ignore_index=True) + frame.attrs.update(attrs) + elif wanted is not None: + frame = frame[frame["variable"].isin({f"{table}_{o:03d}" for o in wanted})].reset_index(drop=True) if state is not None and sumlev not in PER_STATE_SUMLEVS: frame = frame[frame["GEOID"].str.startswith(state)].reset_index(drop=True) frame.attrs.update( @@ -976,6 +1076,15 @@ def fetch_vre( cache_path=path, acquisition=meta.get("source", "download"), ) + if substitution is not None: + note = ( + f"{substitution['requested_url']} is unavailable ({substitution['reason']}); " + + ", ".join(f"{k} is {v}" for k, v in substitution["lines"].items()) + + f" of {url} (SHA-256 {substitution['source_sha256']}), relabelled. {VRE_TOTAL_FALLBACK_VERIFICATION}" + ) + frame.attrs["substitution"] = substitution + frame.attrs["notes"] = [f"substituted_source: {note}"] + warnings.warn(f"VRE substitute file used: {note}", stacklevel=2) if check_moe: report = verify_published_moe(frame) frame.attrs["moe_check"] = {k: v for k, v in report.items() if k != "mismatches"} diff --git a/docs/audits/acs_sampling_covariance.md b/docs/audits/acs_sampling_covariance.md index e371317..66f57f9 100644 --- a/docs/audits/acs_sampling_covariance.md +++ b/docs/audits/acs_sampling_covariance.md @@ -1,6 +1,6 @@ # ACS sampling-error covariance between areas (VRE replicates) -Generated 2026-11-09 06:54 UTC by `scripts/acs_error_covariance.py` (commit v0.14.0-29-g1847007-dirty, 10.0 min). PLAN_ROUND_16 workstream C (C1-C2). Status: **info** (gates nothing). Per-class numbers: `docs/audits/acs_sampling_covariance.csv` (written by the script; git-ignored like the other audit CSVs). +Generated 2026-11-09 15:28 UTC by `scripts/acs_error_covariance.py` (commit v0.15.0-1-g981e832-dirty, 9.2 min). PLAN_ROUND_16 workstream C (C1-C2). Status: **info** (gates nothing). Per-class numbers: `docs/audits/acs_sampling_covariance.csv` (written by the script; git-ignored like the other audit CSVs). ## Question @@ -38,6 +38,7 @@ Releases 2020-2024, 2015-2019; states Michigan (26) and New York (36). Files (ca | VRE | 2024 | county | `B17001.csv.zip` | `3d24fa18e9f005b3c9455d3bda4a4ab843eb8f443d423b56b00496b58ad0c605` | | VRE | 2024 | county | `B19001.csv.zip` | `95cb48eabcaef5c44fc68377f8b596f31cbb079d846bbea9c6bf7e83ba896d0f` | | boundary | 2024 | county | `cb_2024_us_county_500k.zip` | `da4051717caec55c75e3748c3608c2a3dbde8d1ff401bbaf4f952e3c3fb63ef1` | +| VRE | 2019 | tract | `B01001_26.csv.zip` (substitute for `B01003_26.csv.zip`) | `4ef59dbb65e02ce8de6779cd40d382f3e143a17079b1ffa56dd886f60d72970a` | | VRE | 2019 | tract | `B25003_26.csv.zip` | `0b6a3f85d252700385d92a435396f24750d659b89f114e3d6006e525231317d4` | | VRE | 2019 | tract | `B17001_26.csv.zip` | `8f6e7fb8d21f80e605d03ce4fab11fc3a372184cb1e550d64b84cf779569d729` | | VRE | 2019 | tract | `B19001_26.csv.zip` | `0cd0858b255228c403627e3bd08094950a56e3d15d414446fd77a719ecf99168` | @@ -53,15 +54,15 @@ Releases 2020-2024, 2015-2019; states Michigan (26) and New York (36). Files (ca | VRE | 2019 | county | `B19001.csv.zip` | `56374faae955dd44c9eec5f45c7b2d145781b6271028d714eae6ebafc2f260d3` | | boundary | 2019 | county | `cb_2019_us_county_500k.zip` | `1c4b703cd34c3f5e5e6368f91f1bd80de9281ef81ad91e005844848df9976c4b` | -**Unavailable files** (skipped; the variables that need them are not analysed for that state and release, everything else is): +**Substituted files** (census.replicate.fetch_vre, `VRE_TOTAL_FALLBACKS`): -| year | level | state | table | URL | reason | +| year | level | state | requested | reason | used instead | | :--- | :--- | :--- | :--- | :--- | :--- | -| 2019 | tract | Michigan | B01003 | https://www2.census.gov/programs-surveys/acs/replicate_estimates/2019/data/5-year/140/B01003_26.csv.zip | census.gov firewall 'Request Rejected' page (support ID 13979921158196382640), also on the single delayed retry | +| 2019 | tract | Michigan | https://www2.census.gov/programs-surveys/acs/replicate_estimates/2019/data/5-year/140/B01003_26.csv.zip | census.gov firewall 'Request Rejected' page (support ID 13979921158196382640) | B01003_001 = B01001_001 of https://www2.census.gov/programs-surveys/acs/replicate_estimates/2019/data/5-year/140/B01001_26.csv.zip | -A rejection of one specific URL can persist while every other file in the same directory downloads normally, so it is not retried further. To include such a file, download it in a browser and save it as `$CENSUS_VRE_DIR/2019/140/B01003_26.csv.zip` (default `$CENSUS_VRE_DIR` = `.cache/vre`); census.replicate then records its SHA-256 with source 'manual download' and the report marks it. +Basis: B01003 line 1 (Total population) equals B01001 line 1 (Total:) cell for cell: verified 2026-11-09 on the 2015-2019 New York tract VRE files (4,918 tracts; ESTIMATE, MOE and Var_Rep1-80 identical, 0 differing cells); re-checked by tests/test_replicate_live.py (CENSUS_LIVE_TESTS=1). The substitute file's SHA-256 is the one listed above. -Versions: census-acs-toolkit 0.14.0, python 3.14.4, numpy 2.5.4, scipy 1.18.1, pandas 3.1.1, geopandas 1.2.0, libpysal 4.15.1 +Versions: census-acs-toolkit 0.15.1, python 3.14.4, numpy 2.5.4, scipy 1.18.1, pandas 3.1.1, geopandas 1.2.0, libpysal 4.15.1 Areas analysed (after exclusions): @@ -89,6 +90,7 @@ Areas analysed (after exclusions): | 2024 | New York | county | poverty_rate | proportion | 62 | 62 | 0 | | | 2024 | New York | county | owner_share | proportion | 62 | 62 | 0 | | | 2024 | New York | county | median_hh_income | median | 62 | 62 | 0 | 0 top/bottom-coded medians excluded | +| 2019 | Michigan | tract | population | count | 2772 | 2754 | 18 | | | 2019 | Michigan | tract | households | count | 2772 | 2738 | 34 | | | 2019 | Michigan | tract | poverty_count | count | 2772 | 2736 | 36 | | | 2019 | Michigan | tract | poverty_rate | proportion | 2772 | 2734 | 8 | | @@ -111,7 +113,7 @@ Areas analysed (after exclusions): | 2019 | New York | county | owner_share | proportion | 62 | 62 | 0 | | | 2019 | New York | county | median_hh_income | median | 62 | 62 | 0 | 0 top/bottom-coded medians excluded | -Skipped: 2024 Michigan county population: 83 controlled (no sampling error); 2024 New York county population: 62 controlled (no sampling error); 2019 Michigan tract population: VRE file B01003 unavailable (see Data: unavailable files); 2019 Michigan county population: 83 controlled (no sampling error); 2019 New York county population: 62 controlled (no sampling error). +Skipped: 2024 Michigan county population: 83 controlled (no sampling error); 2024 New York county population: 62 controlled (no sampling error); 2019 Michigan county population: 83 controlled (no sampling error); 2019 New York county population: 62 controlled (no sampling error). ## Results 2020-2024 @@ -542,6 +544,7 @@ Mean pairwise r [95% interval] (number of pairs) by queen contiguity order. | state | variable | order 1 | order 2 | order 3 | beyond order 3 | | :--- | :--- | :--- | :--- | :--- | :--- | +| Michigan | population | -0.066 [-0.088, -0.044] (n=8661) | -0.014 [-0.017, -0.011] (n=20170) | -0.004 [-0.005, -0.003] (n=34283) | -0.000 [-0.001, -0.000] (n=235969) | | Michigan | households | 0.025 [0.016, 0.033] (n=8614) | 0.006 [0.000, 0.013] (n=20000) | 0.001 [-0.002, 0.005] (n=33974) | -0.000 [-0.001, 0.001] (n=234537) | | Michigan | poverty_count | -0.004 [-0.007, -0.001] (n=8601) | -0.002 [-0.004, 0.000] (n=19957) | 0.000 [-0.002, 0.002] (n=33911) | -0.000 [-0.000, 0.000] (n=234178) | | Michigan | poverty_rate | -0.000 [-0.003, 0.003] (n=8581) | -0.001 [-0.003, 0.001] (n=19916) | 0.000 [-0.002, 0.002] (n=33847) | 0.000 [-0.001, 0.001] (n=233863) | @@ -573,6 +576,7 @@ Mean pairwise r [95% interval] (number of pairs) by queen contiguity order. | state | variable | order 1 / within county | order 1 / across county | order 2 / within county | order 2 / across county | beyond order 3 / within county | beyond order 3 / across county | | :--- | :--- | :--- | :--- | :--- | :--- | :--- | :--- | +| Michigan | population | -0.075 [-0.103, -0.046] (n=7608) | -0.005 [-0.014, 0.004] (n=1053) | -0.019 [-0.026, -0.013] (n=14988) | 0.001 [-0.003, 0.005] (n=5182) | -0.001 [-0.001, 0.000] (n=146947) | -0.000 [-0.001, 0.001] (n=89022) | | Michigan | households | 0.028 [0.020, 0.035] (n=7563) | 0.001 [-0.012, 0.014] (n=1051) | 0.008 [0.001, 0.016] (n=14828) | 0.001 [-0.003, 0.005] (n=5172) | -0.001 [-0.003, 0.002] (n=146625) | 0.000 [-0.001, 0.001] (n=87912) | | Michigan | poverty_count | -0.005 [-0.008, -0.002] (n=7549) | 0.001 [-0.007, 0.010] (n=1052) | -0.001 [-0.003, 0.001] (n=14793) | -0.003 [-0.008, 0.001] (n=5164) | 0.000 [-0.000, 0.001] (n=146388) | -0.001 [-0.002, 0.000] (n=87790) | | Michigan | poverty_rate | -0.000 [-0.003, 0.002] (n=7531) | 0.001 [-0.007, 0.009] (n=1050) | 0.000 [-0.002, 0.003] (n=14758) | -0.003 [-0.007, 0.002] (n=5158) | 0.001 [0.000, 0.001] (n=146203) | -0.001 [-0.002, 0.000] (n=87660) | @@ -591,6 +595,7 @@ Tracts: | state | variable | order 1 rook (shared edge) | order 1 queen-only (corner) | | :--- | :--- | :--- | :--- | +| Michigan | population | -0.075 [-0.099, -0.051] (n=7076) | -0.028 [-0.035, -0.020] (n=1585) | | Michigan | households | 0.025 [0.017, 0.034] (n=7033) | 0.021 [0.009, 0.033] (n=1581) | | Michigan | poverty_count | -0.005 [-0.008, -0.001] (n=7018) | -0.002 [-0.008, 0.004] (n=1583) | | Michigan | poverty_rate | -0.000 [-0.003, 0.003] (n=7001) | 0.001 [-0.006, 0.007] (n=1580) | @@ -624,6 +629,7 @@ Tracts: | state | variable | 0-2 km | 2-5 km | 5-10 km | 10-20 km | 20-50 km | 50-100 km | 100-inf km | | :--- | :--- | :--- | :--- | :--- | :--- | :--- | :--- | :--- | +| Michigan | population | -0.039 [-0.078, 0.001] (n=3830) | -0.021 [-0.050, 0.008] (n=19110) | -0.008 [-0.019, 0.003] (n=35824) | -0.002 [-0.005, 0.001] (n=69777) | -0.001 [-0.004, 0.002] (n=91586) | -0.000 [-0.003, 0.002] (n=20557) | -0.000 [-0.001, 0.001] (n=58399) | | Michigan | households | 0.032 [0.006, 0.058] (n=3786) | 0.014 [0.005, 0.023] (n=18947) | 0.003 [0.001, 0.006] (n=35548) | -0.002 [-0.005, 0.002] (n=69531) | -0.001 [-0.003, 0.000] (n=91378) | -0.001 [-0.004, 0.002] (n=20313) | 0.001 [-0.001, 0.002] (n=57622) | | Michigan | poverty_count | -0.003 [-0.010, 0.004] (n=3789) | -0.002 [-0.005, 0.001] (n=18945) | -0.001 [-0.002, -0.000] (n=35516) | 0.000 [-0.001, 0.001] (n=69418) | 0.000 [-0.000, 0.001] (n=91157) | -0.000 [-0.003, 0.002] (n=20216) | -0.000 [-0.002, 0.001] (n=57606) | | Michigan | poverty_rate | 0.001 [-0.006, 0.008] (n=3778) | -0.001 [-0.002, 0.001] (n=18895) | -0.001 [-0.002, 0.000] (n=35416) | 0.001 [-0.000, 0.001] (n=69298) | 0.000 [-0.000, 0.001] (n=91079) | -0.001 [-0.003, 0.002] (n=20196) | -0.000 [-0.002, 0.001] (n=57545) | @@ -655,6 +661,7 @@ Counties: | state | variable | counties | tracts | pooled ratio (SE) | median county ratio | implied mean within-county r (SE) | | :--- | :--- | :--- | :--- | :--- | :--- | :--- | +| Michigan | population | 83 | 2754 | 0.000 (0.000) | 0.000 | -0.0046 (0.0000) | | Michigan | households | 82 | 2737 | 0.902 (0.053) | 0.798 | -0.0006 (0.0003) | | Michigan | poverty_count | 83 | 2736 | 0.886 (0.043) | 0.939 | -0.0006 (0.0002) | | New York | population | 62 | 4854 | 0.000 (0.000) | 0.000 | -0.0032 (0.0000) | @@ -669,11 +676,12 @@ Michigan x New York pairs (independent samples): | geography | variable | pairs | mean r (SE) | SD of r | 95% |r| | share |r| > analytic critical | | :--- | :--- | :--- | :--- | :--- | :--- | :--- | -| tract | households | 50000 | 0.0003 (0.0005) | 0.116 | 0.229 | 0.060 | -| tract | poverty_count | 50000 | -0.0005 (0.0004) | 0.115 | 0.231 | 0.062 | -| tract | poverty_rate | 50000 | -0.0007 (0.0004) | 0.116 | 0.234 | 0.063 | -| tract | owner_share | 50000 | -0.0002 (0.0004) | 0.115 | 0.226 | 0.058 | -| tract | median_hh_income | 50000 | 0.0014 (0.0005) | 0.116 | 0.227 | 0.059 | +| tract | population | 50000 | -0.0006 (0.0006) | 0.115 | 0.224 | 0.056 | +| tract | households | 50000 | 0.0000 (0.0004) | 0.116 | 0.228 | 0.060 | +| tract | poverty_count | 50000 | 0.0000 (0.0004) | 0.116 | 0.234 | 0.064 | +| tract | poverty_rate | 50000 | -0.0001 (0.0005) | 0.116 | 0.232 | 0.063 | +| tract | owner_share | 50000 | 0.0001 (0.0005) | 0.115 | 0.224 | 0.056 | +| tract | median_hh_income | 50000 | 0.0004 (0.0004) | 0.115 | 0.223 | 0.056 | | county | households | 5146 | 0.0285 (0.0028) | 0.120 | 0.240 | 0.078 | | county | poverty_count | 5146 | 0.0055 (0.0020) | 0.116 | 0.228 | 0.061 | | county | poverty_rate | 5146 | 0.0055 (0.0020) | 0.116 | 0.229 | 0.063 | @@ -684,6 +692,7 @@ Derangement null on far pairs (same state, beyond order 3): | state | geography | variable | pairs | mean | SD | 95% |r| | | :--- | :--- | :--- | :--- | :--- | :--- | :--- | +| Michigan | tract | population | 235969 | 0.0000 | 0.112 | 0.219 | | Michigan | tract | households | 234537 | 0.0004 | 0.113 | 0.222 | | Michigan | tract | poverty_count | 234178 | 0.0002 | 0.112 | 0.219 | | Michigan | tract | poverty_rate | 233863 | 0.0003 | 0.112 | 0.219 | @@ -710,6 +719,7 @@ Replicate design spectrum (lowest eigenvalues relative to the mean; Marchenko-Pa | state | geography | variable | lowest 4 eigenvalues (MP edge) | near-null | effective replicates | null SD 1/sqrt(eff.) | | :--- | :--- | :--- | :--- | :--- | :--- | :--- | +| Michigan | tract | population | 0.042 0.050 0.077 0.432 (0.688) | 3 | 77 | 0.114 | | Michigan | tract | households | 0.052 0.054 0.078 0.400 (0.687) | 3 | 77 | 0.114 | | Michigan | tract | poverty_count | 0.025 0.025 0.052 0.410 (0.687) | 3 | 77 | 0.114 | | Michigan | tract | poverty_rate | 0.026 0.026 0.054 0.409 (0.687) | 3 | 77 | 0.114 | @@ -736,50 +746,54 @@ Spread of pairwise r per class: SD, excess SD over the analytic null (sqrt(SD^2 | state | geography | variable | class | SD of r | excess SD (analytic) | excess SD (cross-state) | share |r| > crit | | :--- | :--- | :--- | :--- | :--- | :--- | :--- | :--- | +| Michigan | tract | population | order 1 | 0.185 | 0.148 | 0.145 | 0.163 | +| Michigan | tract | population | order 2 | 0.126 | 0.059 | 0.053 | 0.076 | +| Michigan | tract | population | order 3 | 0.119 | 0.040 | 0.031 | 0.063 | +| Michigan | tract | population | beyond order 3 | 0.115 | 0.028 | 0.012 | 0.058 | | Michigan | tract | households | order 1 | 0.140 | 0.085 | 0.079 | 0.115 | -| Michigan | tract | households | order 2 | 0.124 | 0.054 | 0.044 | 0.076 | -| Michigan | tract | households | order 3 | 0.119 | 0.041 | 0.026 | 0.066 | -| Michigan | tract | households | beyond order 3 | 0.117 | 0.034 | 0.011 | 0.061 | +| Michigan | tract | households | order 2 | 0.124 | 0.054 | 0.043 | 0.076 | +| Michigan | tract | households | order 3 | 0.119 | 0.041 | 0.025 | 0.066 | +| Michigan | tract | households | beyond order 3 | 0.117 | 0.034 | 0.010 | 0.061 | | Michigan | tract | poverty_count | order 1 | 0.114 | 0.024 | 0.000 | 0.061 | -| Michigan | tract | poverty_count | order 2 | 0.116 | 0.030 | 0.010 | 0.064 | -| Michigan | tract | poverty_count | order 3 | 0.115 | 0.029 | 0.007 | 0.064 | -| Michigan | tract | poverty_count | beyond order 3 | 0.116 | 0.031 | 0.013 | 0.064 | +| Michigan | tract | poverty_count | order 2 | 0.116 | 0.030 | 0.000 | 0.064 | +| Michigan | tract | poverty_count | order 3 | 0.115 | 0.029 | 0.000 | 0.064 | +| Michigan | tract | poverty_count | beyond order 3 | 0.116 | 0.031 | 0.008 | 0.064 | | Michigan | tract | poverty_rate | order 1 | 0.114 | 0.023 | 0.000 | 0.060 | | Michigan | tract | poverty_rate | order 2 | 0.116 | 0.030 | 0.000 | 0.062 | | Michigan | tract | poverty_rate | order 3 | 0.116 | 0.030 | 0.000 | 0.062 | -| Michigan | tract | poverty_rate | beyond order 3 | 0.116 | 0.030 | 0.005 | 0.063 | -| Michigan | tract | owner_share | order 1 | 0.115 | 0.027 | 0.000 | 0.058 | +| Michigan | tract | poverty_rate | beyond order 3 | 0.116 | 0.030 | 0.006 | 0.063 | +| Michigan | tract | owner_share | order 1 | 0.115 | 0.027 | 0.010 | 0.058 | | Michigan | tract | owner_share | order 2 | 0.115 | 0.025 | 0.000 | 0.058 | -| Michigan | tract | owner_share | order 3 | 0.115 | 0.028 | 0.006 | 0.059 | -| Michigan | tract | owner_share | beyond order 3 | 0.115 | 0.029 | 0.009 | 0.059 | +| Michigan | tract | owner_share | order 3 | 0.115 | 0.028 | 0.013 | 0.059 | +| Michigan | tract | owner_share | beyond order 3 | 0.115 | 0.029 | 0.014 | 0.059 | | Michigan | tract | median_hh_income | order 1 | 0.113 | 0.019 | 0.000 | 0.055 | -| Michigan | tract | median_hh_income | order 2 | 0.116 | 0.032 | 0.009 | 0.060 | -| Michigan | tract | median_hh_income | order 3 | 0.116 | 0.029 | 0.000 | 0.058 | -| Michigan | tract | median_hh_income | beyond order 3 | 0.115 | 0.027 | 0.000 | 0.057 | -| New York | tract | population | order 1 | 0.173 | 0.133 | - | 0.122 | -| New York | tract | population | order 3 | 0.118 | 0.038 | - | 0.060 | -| New York | tract | population | order 2 | 0.124 | 0.053 | - | 0.069 | -| New York | tract | population | beyond order 3 | 0.115 | 0.029 | - | 0.058 | +| Michigan | tract | median_hh_income | order 2 | 0.116 | 0.032 | 0.019 | 0.060 | +| Michigan | tract | median_hh_income | order 3 | 0.116 | 0.029 | 0.015 | 0.058 | +| Michigan | tract | median_hh_income | beyond order 3 | 0.115 | 0.027 | 0.009 | 0.057 | +| New York | tract | population | order 1 | 0.173 | 0.133 | 0.130 | 0.122 | +| New York | tract | population | order 3 | 0.118 | 0.038 | 0.028 | 0.060 | +| New York | tract | population | order 2 | 0.124 | 0.053 | 0.046 | 0.069 | +| New York | tract | population | beyond order 3 | 0.115 | 0.029 | 0.013 | 0.058 | | New York | tract | households | order 1 | 0.131 | 0.067 | 0.059 | 0.090 | -| New York | tract | households | order 3 | 0.118 | 0.038 | 0.021 | 0.065 | -| New York | tract | households | order 2 | 0.120 | 0.044 | 0.030 | 0.069 | -| New York | tract | households | beyond order 3 | 0.117 | 0.033 | 0.009 | 0.061 | -| New York | tract | poverty_count | order 1 | 0.115 | 0.028 | 0.005 | 0.063 | +| New York | tract | households | order 3 | 0.118 | 0.038 | 0.020 | 0.065 | +| New York | tract | households | order 2 | 0.120 | 0.044 | 0.029 | 0.069 | +| New York | tract | households | beyond order 3 | 0.117 | 0.033 | 0.006 | 0.061 | +| New York | tract | poverty_count | order 1 | 0.115 | 0.028 | 0.000 | 0.063 | | New York | tract | poverty_count | order 3 | 0.115 | 0.027 | 0.000 | 0.062 | -| New York | tract | poverty_count | order 2 | 0.116 | 0.029 | 0.008 | 0.064 | -| New York | tract | poverty_count | beyond order 3 | 0.115 | 0.029 | 0.007 | 0.064 | +| New York | tract | poverty_count | order 2 | 0.116 | 0.029 | 0.000 | 0.064 | +| New York | tract | poverty_count | beyond order 3 | 0.115 | 0.029 | 0.000 | 0.064 | | New York | tract | poverty_rate | order 1 | 0.115 | 0.027 | 0.000 | 0.066 | | New York | tract | poverty_rate | order 3 | 0.115 | 0.027 | 0.000 | 0.062 | | New York | tract | poverty_rate | order 2 | 0.115 | 0.029 | 0.000 | 0.062 | | New York | tract | poverty_rate | beyond order 3 | 0.115 | 0.029 | 0.000 | 0.063 | -| New York | tract | owner_share | order 1 | 0.115 | 0.025 | 0.000 | 0.059 | -| New York | tract | owner_share | order 3 | 0.116 | 0.029 | 0.009 | 0.059 | -| New York | tract | owner_share | order 2 | 0.115 | 0.028 | 0.003 | 0.059 | -| New York | tract | owner_share | beyond order 3 | 0.115 | 0.029 | 0.008 | 0.059 | -| New York | tract | median_hh_income | order 1 | 0.115 | 0.029 | 0.000 | 0.059 | -| New York | tract | median_hh_income | order 3 | 0.115 | 0.028 | 0.000 | 0.057 | -| New York | tract | median_hh_income | order 2 | 0.115 | 0.027 | 0.000 | 0.057 | -| New York | tract | median_hh_income | beyond order 3 | 0.115 | 0.027 | 0.000 | 0.058 | +| New York | tract | owner_share | order 1 | 0.115 | 0.025 | 0.003 | 0.059 | +| New York | tract | owner_share | order 3 | 0.116 | 0.029 | 0.014 | 0.059 | +| New York | tract | owner_share | order 2 | 0.115 | 0.028 | 0.012 | 0.059 | +| New York | tract | owner_share | beyond order 3 | 0.115 | 0.029 | 0.014 | 0.059 | +| New York | tract | median_hh_income | order 1 | 0.115 | 0.029 | 0.014 | 0.059 | +| New York | tract | median_hh_income | order 3 | 0.115 | 0.028 | 0.011 | 0.057 | +| New York | tract | median_hh_income | order 2 | 0.115 | 0.027 | 0.008 | 0.057 | +| New York | tract | median_hh_income | beyond order 3 | 0.115 | 0.027 | 0.009 | 0.058 | | Michigan | county | households | order 1 | 0.124 | 0.054 | 0.032 | 0.096 | | Michigan | county | households | order 3 | 0.115 | 0.027 | 0.000 | 0.064 | | Michigan | county | households | order 2 | 0.117 | 0.033 | 0.000 | 0.062 | @@ -827,6 +841,10 @@ Per class: half-A and half-B class means (random split), and the reliability of | state | geography | variable | class | pairs | mean (half A) | mean (half B) | reliability (random / odd-even / first-last) | | :--- | :--- | :--- | :--- | :--- | :--- | :--- | :--- | +| Michigan | tract | population | order 1 | 8661 | -0.067 | -0.064 | 0.64 / 0.65 / 0.64 | +| Michigan | tract | population | order 2 | 20170 | -0.013 | -0.015 | 0.21 / 0.22 / 0.22 | +| Michigan | tract | population | order 3 | 34283 | -0.005 | -0.004 | 0.12 / 0.11 / 0.10 | +| Michigan | tract | population | beyond order 3 | 235969 | -0.001 | -0.000 | 0.06 / 0.06 / 0.03 | | Michigan | tract | households | order 1 | 8614 | 0.022 | 0.027 | 0.37 / 0.37 / 0.35 | | Michigan | tract | households | order 2 | 20000 | 0.005 | 0.007 | 0.19 / 0.19 / 0.20 | | Michigan | tract | households | order 3 | 33974 | 0.001 | 0.002 | 0.11 / 0.12 / 0.13 | @@ -918,6 +936,18 @@ Mean r per class in each release (different tract vintages: 2010 tracts in 2015- | state | geography | variable | class | 2019 | 2024 | difference (SE) | | :--- | :--- | :--- | :--- | :--- | :--- | :--- | +| Michigan | tract | population | order 1 | -0.066 [-0.088, -0.044] | -0.071 [-0.090, -0.052] | -0.004 (0.015) | +| Michigan | tract | population | order 2 | -0.014 [-0.017, -0.011] | -0.014 [-0.017, -0.011] | 0.000 (0.002) | +| Michigan | tract | population | order 3 | -0.004 [-0.005, -0.003] | -0.004 [-0.005, -0.003] | 0.000 (0.001) | +| Michigan | tract | population | beyond order 3 | -0.000 [-0.001, -0.000] | -0.001 [-0.001, 0.000] | -0.000 (0.000) | +| Michigan | tract | population | order 1 / within county | -0.075 [-0.103, -0.046] | -0.081 [-0.106, -0.056] | -0.006 (0.019) | +| Michigan | tract | population | order 2 / within county | -0.019 [-0.026, -0.013] | -0.018 [-0.024, -0.012] | 0.001 (0.005) | +| Michigan | tract | population | order 2 / across county | 0.001 [-0.003, 0.005] | -0.001 [-0.005, 0.002] | -0.002 (0.003) | +| Michigan | tract | population | order 1 / across county | -0.005 [-0.014, 0.004] | 0.003 [-0.007, 0.014] | 0.008 (0.007) | +| Michigan | tract | population | order 3 / across county | -0.000 [-0.003, 0.002] | 0.001 [-0.001, 0.003] | 0.001 (0.001) | +| Michigan | tract | population | order 3 / within county | -0.007 [-0.009, -0.004] | -0.007 [-0.009, -0.006] | -0.001 (0.001) | +| Michigan | tract | population | beyond order 3 / within county | -0.001 [-0.001, 0.000] | -0.001 [-0.001, -0.000] | -0.000 (0.000) | +| Michigan | tract | population | beyond order 3 / across county | -0.000 [-0.001, 0.001] | 0.000 [-0.001, 0.001] | 0.000 (0.001) | | Michigan | tract | households | order 1 | 0.025 [0.016, 0.033] | -0.025 [-0.033, -0.017] | -0.050 (0.006) | | Michigan | tract | households | order 2 | 0.006 [0.000, 0.013] | -0.008 [-0.011, -0.004] | -0.014 (0.004) | | Michigan | tract | households | order 3 | 0.001 [-0.002, 0.005] | -0.003 [-0.005, -0.002] | -0.005 (0.002) | @@ -1151,19 +1181,27 @@ stated). r is the replicate correlation of two areas' sampling errors. scored on held-out replicates after removing the design directions, it is worse than independence for every variable (-25 to -50 nats for tracts). Without that projection it appeared to gain 30-90 nats, entirely from the cross-half dependence created by the near-null design directions. -6. **Stability across releases (tracts and counties, 2015-2019 vs 2020-2024).** One 2015-2019 file is - missing: Michigan tract total population (`2019/data/5-year/140/B01003_26.csv.zip`). census.gov's - firewall answered that one URL with its 'Request Rejected' page (support ID 13979921158196382640) on - the first request and on the single delayed retry, while every other 2019 tract file, including New - York's B01003_36 from the same directory, downloaded normally; the rejection is URL-specific, not - rate limiting (see Data: unavailable files). So the 2015-2019 tract comparison has no Michigan - population row; everything else was analysed. Results: (a) **Uncontrolled variables are stable.** +6. **Stability across releases (tracts and counties, 2015-2019 vs 2020-2024).** One 2015-2019 file, + Michigan tract total population (`2019/data/5-year/140/B01003_26.csv.zip`), is refused by census.gov's + firewall ('Request Rejected', support ID 13979921158196382640) for every client, while every other + 2019 tract file, including New York's B01003_36 from the same directory, downloads normally. Its only + line is read instead from line 1 ("Total:") of the same release's Michigan tract B01001 file + (B01001_26; see Data: substituted files). The two lines are the same total population: in the New York + 2015-2019 tract files B01003 line 1 and B01001 line 1 agree in ESTIMATE, MOE and all 80 replicates for + all 4,918 tracts (0 differing cells), a check re-run by tests/test_replicate_live.py. All variables are + therefore analysed in both states and releases. Results: (a) **Uncontrolled variables are stable.** Poverty count, poverty rate, owner share and median income stay at 0 to within +/-0.005 at every order in both states and releases; of 108 tract class comparisons for them, 3 differ by more than 2 combined - SEs (about 5 expected by chance; largest |z| 2.5, all differences <= 0.004). (b) **Population (New - York) is stable**: order 1 -0.039 [-0.060, -0.018] in 2015-2019 vs -0.040 [-0.060, -0.020] (within - county -0.042 vs -0.043), the same fast decay, a county aggregation ratio of 0.000 again (tract errors - sum to zero within a county), and split-half pair reliability 0.60 (2020-2024: 0.61). (c) + SEs (about 5 expected by chance; largest |z| 2.5, all differences <= 0.004). (b) **Population is stable + in both states.** New York: order 1 -0.039 [-0.060, -0.018] in 2015-2019 vs -0.040 [-0.060, -0.020] + (within county -0.042 vs -0.043), the same fast decay, a county aggregation ratio of 0.000 again (tract + errors sum to zero within a county), and split-half pair reliability 0.60 (2020-2024: 0.61). Michigan + (B01001 line 1): order 1 -0.066 [-0.088, -0.044] vs -0.071 [-0.090, -0.052]; within county -0.075 + [-0.103, -0.046] vs -0.081 [-0.106, -0.056]; across county -0.005 [-0.014, 0.004] vs +0.003 [-0.007, + 0.014]; order 2 within county -0.019 vs -0.018; rook vs corner-only -0.075 vs -0.028 (2020-2024: -0.080 + vs -0.029); aggregation ratio 0.000 in both releases; split-half pair reliability at order 1 0.64 + (2020-2024: 0.67). Every Michigan population class difference between releases is at most 0.008 + (order 1 across county, 1.1 combined SEs). As in 2020-2024, Michigan's within-county order-1 correlation is nearly twice New York's. (c) **Households change sign.** Occupied housing units (B25003_001) in 2015-2019 have *positive* mean r between neighbouring tracts in the same county - Michigan order 1 +0.025 [0.016, 0.033] (within county +0.028), New York +0.012 [0.003, 0.020] - against -0.025 and -0.015 in 2020-2024 (differences 8.4 and @@ -1209,8 +1247,8 @@ analyses map it is indistinguishable from zero.** stress tests outside the observed range, not plausible ACS conditions for these variables; the realistic end of the grid is -0.1..0.05. Correlated-error scenarios with large positive rho should be stated as hypothetical (for example for variables or survey designs not studied here). -* Limits: two states, two releases at tract level (2015-2019 without Michigan population: firewall - rejection of that one file), 80 replicates (about 77 effective), and counties as a proxy for weighting +* Limits: two states, two releases at tract level (2015-2019 Michigan population read from B01001 line + 1, verified identical to B01003 line 1; Findings 6), 80 replicates (about 77 effective), and counties as a proxy for weighting areas. The 95% |r| of 0.23 for a single pair means no individual pair can be flagged; only class means are evidence. diff --git a/scripts/acs_error_covariance.py b/scripts/acs_error_covariance.py index 1f3815b..a6a4f72 100644 --- a/scripts/acs_error_covariance.py +++ b/scripts/acs_error_covariance.py @@ -118,6 +118,8 @@ def _load_tables(year, level, state, files): files.append({"kind": "VRE", "year": year, "level": level, "state": state, "table": t, "url": f.attrs["source_url"], "sha256": f.attrs["sha256"], "acquisition": f.attrs.get("acquisition", "download")}) + if f.attrs.get("substitution"): # e.g. B01003 total read from B01001 (census.replicate.fetch_vre) + files[-1]["substitution"] = f.attrs["substitution"] frames[t] = f return frames @@ -519,8 +521,20 @@ def write_report(df, extras, files, years, elapsed, findings, conclusion): seen.add(f["url"]) name = f"`{f['url'].rsplit('/', 1)[-1]}`" + (" (manual download)" if f.get("acquisition") == "manual download" else "") + if f.get("substitution"): + name += f" (substitute for `{f['substitution']['requested_url'].rsplit('/', 1)[-1]}`)" rows.append([f["kind"], f["year"], f["level"], name, f"`{f['sha256']}`"]) L.append(_md_table(["kind", "year", "level", "file", "SHA-256"], rows) + "\n") + subs = [f for f in files if f.get("substitution")] + if subs: + L.append("**Substituted files** (census.replicate.fetch_vre, `VRE_TOTAL_FALLBACKS`):\n") + L.append(_md_table(["year", "level", "state", "requested", "reason", "used instead"], + [[f["year"], f["level"], STATES.get(f["state"], f["state"]), + f["substitution"]["requested_url"], f["substitution"]["reason"], + ", ".join(f"{k} = {v} of {f['substitution']['source_url']}" + for k, v in f["substitution"]["lines"].items())] for f in subs]) + "\n") + L.append("Basis: " + subs[0]["substitution"]["verification"] + " The substitute file's SHA-256 is the one " + "listed above.\n") missing = [f for f in files if f.get("unavailable")] if missing: L.append("**Unavailable files** (skipped; the variables that need them are not analysed for that state " @@ -816,19 +830,27 @@ def interpret(df, extras, years): scored on held-out replicates after removing the design directions, it is worse than independence for every variable (-25 to -50 nats for tracts). Without that projection it appeared to gain 30-90 nats, entirely from the cross-half dependence created by the near-null design directions. -6. **Stability across releases (tracts and counties, 2015-2019 vs 2020-2024).** One 2015-2019 file is - missing: Michigan tract total population (`2019/data/5-year/140/B01003_26.csv.zip`). census.gov's - firewall answered that one URL with its 'Request Rejected' page (support ID 13979921158196382640) on - the first request and on the single delayed retry, while every other 2019 tract file, including New - York's B01003_36 from the same directory, downloaded normally; the rejection is URL-specific, not - rate limiting (see Data: unavailable files). So the 2015-2019 tract comparison has no Michigan - population row; everything else was analysed. Results: (a) **Uncontrolled variables are stable.** +6. **Stability across releases (tracts and counties, 2015-2019 vs 2020-2024).** One 2015-2019 file, + Michigan tract total population (`2019/data/5-year/140/B01003_26.csv.zip`), is refused by census.gov's + firewall ('Request Rejected', support ID 13979921158196382640) for every client, while every other + 2019 tract file, including New York's B01003_36 from the same directory, downloads normally. Its only + line is read instead from line 1 ("Total:") of the same release's Michigan tract B01001 file + (B01001_26; see Data: substituted files). The two lines are the same total population: in the New York + 2015-2019 tract files B01003 line 1 and B01001 line 1 agree in ESTIMATE, MOE and all 80 replicates for + all 4,918 tracts (0 differing cells), a check re-run by tests/test_replicate_live.py. All variables are + therefore analysed in both states and releases. Results: (a) **Uncontrolled variables are stable.** Poverty count, poverty rate, owner share and median income stay at 0 to within +/-0.005 at every order in both states and releases; of 108 tract class comparisons for them, 3 differ by more than 2 combined - SEs (about 5 expected by chance; largest |z| 2.5, all differences <= 0.004). (b) **Population (New - York) is stable**: order 1 -0.039 [-0.060, -0.018] in 2015-2019 vs -0.040 [-0.060, -0.020] (within - county -0.042 vs -0.043), the same fast decay, a county aggregation ratio of 0.000 again (tract errors - sum to zero within a county), and split-half pair reliability 0.60 (2020-2024: 0.61). (c) + SEs (about 5 expected by chance; largest |z| 2.5, all differences <= 0.004). (b) **Population is stable + in both states.** New York: order 1 -0.039 [-0.060, -0.018] in 2015-2019 vs -0.040 [-0.060, -0.020] + (within county -0.042 vs -0.043), the same fast decay, a county aggregation ratio of 0.000 again (tract + errors sum to zero within a county), and split-half pair reliability 0.60 (2020-2024: 0.61). Michigan + (B01001 line 1): order 1 -0.066 [-0.088, -0.044] vs -0.071 [-0.090, -0.052]; within county -0.075 + [-0.103, -0.046] vs -0.081 [-0.106, -0.056]; across county -0.005 [-0.014, 0.004] vs +0.003 [-0.007, + 0.014]; order 2 within county -0.019 vs -0.018; rook vs corner-only -0.075 vs -0.028 (2020-2024: -0.080 + vs -0.029); aggregation ratio 0.000 in both releases; split-half pair reliability at order 1 0.64 + (2020-2024: 0.67). Every Michigan population class difference between releases is at most 0.008 + (order 1 across county, 1.1 combined SEs). As in 2020-2024, Michigan's within-county order-1 correlation is nearly twice New York's. (c) **Households change sign.** Occupied housing units (B25003_001) in 2015-2019 have *positive* mean r between neighbouring tracts in the same county - Michigan order 1 +0.025 [0.016, 0.033] (within county +0.028), New York +0.012 [0.003, 0.020] - against -0.025 and -0.015 in 2020-2024 (differences 8.4 and @@ -873,8 +895,8 @@ def interpret(df, extras, years): stress tests outside the observed range, not plausible ACS conditions for these variables; the realistic end of the grid is -0.1..0.05. Correlated-error scenarios with large positive rho should be stated as hypothetical (for example for variables or survey designs not studied here). -* Limits: two states, two releases at tract level (2015-2019 without Michigan population: firewall - rejection of that one file), 80 replicates (about 77 effective), and counties as a proxy for weighting +* Limits: two states, two releases at tract level (2015-2019 Michigan population read from B01001 line + 1, verified identical to B01003 line 1; Findings 6), 80 replicates (about 77 effective), and counties as a proxy for weighting areas. The 95% |r| of 0.23 for a single pair means no individual pair can be flagged; only class means are evidence. """ diff --git a/tests/test_http.py b/tests/test_http.py index 0e6cdf6..e5cc18a 100644 --- a/tests/test_http.py +++ b/tests/test_http.py @@ -158,7 +158,8 @@ def test_one_delayed_retry_then_clear_error(self): page = FakeResponse(REJECTED_PAGE, headers={"Content-Type": "text/html"}) s = FakeSession([page, page, FakeResponse(b"never reached")]) with self.assertRaises(rp.VREDownloadError) as ctx: - rp.fetch_vre("B01003", 2019, "tract", "26", session=s, cache_dir=self.tmp, check_moe=False) + rp.fetch_vre("B01003", 2019, "tract", "26", session=s, cache_dir=self.tmp, check_moe=False, + allow_substitute=False) # the B01001 substitute: tests/test_replicate.py exc = ctx.exception path = os.path.join(self.tmp, "140", "B01003_26.csv.zip") self.assertEqual(len(s.calls), 2) # never loops diff --git a/tests/test_replicate.py b/tests/test_replicate.py index 7aba26a..374c745 100644 --- a/tests/test_replicate.py +++ b/tests/test_replicate.py @@ -563,5 +563,157 @@ def test_pre_2018_has_no_csv(self): self.assertEqual(session.calls, []) + +REJECTED_PAGE = (b"Request RejectedThe requested URL was rejected. " + b"Your support ID is: 13979921158196382640

") + + +def _vre_zip(table, title, lines, seed=0): + """Synthetic 2019 tract VRE zip for Michigan: ``lines`` = [(ORDER, TITLE)], 3 tracts, MOEs consistent.""" + rng = np.random.default_rng(seed) + head = ["TBLID", "GEOID", "NAME", "ORDER", "TITLE", "ESTIMATE", "MOE", "CME", "SE"] + [ + f"Var_Rep{i}" for i in range(1, 81)] + out = [",".join(head), f"{table},,,,{title}" + "," * (len(head) - 5), + f"{table},,,,Universe: Total population" + "," * (len(head) - 5)] + for k in range(3): + for order, ltitle in lines: + est = 1000 + 300 * k + 10 * order + reps = est + rng.integers(-60, 61, 80) + se = math.sqrt(4.0 / 80.0 * float(np.sum((reps - est) ** 2))) + moe = round(Z90 * se) + out.append(",".join([table, f"14000US2616300{k}100", f"\"Census Tract {k}, Wayne County, Michigan\"", + str(order), ltitle, str(est), str(moe), f"+/-{moe}", str(round(se))] + + [str(int(r)) for r in reps])) + buf = io.BytesIO() + with zipfile.ZipFile(buf, "w") as zf: + zf.writestr(f"{table}_26.csv", "\n".join(out) + "\n") + return buf.getvalue() + + +class TestTotalFallback(unittest.TestCase): + """B01003 line 1 <- B01001 line 1 when census.gov rejects (or 404s) the B01003 file.""" + + def setUp(self): + self.tmp = tempfile.mkdtemp() + self.url3 = rp.vre_url("B01003", 2019, "tract", "26") + self.url1 = rp.vre_url("B01001", 2019, "tract", "26") + # Identical pair: same seed, same line-1 values; B01001 has more lines after line 1. + self.b01003 = _vre_zip("B01003", "TOTAL POPULATION", [(1, "Total")], seed=3) + self.b01001 = _vre_zip("B01001", "SEX BY AGE", [(1, "Total")], seed=3) + self.b01001 = self._add_lines(self.b01001) + + def tearDown(self): + shutil.rmtree(self.tmp, ignore_errors=True) + + @staticmethod + def _add_lines(raw): + """B01001 copy whose line 1 is titled 'Total:' and which has a line 2 per tract.""" + with zipfile.ZipFile(io.BytesIO(raw)) as zf: + rows = zf.read("B01001_26.csv").decode().rstrip("\n").split("\n") + out = rows[:3] + for r in rows[3:]: + out.append(r.replace(",1,Total,", ",1,Total:,")) + parts = r.replace(",1,Total,", ",2,Male:,").split(",") + out.append(",".join(parts)) + buf = io.BytesIO() + with zipfile.ZipFile(buf, "w") as zf: + zf.writestr("B01001_26.csv", "\n".join(out) + "\n") + return buf.getvalue() + + def _fetch(self, session, **kw): + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + frame = rp.fetch_vre("B01003", 2019, "tract", "26", session=session, cache_dir=self.tmp, **kw) + return frame, [str(w.message) for w in caught] + + def _check_substituted(self, frame, msgs, reason): + sub = frame.attrs["substitution"] + self.assertEqual(sub["requested_table"], "B01003") + self.assertEqual(sub["requested_url"], self.url3) + self.assertEqual(sub["source_table"], "B01001") + self.assertEqual(sub["lines"], {"B01003_001": "B01001_001"}) + self.assertEqual(sub["source_url"], self.url1) + self.assertEqual(sub["source_sha256"], hashlib.sha256(self.b01001).hexdigest()) + self.assertIn(reason, sub["reason"]) + self.assertIn("0 differing cells", sub["verification"]) + self.assertEqual(frame.attrs["source_url"], self.url1) # the file actually read + self.assertEqual(frame.attrs["sha256"], sub["source_sha256"]) + self.assertEqual(frame.attrs["table"], "B01003") + self.assertTrue(frame.attrs["notes"][0].startswith("substituted_source:")) + self.assertTrue(any("substitute" in m and self.url1 in m for m in msgs)) # never silent + self.assertEqual(set(frame["variable"]), {"B01003_001"}) + self.assertEqual(set(frame["title"]), {"Total"}) + self.assertTrue(frame.attrs["moe_check"]["passed"]) + # The B01003 path holds nothing: the substitute is not cached under the requested name. + self.assertFalse(os.path.exists(os.path.join(self.tmp, "140", "B01003_26.csv.zip"))) + + def test_rejection_uses_b01001_with_provenance(self): + page = FakeResponse(REJECTED_PAGE, ctype="text/html") + frame, msgs = self._fetch(FakeSession({self.url3: page, self.url1: FakeResponse(self.b01001)})) + self._check_substituted(frame, msgs, "13979921158196382640") + self.assertEqual(frame.attrs["substitution"]["support_id"], "13979921158196382640") + self.assertIsNone(frame.attrs["substitution"]["http_status"]) + + def test_404_uses_b01001(self): + frame, msgs = self._fetch(FakeSession({self.url1: FakeResponse(self.b01001)})) + self._check_substituted(frame, msgs, "404") + self.assertEqual(frame.attrs["substitution"]["http_status"], 404) + # Cached B01001 is reused on the next call (B01003 is requested again, then B01001 from the cache). + s2 = FakeSession({}) + again, _ = self._fetch(s2) + self.assertEqual(s2.calls, [self.url3]) + pd.testing.assert_frame_equal(again, frame) + + def test_relabelled_frame_equals_b01003_frame(self): + direct = rp.fetch_vre("B01003", 2019, "tract", "26", cache_dir=self.tmp, + session=FakeSession({self.url3: FakeResponse(self.b01003)})) + shutil.rmtree(os.path.join(self.tmp, "140")) + frame, _ = self._fetch(FakeSession({self.url1: FakeResponse(self.b01001)})) + pd.testing.assert_frame_equal(frame, direct, check_like=False) + + def test_substitute_unavailable_raises_original_error(self): + page = FakeResponse(REJECTED_PAGE, ctype="text/html") + with self.assertRaises(rp.VREDownloadError) as ctx: + self._fetch(FakeSession({self.url3: page})) # B01001: 404 + self.assertTrue(ctx.exception.rejected) + self.assertEqual(ctx.exception.url, self.url3) + self.assertIn("13979921158196382640", str(ctx.exception)) + with self.assertRaises(rp.VREDownloadError) as ctx: + self._fetch(FakeSession({})) + self.assertIn(self.url3, str(ctx.exception)) + + def test_other_lines_tables_and_opt_out_do_not_fall_back(self): + page = FakeResponse(REJECTED_PAGE, ctype="text/html") + session = FakeSession({self.url3: page, self.url1: FakeResponse(self.b01001)}) + with self.assertRaises(rp.VREDownloadError) as ctx: + self._fetch(session, lines=[1, 2]) + self.assertIn("No verified substitute", str(ctx.exception)) + self.assertIn("[2]", str(ctx.exception)) + self.assertNotIn(self.url1, session.calls) + with self.assertRaises(rp.VREDownloadError): + self._fetch(session, allow_substitute=False) + self.assertNotIn(self.url1, session.calls) + url = rp.vre_url("B01001", 2019, "tract", "26") + with self.assertRaises(rp.VREDownloadError): # B01001 itself has no substitute + rp.fetch_vre("B01001", 2019, "tract", "26", session=FakeSession({url: page}), cache_dir=self.tmp) + # Line 1 requested explicitly: substitute used. + frame, _ = self._fetch(session, lines=[1]) + self.assertIn("substitution", frame.attrs) + + def test_other_failures_do_not_fall_back(self): + session = FakeSession({self.url3: FakeResponse(b"", status=500, ctype="text/html"), + self.url1: FakeResponse(self.b01001)}) + with self.assertRaises(rp.VREDownloadError) as ctx: + self._fetch(session) + self.assertIn("HTTP 500", str(ctx.exception)) + self.assertNotIn(self.url1, session.calls) + + def test_lines_filter_without_substitution(self): + session = FakeSession({self.url1: FakeResponse(self.b01001)}) + frame = rp.fetch_vre("B01001", 2019, "tract", "26", session=session, cache_dir=self.tmp, lines=[2]) + self.assertEqual(set(frame["variable"]), {"B01001_002"}) + self.assertNotIn("substitution", frame.attrs) + + if __name__ == "__main__": unittest.main() diff --git a/tests/test_replicate_live.py b/tests/test_replicate_live.py index 04cf4f6..102a5f2 100644 --- a/tests/test_replicate_live.py +++ b/tests/test_replicate_live.py @@ -64,6 +64,23 @@ def test_gzip_release(self): self.assertTrue(check["passed"], check) self.assertEqual(check["controlled_rows"], 52) # state total population is controlled + def test_b01003_total_equals_b01001_total(self): + """Basis of the B01003 <- B01001 substitute (census.replicate.VRE_TOTAL_FALLBACKS): line 1 identical.""" + from census.replicate import REPLICATE_COLUMNS + + kw = dict(check_moe=False, allow_substitute=False) + b3 = self._fetch("B01003", 2019, "tract", state_code="36", **kw) + b1 = self._fetch("B01001", 2019, "tract", state_code="36", lines=[1], **kw) + self.assertNotIn("substitution", b3.attrs) + self.assertEqual(len(b3), len(b1)) + self.assertGreater(len(b3), 4000) + cols = ["GEOID", "NAME", "estimate", "moe", "se_published", "controlled"] + REPLICATE_COLUMNS + left = b3.set_index("GEOID", drop=False)[cols].sort_index() + right = b1.set_index("GEOID", drop=False)[cols].sort_index() + self.assertTrue(left.index.equals(right.index)) + differing = int(((left != right) & ~(left.isna() & right.isna())).to_numpy().sum()) + self.assertEqual(differing, 0) + def test_published_table_list_matches_registry(self): from census.replicate import vre_tables