diff --git a/.github/country_main_tests.py b/.github/country_main_tests.py new file mode 100644 index 000000000..d2fb15c75 --- /dev/null +++ b/.github/country_main_tests.py @@ -0,0 +1,162 @@ +#!/usr/bin/env python +"""Run policyengine-us main's Core-facing tests against the Core under test. + +The country smoke job installs a pinned policyengine-us release and checks +that its system loads, so it cannot see a policyengine-us test that a Core +change breaks. On 2026-10-27, 3.33.13 broke policyengine-us main's +test_social_security_input_state_machine.py within minutes of its release. + +The PolicyEngineUSMain job checks out policyengine-us main and runs the files +in .github/requirements/policyengine-us-main-tests.txt twice, each with a +JUnit report: first with the Core release on PyPI, then with this checkout. + +- `select LIST REPO` prints the listed files that exist in REPO, one per + line, and warns about the rest, which policyengine-us renamed or removed. +- `compare RELEASED CANDIDATE` fails on each test that fails with this Core + but did not fail with the release. A test that fails with both is a + policyengine-us failure, so it is only a warning: policyengine-us main is + sometimes red for its own reasons. +""" + +import argparse +import os +import sys +from pathlib import Path +from xml.etree import ElementTree + + +def listed_files(list_path): + """The paths in a list file, without comments or blank lines.""" + paths = [] + for line in Path(list_path).read_text().splitlines(): + line = line.split("#", 1)[0].strip() + if line: + paths.append(line) + return paths + + +def select(list_path, repo): + """The listed files that exist in the checkout, and those that do not.""" + present, missing = [], [] + for path in listed_files(list_path): + (present if (Path(repo) / path).is_file() else missing).append(path) + return present, missing + + +def outcomes(report): + """Each test's outcome in a JUnit report: passed, failed or skipped. + + A test can appear twice, as when its teardown errors after it passed; + a failure in either entry counts. + """ + results = {} + for case in ElementTree.parse(report).getroot().iter("testcase"): + test = f"{case.get('classname', '')}::{case.get('name', '')}" + tags = {child.tag for child in case} + if tags & {"failure", "error"}: + outcome = "failed" + elif "skipped" in tags: + outcome = "skipped" + else: + outcome = "passed" + if results.get(test) != "failed": + results[test] = outcome + return results + + +def compare(released, candidate): + """Tests this Core breaks, and tests that fail with both Cores.""" + broken = sorted( + test + for test, outcome in candidate.items() + if outcome == "failed" and released.get(test) != "failed" + ) + both = sorted( + test + for test, outcome in candidate.items() + if outcome == "failed" and released.get(test) == "failed" + ) + return broken, both + + +def summary(released, candidate, broken, both): + lines = [ + "### policyengine-us main against this Core", + "", + f"{len(candidate)} tests ran with this Core and {len(released)} with " + "the Core release.", + "", + ] + if broken: + lines.append(f"**This Core breaks {len(broken)}:**") + lines += [f"- `{test}`" for test in broken] + lines.append("") + if both: + lines.append(f"Failing with both Cores (policyengine-us's own): {len(both)}") + lines += [f"- `{test}`" for test in both] + if not broken and not both: + lines.append("Every test passes with this Core.") + return "\n".join(lines) + "\n" + + +def run_compare(released_report, candidate_report, summary_path=None): + if not Path(candidate_report).exists(): + print( + "::error::This Core's run wrote no JUnit report: pytest could not " + "start or was killed." + ) + return 1 + if not Path(released_report).exists(): + print( + "::warning::The Core release's run wrote no JUnit report, so " + "nothing can be blamed on this Core." + ) + return 0 + released = outcomes(released_report) + candidate = outcomes(candidate_report) + broken, both = compare(released, candidate) + for test in both: + print(f"::warning::Fails with the Core release too: {test}") + for test in broken: + print(f"::error::This Core breaks {test}") + text = summary(released, candidate, broken, both) + print(text) + if summary_path: + with open(summary_path, "a") as output: + output.write(text) + return 1 if broken else 0 + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + commands = parser.add_subparsers(dest="command", required=True) + chooser = commands.add_parser("select", help="list the files to run") + chooser.add_argument("list") + chooser.add_argument("repo") + checker = commands.add_parser("compare", help="compare the two runs") + checker.add_argument("released") + checker.add_argument("candidate") + checker.add_argument( + "--summary", + default=os.environ.get("GITHUB_STEP_SUMMARY"), + help="file to append a Markdown summary to", + ) + args = parser.parse_args(argv) + if args.command == "select": + present, missing = select(args.list, args.repo) + for path in missing: + print( + f"::warning::policyengine-us main has no {path}; update " + ".github/requirements/policyengine-us-main-tests.txt.", + file=sys.stderr, + ) + if not present: + print("::error::None of the listed files exist.", file=sys.stderr) + return 1 + print("\n".join(present)) + return 0 + return run_compare(args.released, args.candidate, args.summary) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/requirements/policyengine-us-main-tests.txt b/.github/requirements/policyengine-us-main-tests.txt new file mode 100644 index 000000000..84eb2e5e5 --- /dev/null +++ b/.github/requirements/policyengine-us-main-tests.txt @@ -0,0 +1,26 @@ +# policyengine-us test files the PolicyEngineUSMain job runs from a checkout of +# policyengine-us main, first with the Core release on PyPI and then with the +# Core under test (.github/country_main_tests.py). Paths are relative to the +# policyengine-us repository root. +# +# Each file drives Core internals (simulations, holders, data storage, tools, +# populations, parameters or reforms) and is fast: together they took about +# 95 seconds in policyengine-us release run 40544965026. The state machine test +# runs Core's installed source bodies in namespaces it builds; Core 3.33.13 +# broke it on 2026-10-27, which this job would have shown on the PR. +# +# A file policyengine-us renames or removes is skipped with a warning, so keep +# the list in step when that happens. +policyengine_us/tests/core/test_social_security_input_state_machine.py +policyengine_us/tests/core/test_social_security_real_core.py +policyengine_us/tests/core/test_loss_ald_extraction_parity.py +policyengine_us/tests/core/test_capital_gains_rate_worksheet_properties.py +policyengine_us/tests/core/test_ald_determinism.py +policyengine_us/tests/core/test_pandas3_compatibility.py +policyengine_us/tests/core/test_social_security_lever_properties.py +policyengine_us/tests/test_locality_sales_tax_rates.py +policyengine_us/tests/test_parameter_scale_thresholds.py +policyengine_us/tests/code_health/test_no_group_variables_in_person_adds.py +policyengine_us/tests/microsimulation/data/test_extend_single_year_dataset.py +policyengine_us/tests/policy/baseline/parameters/test_uprating_extensions.py +policyengine_us/tests/policy/baseline/gov/states/or/tax/income/credits/test_or_ctc_sunset.py diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index c90189cb6..7df05951e 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -4,10 +4,10 @@ on: branches: [master] # A new push to a pull request supersedes the run on its previous head. Cancel -# that run instead of letting its 19 jobs (8 Test, 8 smoke, 3 checks) keep -# holding org-wide shared runners. From 2026-10-06 to 10-08, 21 of 55 PR runs -# were superseded while still running, and finishing them used 850 -# runner-minutes (26% of this workflow). +# that run instead of letting its 20 jobs (8 Test, 8 smoke, 1 policyengine-us +# main, 3 checks) keep holding org-wide shared runners. From 2026-10-06 to +# 10-08, 21 of 55 PR runs were superseded while still running, and finishing +# them used 850 runner-minutes (26% of this workflow). concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number }} cancel-in-progress: true @@ -121,3 +121,57 @@ jobs: env: RUN_SMOKE_TESTS: "0" POLICYENGINE_GITHUB_MICRODATA_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + PolicyEngineUSMain: + name: Test policyengine-us main against this Core + runs-on: ubuntu-latest + needs: Lint + timeout-minutes: 30 + permissions: + contents: read + env: + PYTHONUNBUFFERED: 1 + steps: + - name: Checkout repo + uses: actions/checkout@v6 + with: + path: core + persist-credentials: false + - name: Checkout policyengine-us main + uses: actions/checkout@v6 + with: + repository: PolicyEngine/policyengine-us + ref: main + path: policyengine-us + persist-credentials: false + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: "3.14" + - name: Install uv + uses: astral-sh/setup-uv@v8.1.0 + # As policyengine-us's own CI does: its lock with the newest Core release. + - name: Install policyengine-us main with the Core release + working-directory: policyengine-us + run: | + uv lock --upgrade-package policyengine-core + uv sync --locked --extra dev + bash ./update_itemization.sh + - name: Select the policyengine-us tests + run: python core/.github/country_main_tests.py select core/.github/requirements/policyengine-us-main-tests.txt policyengine-us > selected.txt + # Both runs draw the same Hypothesis examples, so a difference between + # them comes from Core. A failing run still writes its report. + - name: Run them with the Core release + working-directory: policyengine-us + run: | + uv pip list | grep -i '^policyengine-core ' + xargs uv run --no-sync python -m pytest -p no:cacheprovider --hypothesis-seed=0 --junitxml=../released.xml < ../selected.txt || true + - name: Install this Core + working-directory: policyengine-us + run: | + uv pip install --python .venv/bin/python --reinstall-package policyengine-core ../core + uv run --no-sync python -c "import json; from importlib.metadata import distribution; url = json.loads(distribution('policyengine-core').read_text('direct_url.json'))['url']; print(url); assert url.startswith('file://'), url" + - name: Run them with this Core + working-directory: policyengine-us + run: xargs uv run --no-sync python -m pytest -p no:cacheprovider --hypothesis-seed=0 --junitxml=../candidate.xml < ../selected.txt || true + - name: Fail on tests this Core breaks + run: python core/.github/country_main_tests.py compare released.xml candidate.xml diff --git a/changelog.d/test-policyengine-us-main.changed.md b/changelog.d/test-policyengine-us-main.changed.md new file mode 100644 index 000000000..52ab3be87 --- /dev/null +++ b/changelog.d/test-policyengine-us-main.changed.md @@ -0,0 +1 @@ +Add a PR job that runs policyengine-us main's Core-facing tests with the Core release and then with the Core under test, and fails on tests the change breaks. diff --git a/tests/core/test_country_main_tests.py b/tests/core/test_country_main_tests.py new file mode 100644 index 000000000..2cf14c6ee --- /dev/null +++ b/tests/core/test_country_main_tests.py @@ -0,0 +1,169 @@ +"""The PolicyEngineUSMain job blames Core only for the tests Core breaks. + +It runs policyengine-us main's listed tests with the Core release and then +with the Core under test (.github/country_main_tests.py). These tests check +the comparison on synthetic JUnit reports, the file selection, the list, and +the order of the job's steps. +""" + +import importlib.util +from pathlib import Path + +import pytest +import yaml + +ROOT = Path(__file__).resolve().parents[2] +SCRIPT = ROOT / ".github" / "country_main_tests.py" +LIST = ROOT / ".github" / "requirements" / "policyengine-us-main-tests.txt" +WORKFLOW = ROOT / ".github" / "workflows" / "pr.yaml" +SPEC = importlib.util.spec_from_file_location("country_main_tests", SCRIPT) +country = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(country) + + +def junit(path, cases): + """Write a JUnit report of (classname, name, outcome) cases.""" + body = "" + for classname, name, outcome in cases: + child = { + "passed": "", + "failed": '', + "error": '', + "skipped": '', + }[outcome] + body += f'{child}' + path.write_text(f"{body}") + return path + + +def test_outcomes_count_a_failure_in_any_entry_of_a_test(tmp_path): + report = junit( + tmp_path / "r.xml", + [ + ("a.test_x", "test_pass", "passed"), + ("a.test_x", "test_fail", "failed"), + ("a.test_x", "test_skip", "skipped"), + ("a.test_x.TestCase", "runTest", "passed"), + ("a.test_x.TestCase", "runTest", "error"), + ], + ) + assert country.outcomes(report) == { + "a.test_x::test_pass": "passed", + "a.test_x::test_fail": "failed", + "a.test_x::test_skip": "skipped", + "a.test_x.TestCase::runTest": "failed", + } + + +def test_only_tests_failing_with_this_core_alone_are_blamed_on_it(): + released = {"t::a": "passed", "t::b": "failed", "t::c": "skipped"} + candidate = { + "t::a": "failed", # this Core breaks it + "t::b": "failed", # policyengine-us's own failure + "t::c": "failed", # skipped before, fails now + "t::d": "failed", # absent before, such as a new collection error + "t::e": "passed", + } + assert country.compare(released, candidate) == ( + ["t::a", "t::c", "t::d"], + ["t::b"], + ) + + +@pytest.mark.parametrize( + "released_cases, candidate_cases, status", + [ + ([("t", "a", "passed")], [("t", "a", "failed")], 1), + ([("t", "a", "failed")], [("t", "a", "failed")], 0), + ([("t", "a", "passed")], [("t", "a", "passed")], 0), + (None, [("t", "a", "failed")], 0), + ([("t", "a", "passed")], None, 1), + ], + ids=["broken", "fails-with-both", "passes", "no-release-report", "no-report"], +) +def test_compare_fails_only_on_breaks_or_a_missing_report( + tmp_path, capsys, released_cases, candidate_cases, status +): + released = tmp_path / "released.xml" + candidate = tmp_path / "candidate.xml" + if released_cases is not None: + junit(released, released_cases) + if candidate_cases is not None: + junit(candidate, candidate_cases) + summary = tmp_path / "summary.md" + assert ( + country.main( + ["compare", str(released), str(candidate), "--summary", str(summary)] + ) + == status + ) + output = capsys.readouterr().out + if status and candidate_cases is not None: + assert "::error::This Core breaks t::a" in output + assert "**This Core breaks 1:**" in summary.read_text() + if released_cases == candidate_cases == [("t", "a", "failed")]: + assert "::warning::Fails with the Core release too: t::a" in output + + +def test_select_skips_files_the_checkout_lacks(tmp_path, capsys): + repo = tmp_path / "policyengine-us" + (repo / "tests").mkdir(parents=True) + (repo / "tests" / "test_kept.py").write_text("") + listing = tmp_path / "list.txt" + listing.write_text( + "# comment\n\ntests/test_kept.py # trailing comment\ntests/test_gone.py\n" + ) + assert country.select(listing, repo) == ( + ["tests/test_kept.py"], + ["tests/test_gone.py"], + ) + assert country.main(["select", str(listing), str(repo)]) == 0 + captured = capsys.readouterr() + assert captured.out == "tests/test_kept.py\n" + assert "::warning::policyengine-us main has no tests/test_gone.py" in captured.err + listing.write_text("tests/test_gone.py\n") + assert country.main(["select", str(listing), str(repo)]) == 1 + + +def test_the_list_names_distinct_policyengine_us_test_files(): + paths = country.listed_files(LIST) + assert paths, "the list is empty" + assert len(paths) == len(set(paths)), "a file is listed twice" + for path in paths: + assert path.startswith("policyengine_us/tests/"), path + assert Path(path).name.startswith("test_") and path.endswith(".py"), path + + +def test_the_job_runs_the_release_then_this_core_and_compares_them(): + job = yaml.safe_load(WORKFLOW.read_text())["jobs"]["PolicyEngineUSMain"] + names = [step["name"] for step in job["steps"]] + assert names == [ + "Checkout repo", + "Checkout policyengine-us main", + "Set up Python", + "Install uv", + "Install policyengine-us main with the Core release", + "Select the policyengine-us tests", + "Run them with the Core release", + "Install this Core", + "Run them with this Core", + "Fail on tests this Core breaks", + ] + steps = {step["name"]: step for step in job["steps"]} + us = steps["Checkout policyengine-us main"]["with"] + assert (us["repository"], us["ref"]) == ("PolicyEngine/policyengine-us", "main") + released = steps["Run them with the Core release"]["run"] + candidate = steps["Run them with this Core"]["run"] + # The same tests and Hypothesis seed, with separate reports. + assert released.splitlines()[-1].replace("released.xml", "candidate.xml") == ( + candidate + ) + assert "--hypothesis-seed=0" in candidate + # The candidate replaces the release even at the same version number, + # and the job checks that it was installed from this checkout. + install = steps["Install this Core"]["run"] + assert "--reinstall-package policyengine-core ../core" in install + assert "assert url.startswith('file://')" in install + assert steps["Fail on tests this Core breaks"]["run"].endswith( + "compare released.xml candidate.xml" + )