From 3eeb4db946ff089d9e222423e9dd11d27183b44b Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Mon, 5 Oct 2026 22:39:22 -0700 Subject: [PATCH 01/25] gh-158895: Teach librertest to output github job summaries Goal is to make it faster to triage individual test failures while keeping the full existing regrtest fidelity. Fairly minimal integration to hopefully save some human eyeball time. Setup so only the test summary process should output the summary, subprocesses have the env var which will cause output unset. --- Lib/test/libregrtest/main.py | 23 ++++++++++++++++++- Lib/test/libregrtest/worker.py | 2 ++ ...-10-05-22-36-06.gh-issue-158895.3Dmc4J.rst | 2 ++ 3 files changed, 26 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Tests/2026-10-05-22-36-06.gh-issue-158895.3Dmc4J.rst diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index 2e8397a8a91d324..acaebf96e9ce533 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -1,3 +1,5 @@ +import contextlib +import io import os import random import re @@ -6,7 +8,7 @@ import sysconfig import time import trace -from _colorize import get_colors # type: ignore[import-not-found] +from _colorize import decolor, get_colors # type: ignore[import-not-found] from typing import NoReturn from test.support import os_helper, MS_WINDOWS, flush_std_streams @@ -480,6 +482,25 @@ def finalize_tests(self, coverage: trace.CoverageResults | None) -> None: if self.junit_filename: self.results.write_junit(self.junit_filename) + self.write_github_summary() + + def write_github_summary(self) -> None: + filename = os.environ.get("GITHUB_STEP_SUMMARY") + if not filename: + return + with contextlib.redirect_stdout(io.StringIO()) as summary: + self.display_summary() + lines = ["## Test results", + f"```\n{decolor(summary.getvalue()).strip()}\n```"] + # Tests which failed in the last run (the re-run, if any) + for result in self.results.rerun_results: + lines.append(f"### {decolor(str(result))}") + for name, traceback in (result.errors or []) + (result.failures or []): + lines.append(f"
{name}\n\n" + f"```\n{traceback}\n```\n
") + with open(filename, "a", encoding="utf-8") as fp: + fp.write("\n\n".join(lines) + "\n") + def display_summary(self) -> None: if self.first_runtests is None: raise ValueError( diff --git a/Lib/test/libregrtest/worker.py b/Lib/test/libregrtest/worker.py index 4e69ab9d8fad05f..24ae95ed23914f6 100644 --- a/Lib/test/libregrtest/worker.py +++ b/Lib/test/libregrtest/worker.py @@ -28,6 +28,8 @@ def create_worker_process(runtests: WorkerRunTests, output_fd: int, cmd.extend(['-m', 'test.libregrtest.worker', worker_json]) env = dict(os.environ) + # Only the main process writes the GitHub Actions job summary + env.pop('GITHUB_STEP_SUMMARY', None) if tmp_dir is not None: env['TMPDIR'] = tmp_dir env['TEMP'] = tmp_dir diff --git a/Misc/NEWS.d/next/Tests/2026-10-05-22-36-06.gh-issue-158895.3Dmc4J.rst b/Misc/NEWS.d/next/Tests/2026-10-05-22-36-06.gh-issue-158895.3Dmc4J.rst new file mode 100644 index 000000000000000..ebeca7174e5903a --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2026-10-05-22-36-06.gh-issue-158895.3Dmc4J.rst @@ -0,0 +1,2 @@ +Individual test failures and an overall summary is now written by +:mod:`test.regrtest` to the Github Action Summary. From 8fcfa5d0f2f8a68424594d4e22824cf1817af2cf Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Mon, 5 Oct 2026 23:09:38 -0700 Subject: [PATCH 02/25] Only output errors, too many jobs. Try coloring a bit more --- Lib/test/libregrtest/main.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index acaebf96e9ce533..40fcfe5f5c577f0 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -486,7 +486,10 @@ def finalize_tests(self, coverage: trace.CoverageResults | None) -> None: def write_github_summary(self) -> None: filename = os.environ.get("GITHUB_STEP_SUMMARY") - if not filename: + # Only write a summary on failure: a CI run has many test jobs + exitcode = self.results.get_exitcode(self.fail_env_changed, + self.fail_rerun) + if not filename or not exitcode: return with contextlib.redirect_stdout(io.StringIO()) as summary: self.display_summary() @@ -497,7 +500,7 @@ def write_github_summary(self) -> None: lines.append(f"### {decolor(str(result))}") for name, traceback in (result.errors or []) + (result.failures or []): lines.append(f"
{name}\n\n" - f"```\n{traceback}\n```\n
") + f"```pytb\n{decolor(traceback)}\n```\n") with open(filename, "a", encoding="utf-8") as fp: fp.write("\n\n".join(lines) + "\n") From e6e09bd77e81ba8def2a1d2b399a11521d995e90 Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Mon, 5 Oct 2026 23:10:30 -0700 Subject: [PATCH 03/25] Fail CI on linux for experimenting with output formatting --- Lib/test/test_os/test_os.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_os/test_os.py b/Lib/test/test_os/test_os.py index 81b3043eb7e75bc..04321264e6f121d 100644 --- a/Lib/test/test_os/test_os.py +++ b/Lib/test/test_os/test_os.py @@ -4080,7 +4080,8 @@ def test_memfd_create(self): self.assertFalse(os.get_inheritable(fd)) with open(fd, "wb", closefd=False) as f: f.write(b'memfd_create') - self.assertEqual(f.tell(), 12) + # XXX: Intentional failure to test GitHub Actions job summaries + self.assertEqual(f.tell(), 13) fd2 = os.memfd_create("Hi") self.addCleanup(os.close, fd2) From 54d123fd515a812147582e8dd1a8264ac1e13cb2 Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Mon, 5 Oct 2026 23:42:57 -0700 Subject: [PATCH 04/25] Reformat failures to be failure first, easier visibility. Simplify formatting code some --- Lib/test/libregrtest/main.py | 35 +++++++++++++++++++++++------------ 1 file changed, 23 insertions(+), 12 deletions(-) diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index 40fcfe5f5c577f0..42dbf166837a6fc 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -1,5 +1,3 @@ -import contextlib -import io import os import random import re @@ -491,18 +489,31 @@ def write_github_summary(self) -> None: self.fail_rerun) if not filename or not exitcode: return - with contextlib.redirect_stdout(io.StringIO()) as summary: - self.display_summary() - lines = ["## Test results", - f"```\n{decolor(summary.getvalue()).strip()}\n```"] # Tests which failed in the last run (the re-run, if any) - for result in self.results.rerun_results: - lines.append(f"### {decolor(str(result))}") - for name, traceback in (result.errors or []) + (result.failures or []): - lines.append(f"
{name}\n\n" - f"```pytb\n{decolor(traceback)}\n```\n
") + failed = self.results.rerun_results + cases = [(result.errors or []) + (result.failures or []) + for result in failed] + ncase = sum(map(len, cases)) with open(filename, "a", encoding="utf-8") as fp: - fp.write("\n\n".join(lines) + "\n") + def write(text: str) -> None: + # Separate Markdown blocks with an empty line + print(text, end="\n\n", file=fp) + + write(f"## {decolor(self.get_state())}: " + f"{count(len(failed), 'test file')} and " + f"{count(ncase, 'test case')} failed") + for result, result_cases in zip(failed, cases): + write(f"### {decolor(str(result))}") + if result.env_changed_reasons: + write("\n".join(f"- {reason}" + for reason in result.env_changed_reasons)) + for name, traceback in result_cases: + # Expand short tracebacks when there are only a few + is_open = ncase <= 5 and traceback.count("\n") < 30 + write(f"" + f"{name}") + write(f"```pytb\n{decolor(traceback).rstrip()}\n```") + write("") def display_summary(self) -> None: if self.first_runtests is None: From 3468742f5b333d261a0544c1c807283a2111842d Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Mon, 5 Oct 2026 23:44:54 -0700 Subject: [PATCH 05/25] add error annotations --- Lib/test/libregrtest/main.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index 42dbf166837a6fc..f191dd123ed8961 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -30,6 +30,15 @@ get_process_memory_usage, EXIT_TIMEOUT) +def github_error(title: str, message: str) -> None: + # Emit a GitHub Actions error annotation (workflow command) + def escape(text: str) -> str: + return (text.replace("%", "%25").replace("\r", "%0D") + .replace("\n", "%0A")) + title = escape(title).replace(":", "%3A").replace(",", "%2C") + print(f"::error title={title}::{escape(message)}", flush=True) + + class Regrtest: """Execute a test suite. @@ -504,10 +513,14 @@ def write(text: str) -> None: f"{count(ncase, 'test case')} failed") for result, result_cases in zip(failed, cases): write(f"### {decolor(str(result))}") + if not result_cases: + github_error(result.test_name, decolor(str(result))) if result.env_changed_reasons: write("\n".join(f"- {reason}" for reason in result.env_changed_reasons)) for name, traceback in result_cases: + github_error(f"{result.test_name}: {name}", + decolor(traceback)) # Expand short tracebacks when there are only a few is_open = ncase <= 5 and traceback.count("\n") < 30 write(f"" From ff7b75a45b6302ee2ab25d99265262882ddee709 Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Tue, 6 Oct 2026 00:14:59 -0700 Subject: [PATCH 06/25] Add groups more of the time --- Lib/test/libregrtest/main.py | 13 ------------- Lib/test/libregrtest/results.py | 20 ++++++++++++++++++-- Lib/test/libregrtest/run_workers.py | 26 ++++++++++++++++++++++++++ Lib/test/libregrtest/utils.py | 23 +++++++++++++++++++++++ 4 files changed, 67 insertions(+), 15 deletions(-) diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index f191dd123ed8961..42dbf166837a6fc 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -30,15 +30,6 @@ get_process_memory_usage, EXIT_TIMEOUT) -def github_error(title: str, message: str) -> None: - # Emit a GitHub Actions error annotation (workflow command) - def escape(text: str) -> str: - return (text.replace("%", "%25").replace("\r", "%0D") - .replace("\n", "%0A")) - title = escape(title).replace(":", "%3A").replace(",", "%2C") - print(f"::error title={title}::{escape(message)}", flush=True) - - class Regrtest: """Execute a test suite. @@ -513,14 +504,10 @@ def write(text: str) -> None: f"{count(ncase, 'test case')} failed") for result, result_cases in zip(failed, cases): write(f"### {decolor(str(result))}") - if not result_cases: - github_error(result.test_name, decolor(str(result))) if result.env_changed_reasons: write("\n".join(f"- {reason}" for reason in result.env_changed_reasons)) for name, traceback in result_cases: - github_error(f"{result.test_name}: {name}", - decolor(traceback)) # Expand short tracebacks when there are only a few is_open = ncase <= 5 and traceback.count("\n") < 30 write(f"" diff --git a/Lib/test/libregrtest/results.py b/Lib/test/libregrtest/results.py index 3475f645729182c..9015cc1fe5f8a13 100644 --- a/Lib/test/libregrtest/results.py +++ b/Lib/test/libregrtest/results.py @@ -1,13 +1,14 @@ +import os import sys import trace -from _colorize import get_colors # type: ignore[import-not-found] +from _colorize import decolor, get_colors # type: ignore[import-not-found] lazy from xml.etree.ElementTree import Element from .runtests import RunTests from .result import State, TestResult, TestStats, Location from .utils import ( StrPath, TestName, TestTuple, TestList, FilterDict, - printlist, count, format_duration) + printlist, count, format_duration, github_annotation) # Python uses exit code 1 when an exception is not caught @@ -19,6 +20,16 @@ EXITCODE_INTERRUPTED = 130 # 128 + signal.SIGINT=2 +def annotate_github(result: TestResult, level: str) -> None: + cases = (result.errors or []) + (result.failures or []) + for name, traceback in cases: + github_annotation(level, f"{result.test_name}: {name}", + decolor(traceback)) + if not cases: + message = "\n".join([str(result), *(result.env_changed_reasons or ())]) + github_annotation(level, result.test_name, decolor(message)) + + class TestResults: def __init__(self) -> None: self.bad: TestList = [] @@ -128,6 +139,11 @@ def accumulate_result(self, result: TestResult, runtests: RunTests) -> None: if result.state == State.WORKER_BUG: self.worker_bug = True + if (os.environ.get("GITHUB_STEP_SUMMARY") + and result.is_failed(fail_env_changed)): + # A failure is only an error if it fails again when re-run + annotate_github(result, "error" if rerun else "warning") + if result.has_meaningful_duration() and not rerun: if result.duration is None: raise ValueError("result.duration is None") diff --git a/Lib/test/libregrtest/run_workers.py b/Lib/test/libregrtest/run_workers.py index c6db78e0882ba31..69ff5edb2607da9 100644 --- a/Lib/test/libregrtest/run_workers.py +++ b/Lib/test/libregrtest/run_workers.py @@ -10,6 +10,7 @@ import threading import time import traceback +from _colorize import decolor # type: ignore[import-not-found] from typing import Any, Literal, TextIO from test import support @@ -519,6 +520,11 @@ def __init__(self, num_workers: int, runtests: RunTests, else: self.worker_timeout = None self.workers: list[WorkerThread] = [] + # On GitHub Actions, fold the first run's log into groups: ranges of + # passing tests, and one group per failed test. Re-runs aren't folded. + self.github_groups = (not runtests.rerun + and bool(os.environ.get("GITHUB_STEP_SUMMARY"))) + self.in_github_group = False jobs = self.runtests.get_jobs() if jobs is not None: @@ -611,7 +617,16 @@ def _process_result(self, item: QueueOutput) -> TestResult: self.test_index += 1 mp_result = item[1] result = mp_result.result + failed = result.is_failed(self.runtests.fail_env_changed) + if self.github_groups and failed: + # Keep the failure annotation out of collapsed groups + self.github_group(None) self.results.accumulate_result(result, self.runtests) + if self.github_groups: + if failed: + self.github_group(decolor(str(result))) + elif not self.in_github_group: + self.github_group(f"Tests from #{self.test_index}") self.display_result(mp_result) # Display worker stdout @@ -624,9 +639,19 @@ def _process_result(self, item: QueueOutput) -> TestResult: stdout = mp_result.worker_stdout if stdout: print(stdout, flush=True) + if self.github_groups and failed: + self.github_group(None) return result + def github_group(self, title: str | None) -> None: + """End the current GitHub Actions log group, start a new one if title.""" + if self.in_github_group: + print("::endgroup::", flush=True) + self.in_github_group = title is not None + if title is not None: + print(f"::group::{title}", flush=True) + def get_mem_usage(self): usage = 0 main_mem = get_process_memory_usage(os.getpid()) @@ -659,6 +684,7 @@ def run(self) -> None: print() self.results.interrupted = True finally: + self.github_group(None) if self.timeout is not None: faulthandler.cancel_dump_traceback_later() diff --git a/Lib/test/libregrtest/utils.py b/Lib/test/libregrtest/utils.py index 83e0575619c99a0..c31022ea044ece6 100644 --- a/Lib/test/libregrtest/utils.py +++ b/Lib/test/libregrtest/utils.py @@ -140,6 +140,29 @@ def print_warning(msg: str) -> None: orig_unraisablehook: Callable[..., None] | None = None +def github_annotation(level: str, title: str, message: str) -> None: + """Emit a GitHub Actions annotation (workflow command). + + Point it at the last frame of the traceback in message, if any. + """ + def escape(text: str) -> str: + return (text.replace("%", "%25").replace("\r", "%0D") + .replace("\n", "%0A")) + + def escape_property(text: str) -> str: + return escape(text).replace(":", "%3A").replace(",", "%2C") + + props = f"title={escape_property(title)}" + frames = re.findall(r'^ File "(.+)", line (\d+)', message, re.MULTILINE) + # Source checkout which contains Lib/ + srcdir = os.path.dirname(os.path.dirname(os.__file__)) + os.sep + if frames and frames[-1][0].startswith(srcdir): + filename, line = frames[-1] + filename = filename.removeprefix(srcdir).replace(os.sep, "/") + props = f"file={escape_property(filename)},line={line},{props}" + print(f"::{level} {props}::{escape(message)}", flush=True) + + def regrtest_unraisable_hook(unraisable) -> None: global orig_unraisablehook support.set_environment_altered( From 946c801af026df3da4514eaa4cbca2e9bb9b18bf Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Tue, 6 Oct 2026 00:50:16 -0700 Subject: [PATCH 07/25] drop groups, error only on final failure --- Lib/test/libregrtest/main.py | 6 +++++- Lib/test/libregrtest/results.py | 12 ++++++------ Lib/test/libregrtest/run_workers.py | 26 -------------------------- Lib/test/libregrtest/utils.py | 11 ++++++----- 4 files changed, 17 insertions(+), 38 deletions(-) diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index 42dbf166837a6fc..1120dacf3b5a6a1 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -16,7 +16,7 @@ from .logger import Logger from .pgo import setup_pgo_tests from .result import TestResult -from .results import TestResults, EXITCODE_INTERRUPTED +from .results import TestResults, EXITCODE_INTERRUPTED, annotate_github from .runtests import RunTests, HuntRefleak from .setup import setup_process, setup_test_dir from .single import run_single_test, PROGRESS_MIN_TIME @@ -491,6 +491,10 @@ def write_github_summary(self) -> None: return # Tests which failed in the last run (the re-run, if any) failed = self.results.rerun_results + if not self.results.rerun: + # Failed tests were not re-run (ex: --python): annotate them now + for result in failed: + annotate_github(result) cases = [(result.errors or []) + (result.failures or []) for result in failed] ncase = sum(map(len, cases)) diff --git a/Lib/test/libregrtest/results.py b/Lib/test/libregrtest/results.py index 9015cc1fe5f8a13..a860ad0f9c8549e 100644 --- a/Lib/test/libregrtest/results.py +++ b/Lib/test/libregrtest/results.py @@ -20,14 +20,14 @@ EXITCODE_INTERRUPTED = 130 # 128 + signal.SIGINT=2 -def annotate_github(result: TestResult, level: str) -> None: +def annotate_github(result: TestResult) -> None: cases = (result.errors or []) + (result.failures or []) for name, traceback in cases: - github_annotation(level, f"{result.test_name}: {name}", + github_annotation("error", f"{result.test_name}: {name}", decolor(traceback)) if not cases: message = "\n".join([str(result), *(result.env_changed_reasons or ())]) - github_annotation(level, result.test_name, decolor(message)) + github_annotation("error", result.test_name, decolor(message)) class TestResults: @@ -139,10 +139,10 @@ def accumulate_result(self, result: TestResult, runtests: RunTests) -> None: if result.state == State.WORKER_BUG: self.worker_bug = True - if (os.environ.get("GITHUB_STEP_SUMMARY") + if (os.environ.get("GITHUB_STEP_SUMMARY") and rerun and result.is_failed(fail_env_changed)): - # A failure is only an error if it fails again when re-run - annotate_github(result, "error" if rerun else "warning") + # Only annotate tests which fail again when re-run + annotate_github(result) if result.has_meaningful_duration() and not rerun: if result.duration is None: diff --git a/Lib/test/libregrtest/run_workers.py b/Lib/test/libregrtest/run_workers.py index 69ff5edb2607da9..c6db78e0882ba31 100644 --- a/Lib/test/libregrtest/run_workers.py +++ b/Lib/test/libregrtest/run_workers.py @@ -10,7 +10,6 @@ import threading import time import traceback -from _colorize import decolor # type: ignore[import-not-found] from typing import Any, Literal, TextIO from test import support @@ -520,11 +519,6 @@ def __init__(self, num_workers: int, runtests: RunTests, else: self.worker_timeout = None self.workers: list[WorkerThread] = [] - # On GitHub Actions, fold the first run's log into groups: ranges of - # passing tests, and one group per failed test. Re-runs aren't folded. - self.github_groups = (not runtests.rerun - and bool(os.environ.get("GITHUB_STEP_SUMMARY"))) - self.in_github_group = False jobs = self.runtests.get_jobs() if jobs is not None: @@ -617,16 +611,7 @@ def _process_result(self, item: QueueOutput) -> TestResult: self.test_index += 1 mp_result = item[1] result = mp_result.result - failed = result.is_failed(self.runtests.fail_env_changed) - if self.github_groups and failed: - # Keep the failure annotation out of collapsed groups - self.github_group(None) self.results.accumulate_result(result, self.runtests) - if self.github_groups: - if failed: - self.github_group(decolor(str(result))) - elif not self.in_github_group: - self.github_group(f"Tests from #{self.test_index}") self.display_result(mp_result) # Display worker stdout @@ -639,19 +624,9 @@ def _process_result(self, item: QueueOutput) -> TestResult: stdout = mp_result.worker_stdout if stdout: print(stdout, flush=True) - if self.github_groups and failed: - self.github_group(None) return result - def github_group(self, title: str | None) -> None: - """End the current GitHub Actions log group, start a new one if title.""" - if self.in_github_group: - print("::endgroup::", flush=True) - self.in_github_group = title is not None - if title is not None: - print(f"::group::{title}", flush=True) - def get_mem_usage(self): usage = 0 main_mem = get_process_memory_usage(os.getpid()) @@ -684,7 +659,6 @@ def run(self) -> None: print() self.results.interrupted = True finally: - self.github_group(None) if self.timeout is not None: faulthandler.cancel_dump_traceback_later() diff --git a/Lib/test/libregrtest/utils.py b/Lib/test/libregrtest/utils.py index c31022ea044ece6..0f21c7981c5141b 100644 --- a/Lib/test/libregrtest/utils.py +++ b/Lib/test/libregrtest/utils.py @@ -154,12 +154,13 @@ def escape_property(text: str) -> str: props = f"title={escape_property(title)}" frames = re.findall(r'^ File "(.+)", line (\d+)', message, re.MULTILINE) - # Source checkout which contains Lib/ - srcdir = os.path.dirname(os.path.dirname(os.__file__)) + os.sep - if frames and frames[-1][0].startswith(srcdir): + # Map the stdlib directory (Lib/ in a source checkout or an installed + # lib/python3.X/) to Lib/ in the repository + stdlib_dir = os.path.dirname(os.__file__) + os.sep + if frames and frames[-1][0].startswith(stdlib_dir): filename, line = frames[-1] - filename = filename.removeprefix(srcdir).replace(os.sep, "/") - props = f"file={escape_property(filename)},line={line},{props}" + filename = filename.removeprefix(stdlib_dir).replace(os.sep, "/") + props = f"file=Lib/{escape_property(filename)},line={line},{props}" print(f"::{level} {props}::{escape(message)}", flush=True) From 14394da14e30c9bda7aaef14c950d692c1415fcc Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Tue, 6 Oct 2026 10:27:47 -0700 Subject: [PATCH 08/25] More structured error blocks --- Lib/test/libregrtest/utils.py | 55 ++++++++++++++++++++++++++++------- 1 file changed, 45 insertions(+), 10 deletions(-) diff --git a/Lib/test/libregrtest/utils.py b/Lib/test/libregrtest/utils.py index 0f21c7981c5141b..ee241844098c406 100644 --- a/Lib/test/libregrtest/utils.py +++ b/Lib/test/libregrtest/utils.py @@ -1,5 +1,6 @@ import contextlib import faulthandler +import linecache import locale import math import os.path @@ -140,6 +141,39 @@ def print_warning(msg: str) -> None: orig_unraisablehook: Callable[..., None] | None = None +def traceback_location(traceback: str) -> dict[str, str | int] | None: + """Location of the last frame of a traceback, relative to the repository. + + Return a dict with file and line, plus col and endColumn when the + traceback underlines the failing expression (~~~^^^). + """ + lines = traceback.splitlines() + for index in reversed(range(len(lines))): + if match := re.match(r' File "(.+)", line (\d+)', lines[index]): + break + else: + return None + filename, lineno = match.group(1), int(match.group(2)) + # Map the stdlib directory (Lib/ in a source checkout or an installed + # lib/python3.X/) to Lib/ in the repository + stdlib_dir = os.path.dirname(os.__file__) + os.sep + if not filename.startswith(stdlib_dir): + return None + relpath = filename.removeprefix(stdlib_dir).replace(os.sep, "/") + location: dict[str, str | int] = {"file": f"Lib/{relpath}", "line": lineno} + + # The frame is followed by the dedented source line (" code") and, + # for a single-line expression, by a line of ~ and ^ markers. + markers = lines[index + 2] if index + 2 < len(lines) else "" + source = linecache.getline(filename, lineno) + if markers.strip() and not markers.strip(" ~^") and source: + indent = len(source) - len(source.lstrip()) + start = len(markers) - len(markers.lstrip()) + location["col"] = indent + start - 4 + 1 + location["endColumn"] = indent + len(markers.rstrip()) - 4 + return location + + def github_annotation(level: str, title: str, message: str) -> None: """Emit a GitHub Actions annotation (workflow command). @@ -152,16 +186,17 @@ def escape(text: str) -> str: def escape_property(text: str) -> str: return escape(text).replace(":", "%3A").replace(",", "%2C") - props = f"title={escape_property(title)}" - frames = re.findall(r'^ File "(.+)", line (\d+)', message, re.MULTILINE) - # Map the stdlib directory (Lib/ in a source checkout or an installed - # lib/python3.X/) to Lib/ in the repository - stdlib_dir = os.path.dirname(os.__file__) + os.sep - if frames and frames[-1][0].startswith(stdlib_dir): - filename, line = frames[-1] - filename = filename.removeprefix(stdlib_dir).replace(os.sep, "/") - props = f"file=Lib/{escape_property(filename)},line={line},{props}" - print(f"::{level} {props}::{escape(message)}", flush=True) + props: dict[str, str | int] = {} + if location := traceback_location(message): + props |= location + # The job log only shows the message: start it with the location + position = ":".join(str(location[key]) + for key in ("file", "line", "col") if key in location) + message = f"{position}\n{message}" + props["title"] = title + props_text = ",".join(f"{key}={escape_property(str(value))}" + for key, value in props.items()) + print(f"::{level} {props_text}::{escape(message)}", flush=True) def regrtest_unraisable_hook(unraisable) -> None: From 63598766247dd7c00eb73a02990113730b936e8a Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Tue, 6 Oct 2026 10:39:56 -0700 Subject: [PATCH 09/25] Fail more tests --- Lib/test/test_bool.py | 6 ++++++ Lib/test/test_json/test_decode.py | 4 ++++ Lib/test/test_math.py | 5 +++++ Lib/test/test_textwrap.py | 5 +++++ 4 files changed, 20 insertions(+) diff --git a/Lib/test/test_bool.py b/Lib/test/test_bool.py index dcdf7bdce03b800..2214e7408aa5603 100644 --- a/Lib/test/test_bool.py +++ b/Lib/test/test_bool.py @@ -4,6 +4,7 @@ from test.support import os_helper import os +import sys class BoolTest(unittest.TestCase): @@ -25,6 +26,11 @@ def test_repr(self): self.assertIs(eval(repr(True)), True) def test_str(self): + if sys.platform == "linux": + # XXX: Intentional failure to test GitHub Actions annotations + self.assertEqual( + str(True), + 'Yes') self.assertEqual(str(False), 'False') self.assertEqual(str(True), 'True') diff --git a/Lib/test/test_json/test_decode.py b/Lib/test/test_json/test_decode.py index 7bff16ddf2155e9..d01368254f6701e 100644 --- a/Lib/test/test_json/test_decode.py +++ b/Lib/test/test_json/test_decode.py @@ -1,4 +1,5 @@ import decimal +import sys import unittest.mock from io import StringIO from collections import OrderedDict @@ -9,6 +10,9 @@ class TestDecode: def test_decimal(self): rval = self.loads('1.1', parse_float=decimal.Decimal) + if sys.platform == "linux": + # XXX: Intentional failure to test GitHub Actions annotations + self.loads('{"bad": }') self.assertIsInstance(rval, decimal.Decimal) self.assertEqual(rval, decimal.Decimal('1.1')) diff --git a/Lib/test/test_math.py b/Lib/test/test_math.py index a1250c668223e7f..40dc0d9acc75fff 100644 --- a/Lib/test/test_math.py +++ b/Lib/test/test_math.py @@ -513,6 +513,11 @@ def testFabs(self): def testFloor(self): self.assertRaises(TypeError, math.floor) + if sys.platform == "linux": + # XXX: Intentional failure to test GitHub Actions annotations + for x in (0.5, 1.5): + with self.subTest(x=x): + self.assertEqual(math.floor(x), x) self.assertEqual(int, type(math.floor(0.5))) self.assertEqual(math.floor(0.5), 0) self.assertEqual(math.floor(1.0), 1) diff --git a/Lib/test/test_textwrap.py b/Lib/test/test_textwrap.py index aca1f427656bb50..c1d670c978c9702 100644 --- a/Lib/test/test_textwrap.py +++ b/Lib/test/test_textwrap.py @@ -8,6 +8,8 @@ # $Id$ # +import os +import sys import unittest from textwrap import TextWrapper, wrap, fill, dedent, indent, shorten @@ -66,6 +68,9 @@ def test_simple(self): self.check_wrap(text, 80, [text]) def test_empty_string(self): + if sys.platform == "linux": + # XXX: Intentional failure to test GitHub Actions annotations + os.environ["GHA_ANNOTATION_TEST"] = "1" # Check that wrapping the empty string returns an empty list. self.check_wrap("", 6, []) self.check_wrap("", 6, [], drop_whitespace=False) From 041e57792d78978a97ff19dde573506b894a9e2a Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Tue, 6 Oct 2026 10:44:04 -0700 Subject: [PATCH 10/25] Adjust and simplify a little --- Lib/test/libregrtest/main.py | 15 +++++++++------ Lib/test/libregrtest/results.py | 20 ++------------------ 2 files changed, 11 insertions(+), 24 deletions(-) diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index 1120dacf3b5a6a1..dc6e8ebc232fccd 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -16,7 +16,7 @@ from .logger import Logger from .pgo import setup_pgo_tests from .result import TestResult -from .results import TestResults, EXITCODE_INTERRUPTED, annotate_github +from .results import TestResults, EXITCODE_INTERRUPTED from .runtests import RunTests, HuntRefleak from .setup import setup_process, setup_test_dir from .single import run_single_test, PROGRESS_MIN_TIME @@ -27,7 +27,7 @@ printlist, get_temp_dir, get_work_dir, exit_timeout, display_header, cleanup_temp_dir, print_warning, is_cross_compiled, get_host_runner, display_title, - get_process_memory_usage, EXIT_TIMEOUT) + get_process_memory_usage, github_annotation, EXIT_TIMEOUT) class Regrtest: @@ -491,10 +491,6 @@ def write_github_summary(self) -> None: return # Tests which failed in the last run (the re-run, if any) failed = self.results.rerun_results - if not self.results.rerun: - # Failed tests were not re-run (ex: --python): annotate them now - for result in failed: - annotate_github(result) cases = [(result.errors or []) + (result.failures or []) for result in failed] ncase = sum(map(len, cases)) @@ -511,7 +507,14 @@ def write(text: str) -> None: if result.env_changed_reasons: write("\n".join(f"- {reason}" for reason in result.env_changed_reasons)) + if not result_cases: + message = "\n".join([str(result), + *(result.env_changed_reasons or ())]) + github_annotation("error", result.test_name, + decolor(message)) for name, traceback in result_cases: + github_annotation("error", f"{result.test_name}: {name}", + decolor(traceback)) # Expand short tracebacks when there are only a few is_open = ncase <= 5 and traceback.count("\n") < 30 write(f"" diff --git a/Lib/test/libregrtest/results.py b/Lib/test/libregrtest/results.py index a860ad0f9c8549e..3475f645729182c 100644 --- a/Lib/test/libregrtest/results.py +++ b/Lib/test/libregrtest/results.py @@ -1,14 +1,13 @@ -import os import sys import trace -from _colorize import decolor, get_colors # type: ignore[import-not-found] +from _colorize import get_colors # type: ignore[import-not-found] lazy from xml.etree.ElementTree import Element from .runtests import RunTests from .result import State, TestResult, TestStats, Location from .utils import ( StrPath, TestName, TestTuple, TestList, FilterDict, - printlist, count, format_duration, github_annotation) + printlist, count, format_duration) # Python uses exit code 1 when an exception is not caught @@ -20,16 +19,6 @@ EXITCODE_INTERRUPTED = 130 # 128 + signal.SIGINT=2 -def annotate_github(result: TestResult) -> None: - cases = (result.errors or []) + (result.failures or []) - for name, traceback in cases: - github_annotation("error", f"{result.test_name}: {name}", - decolor(traceback)) - if not cases: - message = "\n".join([str(result), *(result.env_changed_reasons or ())]) - github_annotation("error", result.test_name, decolor(message)) - - class TestResults: def __init__(self) -> None: self.bad: TestList = [] @@ -139,11 +128,6 @@ def accumulate_result(self, result: TestResult, runtests: RunTests) -> None: if result.state == State.WORKER_BUG: self.worker_bug = True - if (os.environ.get("GITHUB_STEP_SUMMARY") and rerun - and result.is_failed(fail_env_changed)): - # Only annotate tests which fail again when re-run - annotate_github(result) - if result.has_meaningful_duration() and not rerun: if result.duration is None: raise ValueError("result.duration is None") From ba449b42c06f60b25d70a5a2d2f12d559bc16e7f Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Tue, 6 Oct 2026 11:01:32 -0700 Subject: [PATCH 11/25] Annotate inline with test output --- Lib/test/libregrtest/main.py | 16 ++++++------ Lib/test/libregrtest/result.py | 39 +++++++++++++++++++++++++++-- Lib/test/libregrtest/run_workers.py | 2 ++ Lib/test/libregrtest/utils.py | 25 ------------------ 4 files changed, 47 insertions(+), 35 deletions(-) diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index dc6e8ebc232fccd..9df71bbcf009b9c 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -27,7 +27,7 @@ printlist, get_temp_dir, get_work_dir, exit_timeout, display_header, cleanup_temp_dir, print_warning, is_cross_compiled, get_host_runner, display_title, - get_process_memory_usage, github_annotation, EXIT_TIMEOUT) + get_process_memory_usage, EXIT_TIMEOUT) class Regrtest: @@ -395,6 +395,8 @@ def run_test( result = run_single_test(test_name, runtests) self.results.accumulate_result(result, runtests) + if runtests.rerun and result.is_failed(runtests.fail_env_changed): + result.print_github_annotations() return result @@ -491,6 +493,11 @@ def write_github_summary(self) -> None: return # Tests which failed in the last run (the re-run, if any) failed = self.results.rerun_results + if not self.results.rerun: + # Failed tests were not re-run (ex: --python): annotate them now. + # Otherwise, they were annotated when they failed again. + for result in failed: + result.print_github_annotations() cases = [(result.errors or []) + (result.failures or []) for result in failed] ncase = sum(map(len, cases)) @@ -507,14 +514,7 @@ def write(text: str) -> None: if result.env_changed_reasons: write("\n".join(f"- {reason}" for reason in result.env_changed_reasons)) - if not result_cases: - message = "\n".join([str(result), - *(result.env_changed_reasons or ())]) - github_annotation("error", result.test_name, - decolor(message)) for name, traceback in result_cases: - github_annotation("error", f"{result.test_name}: {name}", - decolor(traceback)) # Expand short tracebacks when there are only a few is_open = ncase <= 5 and traceback.count("\n") < 30 write(f"" diff --git a/Lib/test/libregrtest/result.py b/Lib/test/libregrtest/result.py index 5324a46bb8cb1c5..e806de29139a98e 100644 --- a/Lib/test/libregrtest/result.py +++ b/Lib/test/libregrtest/result.py @@ -1,11 +1,12 @@ import dataclasses import json -from _colorize import get_colors # type: ignore[import-not-found] +import os +from _colorize import decolor, get_colors # type: ignore[import-not-found] from typing import Any from .utils import ( StrJSON, TestName, FilterTuple, - format_duration, normalize_test_name, print_warning) + format_duration, normalize_test_name, print_warning, traceback_location) @dataclasses.dataclass(slots=True) @@ -178,6 +179,40 @@ def __str__(self) -> str: def has_meaningful_duration(self): return State.has_meaningful_duration(self.state) + def print_github_annotations(self) -> None: + """Print a GitHub Actions error annotation per failed test case.""" + if not os.environ.get("GITHUB_STEP_SUMMARY"): + return + + def escape(text: str) -> str: + return (text.replace("%", "%25").replace("\r", "%0D") + .replace("\n", "%0A")) + + def escape_property(text: str) -> str: + return escape(text).replace(":", "%3A").replace(",", "%2C") + + annotations = [(f"{self.test_name}: {name}", traceback) + for name, traceback in (self.errors or []) + + (self.failures or [])] + if not annotations: + # No test case details: crash, timeout, env changed, etc. + message = "\n".join([str(self), *(self.env_changed_reasons or ())]) + annotations = [(self.test_name, message)] + for title, message in annotations: + message = decolor(message) + props: dict[str, str | int] = {} + if location := traceback_location(message): + props |= location + # The job log only shows the message: start it with the location + position = ":".join(str(location[key]) + for key in ("file", "line", "col") + if key in location) + message = f"{position}\n{message}" + props["title"] = title + props_text = ",".join(f"{key}={escape_property(str(value))}" + for key, value in props.items()) + print(f"::error {props_text}::{escape(message)}", flush=True) + def set_env_changed(self, *reasons): if self.state is None or self.state == State.PASSED: self.state = State.ENV_CHANGED diff --git a/Lib/test/libregrtest/run_workers.py b/Lib/test/libregrtest/run_workers.py index c6db78e0882ba31..d88787fb30613e7 100644 --- a/Lib/test/libregrtest/run_workers.py +++ b/Lib/test/libregrtest/run_workers.py @@ -624,6 +624,8 @@ def _process_result(self, item: QueueOutput) -> TestResult: stdout = mp_result.worker_stdout if stdout: print(stdout, flush=True) + if self.runtests.rerun and result.is_failed(self.runtests.fail_env_changed): + result.print_github_annotations() return result diff --git a/Lib/test/libregrtest/utils.py b/Lib/test/libregrtest/utils.py index ee241844098c406..e39a995662bc91f 100644 --- a/Lib/test/libregrtest/utils.py +++ b/Lib/test/libregrtest/utils.py @@ -174,31 +174,6 @@ def traceback_location(traceback: str) -> dict[str, str | int] | None: return location -def github_annotation(level: str, title: str, message: str) -> None: - """Emit a GitHub Actions annotation (workflow command). - - Point it at the last frame of the traceback in message, if any. - """ - def escape(text: str) -> str: - return (text.replace("%", "%25").replace("\r", "%0D") - .replace("\n", "%0A")) - - def escape_property(text: str) -> str: - return escape(text).replace(":", "%3A").replace(",", "%2C") - - props: dict[str, str | int] = {} - if location := traceback_location(message): - props |= location - # The job log only shows the message: start it with the location - position = ":".join(str(location[key]) - for key in ("file", "line", "col") if key in location) - message = f"{position}\n{message}" - props["title"] = title - props_text = ",".join(f"{key}={escape_property(str(value))}" - for key, value in props.items()) - print(f"::{level} {props_text}::{escape(message)}", flush=True) - - def regrtest_unraisable_hook(unraisable) -> None: global orig_unraisablehook support.set_environment_altered( From 59b1608fc0b40afcad22e75576d990ea8e4c64b4 Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Tue, 6 Oct 2026 11:18:53 -0700 Subject: [PATCH 12/25] Move annotations before test output --- Lib/test/libregrtest/result.py | 4 +--- Lib/test/libregrtest/run_workers.py | 5 +++-- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/Lib/test/libregrtest/result.py b/Lib/test/libregrtest/result.py index e806de29139a98e..f2fe90cab9a8712 100644 --- a/Lib/test/libregrtest/result.py +++ b/Lib/test/libregrtest/result.py @@ -191,9 +191,7 @@ def escape(text: str) -> str: def escape_property(text: str) -> str: return escape(text).replace(":", "%3A").replace(",", "%2C") - annotations = [(f"{self.test_name}: {name}", traceback) - for name, traceback in (self.errors or []) - + (self.failures or [])] + annotations = (self.errors or []) + (self.failures or []) if not annotations: # No test case details: crash, timeout, env changed, etc. message = "\n".join([str(self), *(self.env_changed_reasons or ())]) diff --git a/Lib/test/libregrtest/run_workers.py b/Lib/test/libregrtest/run_workers.py index d88787fb30613e7..2000b0b469e254f 100644 --- a/Lib/test/libregrtest/run_workers.py +++ b/Lib/test/libregrtest/run_workers.py @@ -613,6 +613,9 @@ def _process_result(self, item: QueueOutput) -> TestResult: result = mp_result.result self.results.accumulate_result(result, self.runtests) self.display_result(mp_result) + if self.runtests.rerun and result.is_failed(self.runtests.fail_env_changed): + # Annotate before the test output + result.print_github_annotations() # Display worker stdout if not self.runtests.output_on_failure: @@ -624,8 +627,6 @@ def _process_result(self, item: QueueOutput) -> TestResult: stdout = mp_result.worker_stdout if stdout: print(stdout, flush=True) - if self.runtests.rerun and result.is_failed(self.runtests.fail_env_changed): - result.print_github_annotations() return result From 75c44b00d81e346f3ec56338a4ae6402dc9cad45 Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Tue, 6 Oct 2026 12:05:54 -0700 Subject: [PATCH 13/25] Have regrtest workers print annotations in better place --- Lib/test/libregrtest/main.py | 9 +++-- Lib/test/libregrtest/result.py | 53 ++++++++++++++--------------- Lib/test/libregrtest/run_workers.py | 5 ++- Lib/test/libregrtest/runtests.py | 2 ++ Lib/test/libregrtest/setup.py | 3 ++ Lib/test/libregrtest/testresult.py | 22 +++++++++++- Lib/test/libregrtest/utils.py | 23 +++++++++++++ 7 files changed, 82 insertions(+), 35 deletions(-) diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index 9df71bbcf009b9c..2df99a0fa19b123 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -284,7 +284,9 @@ def _rerun_failed_tests(self, runtests: RunTests) -> RunTests: forever=False, fail_fast=False, match_tests_dict=match_tests_dict, - output_on_failure=False) + output_on_failure=False, + # Annotate tests which fail again + github_annotations=bool(os.environ.get("GITHUB_STEP_SUMMARY"))) self.logger.set_tests(runtests) msg = f"Re-running {len(tests)} failed tests in verbose mode" @@ -395,8 +397,8 @@ def run_test( result = run_single_test(test_name, runtests) self.results.accumulate_result(result, runtests) - if runtests.rerun and result.is_failed(runtests.fail_env_changed): - result.print_github_annotations() + if runtests.github_annotations: + result.print_github_crash_annotation(runtests.fail_env_changed) return result @@ -559,6 +561,7 @@ def create_run_tests(self, tests: TestTuple) -> RunTests: hunt_refleak=self.hunt_refleak, test_dir=self.test_dir, use_junit=(self.junit_filename is not None), + github_annotations=False, coverage=self.coverage, memory_limit=self.memory_limit, gc_threshold=self.gc_threshold, diff --git a/Lib/test/libregrtest/result.py b/Lib/test/libregrtest/result.py index f2fe90cab9a8712..6ed97a2148b5b4a 100644 --- a/Lib/test/libregrtest/result.py +++ b/Lib/test/libregrtest/result.py @@ -1,12 +1,12 @@ import dataclasses import json import os -from _colorize import decolor, get_colors # type: ignore[import-not-found] +from _colorize import get_colors # type: ignore[import-not-found] from typing import Any from .utils import ( StrJSON, TestName, FilterTuple, - format_duration, normalize_test_name, print_warning, traceback_location) + format_duration, normalize_test_name, print_warning, github_annotation) @dataclasses.dataclass(slots=True) @@ -179,37 +179,34 @@ def __str__(self) -> str: def has_meaningful_duration(self): return State.has_meaningful_duration(self.state) - def print_github_annotations(self) -> None: - """Print a GitHub Actions error annotation per failed test case.""" - if not os.environ.get("GITHUB_STEP_SUMMARY"): - return - - def escape(text: str) -> str: - return (text.replace("%", "%25").replace("\r", "%0D") - .replace("\n", "%0A")) - - def escape_property(text: str) -> str: - return escape(text).replace(":", "%3A").replace(",", "%2C") + def get_github_annotations(self) -> list[tuple[str, str]]: + """(test case name, message) of each failed test case. + Without test case details (crash, timeout, env changed, etc.), return + the test name and the test result. + """ + if not os.environ.get("GITHUB_STEP_SUMMARY"): + return [] annotations = (self.errors or []) + (self.failures or []) if not annotations: - # No test case details: crash, timeout, env changed, etc. message = "\n".join([str(self), *(self.env_changed_reasons or ())]) annotations = [(self.test_name, message)] - for title, message in annotations: - message = decolor(message) - props: dict[str, str | int] = {} - if location := traceback_location(message): - props |= location - # The job log only shows the message: start it with the location - position = ":".join(str(location[key]) - for key in ("file", "line", "col") - if key in location) - message = f"{position}\n{message}" - props["title"] = title - props_text = ",".join(f"{key}={escape_property(str(value))}" - for key, value in props.items()) - print(f"::error {props_text}::{escape(message)}", flush=True) + return annotations + + def print_github_annotations(self) -> None: + """Print a GitHub Actions error annotation per failed test case.""" + for title, message in self.get_github_annotations(): + print(github_annotation(title, message), flush=True) + + def print_github_crash_annotation(self, fail_env_changed: bool) -> None: + """Annotate a failed test without test case failures. + + Test case failures are annotated by the test runner, where they are + reported (see RegressionTestResult.printErrorList()). + """ + if (self.is_failed(fail_env_changed) + and not self.errors and not self.failures): + self.print_github_annotations() def set_env_changed(self, *reasons): if self.state is None or self.state == State.PASSED: diff --git a/Lib/test/libregrtest/run_workers.py b/Lib/test/libregrtest/run_workers.py index 2000b0b469e254f..45bf9d9327fa143 100644 --- a/Lib/test/libregrtest/run_workers.py +++ b/Lib/test/libregrtest/run_workers.py @@ -613,9 +613,8 @@ def _process_result(self, item: QueueOutput) -> TestResult: result = mp_result.result self.results.accumulate_result(result, self.runtests) self.display_result(mp_result) - if self.runtests.rerun and result.is_failed(self.runtests.fail_env_changed): - # Annotate before the test output - result.print_github_annotations() + if self.runtests.github_annotations: + result.print_github_crash_annotation(self.runtests.fail_env_changed) # Display worker stdout if not self.runtests.output_on_failure: diff --git a/Lib/test/libregrtest/runtests.py b/Lib/test/libregrtest/runtests.py index d86b082fb34a604..a116b6e4166fc63 100644 --- a/Lib/test/libregrtest/runtests.py +++ b/Lib/test/libregrtest/runtests.py @@ -93,6 +93,8 @@ class RunTests: hunt_refleak: HuntRefleak | None test_dir: StrPath | None use_junit: bool + # Print a GitHub Actions error annotation before each failure report + github_annotations: bool coverage: bool memory_limit: str | None gc_threshold: int | None diff --git a/Lib/test/libregrtest/setup.py b/Lib/test/libregrtest/setup.py index d62194acd9c29e5..7531e69d0988e64 100644 --- a/Lib/test/libregrtest/setup.py +++ b/Lib/test/libregrtest/setup.py @@ -119,6 +119,9 @@ def setup_tests(runtests: RunTests) -> None: else: support.junit_xml_list = None + from .testresult import RegressionTestResult + RegressionTestResult.GITHUB_ANNOTATIONS = runtests.github_annotations + if runtests.memory_limit is not None: support.set_memlimit(runtests.memory_limit) diff --git a/Lib/test/libregrtest/testresult.py b/Lib/test/libregrtest/testresult.py index 605f1f4e6a89fb6..cfd024a000e8a3f 100644 --- a/Lib/test/libregrtest/testresult.py +++ b/Lib/test/libregrtest/testresult.py @@ -9,10 +9,11 @@ import traceback import unittest from test import support -from test.libregrtest.utils import sanitize_xml +from test.libregrtest.utils import github_annotation, sanitize_xml class RegressionTestResult(unittest.TextTestResult): USE_XML = False + GITHUB_ANNOTATIONS = False def __init__(self, stream, descriptions, verbosity): super().__init__(stream=stream, descriptions=descriptions, @@ -130,6 +131,25 @@ def addUnexpectedSuccess(self, test): self._add_result(test, outcome='UNEXPECTED_SUCCESS') super().addUnexpectedSuccess(test) + def printErrorList(self, flavour, errors): + if not self.GITHUB_ANNOTATIONS: + super().printErrorList(flavour, errors) + return + stream = self.stream + for test, err in errors: + # Precede each failure report with a GitHub Actions annotation + # which has the report as message, so that the annotation links + # to the report in the job log. + self.stream = unittest.runner._WritelnDecorator(io.StringIO()) + try: + super().printErrorList(flavour, [(test, err)]) + report = self.stream.getvalue() + finally: + self.stream = stream + stream.writeln(github_annotation(str(test), report)) + stream.write(report) + stream.flush() + def get_xml_element(self): if not self.USE_XML: raise ValueError("USE_XML is false") diff --git a/Lib/test/libregrtest/utils.py b/Lib/test/libregrtest/utils.py index e39a995662bc91f..d4218fffbf64aea 100644 --- a/Lib/test/libregrtest/utils.py +++ b/Lib/test/libregrtest/utils.py @@ -14,6 +14,7 @@ import tempfile import textwrap import types +from _colorize import decolor # type: ignore[import-not-found] from collections.abc import Callable _winapi: types.ModuleType | None try: @@ -174,6 +175,28 @@ def traceback_location(traceback: str) -> dict[str, str | int] | None: return location +def github_annotation(title: str, message: str) -> str: + """Format a GitHub Actions error annotation. + + Locate it at the last frame of the traceback in message, if any. + """ + def escape(text: str) -> str: + return (text.replace("%", "%25").replace("\r", "%0D") + .replace("\n", "%0A")) + + def escape_property(text: str) -> str: + return escape(text).replace(":", "%3A").replace(",", "%2C") + + message = decolor(message) + props: dict[str, str | int] = {} + if location := traceback_location(message): + props |= location + props["title"] = title + props_text = ",".join(f"{key}={escape_property(str(value))}" + for key, value in props.items()) + return f"::error {props_text}::{escape(message.rstrip())}" + + def regrtest_unraisable_hook(unraisable) -> None: global orig_unraisablehook support.set_environment_altered( From 1222c34f70a4e66c4ad7c321a7ce7e16df51898e Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Tue, 6 Oct 2026 12:37:38 -0700 Subject: [PATCH 14/25] Full annotations from worker --- Lib/test/libregrtest/utils.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/Lib/test/libregrtest/utils.py b/Lib/test/libregrtest/utils.py index d4218fffbf64aea..4953a024d355988 100644 --- a/Lib/test/libregrtest/utils.py +++ b/Lib/test/libregrtest/utils.py @@ -187,14 +187,22 @@ def escape(text: str) -> str: def escape_property(text: str) -> str: return escape(text).replace(":", "%3A").replace(",", "%2C") - message = decolor(message) + message = decolor(message).rstrip() props: dict[str, str | int] = {} + header = title if location := traceback_location(message): props |= location + position = ":".join(str(location[key]) + for key in ("file", "line", "col") + if key in location) + header = f"{position}: {title}" props["title"] = title + # The job log only shows the message, not the properties: start the + # message with the location and the title + message = f"{header}\n{message}" props_text = ",".join(f"{key}={escape_property(str(value))}" for key, value in props.items()) - return f"::error {props_text}::{escape(message.rstrip())}" + return f"::error {props_text}::{escape(message)}" def regrtest_unraisable_hook(unraisable) -> None: From 287df00e91602e457d2bbb839fa9c4a7ad42f3fa Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Tue, 6 Oct 2026 12:53:04 -0700 Subject: [PATCH 15/25] Remove redundant printing --- Lib/test/libregrtest/testresult.py | 19 ++++++------------- Lib/test/libregrtest/utils.py | 9 ++++++--- 2 files changed, 12 insertions(+), 16 deletions(-) diff --git a/Lib/test/libregrtest/testresult.py b/Lib/test/libregrtest/testresult.py index cfd024a000e8a3f..6ad6c2acdc272db 100644 --- a/Lib/test/libregrtest/testresult.py +++ b/Lib/test/libregrtest/testresult.py @@ -135,20 +135,13 @@ def printErrorList(self, flavour, errors): if not self.GITHUB_ANNOTATIONS: super().printErrorList(flavour, errors) return - stream = self.stream for test, err in errors: - # Precede each failure report with a GitHub Actions annotation - # which has the report as message, so that the annotation links - # to the report in the job log. - self.stream = unittest.runner._WritelnDecorator(io.StringIO()) - try: - super().printErrorList(flavour, [(test, err)]) - report = self.stream.getvalue() - finally: - self.stream = stream - stream.writeln(github_annotation(str(test), report)) - stream.write(report) - stream.flush() + # Precede each failure report with a GitHub Actions annotation, + # so that the annotation links to the report in the job log. The + # report follows: don't repeat it in the annotation. + self.stream.writeln(github_annotation(str(test), err, + include_message=False)) + super().printErrorList(flavour, [(test, err)]) def get_xml_element(self): if not self.USE_XML: diff --git a/Lib/test/libregrtest/utils.py b/Lib/test/libregrtest/utils.py index 4953a024d355988..8e04ba960be45f5 100644 --- a/Lib/test/libregrtest/utils.py +++ b/Lib/test/libregrtest/utils.py @@ -175,10 +175,13 @@ def traceback_location(traceback: str) -> dict[str, str | int] | None: return location -def github_annotation(title: str, message: str) -> str: +def github_annotation(title: str, message: str, *, + include_message: bool = True) -> str: """Format a GitHub Actions error annotation. - Locate it at the last frame of the traceback in message, if any. + Locate it at the last frame of the traceback in message, if any. If + include_message is false, the annotation only has the location and the + title. """ def escape(text: str) -> str: return (text.replace("%", "%25").replace("\r", "%0D") @@ -199,7 +202,7 @@ def escape_property(text: str) -> str: props["title"] = title # The job log only shows the message, not the properties: start the # message with the location and the title - message = f"{header}\n{message}" + message = f"{header}\n{message}" if include_message else header props_text = ",".join(f"{key}={escape_property(str(value))}" for key, value in props.items()) return f"::error {props_text}::{escape(message)}" From 53024ed83c9815984b7f1caba450f018aebb4985 Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Tue, 6 Oct 2026 13:32:22 -0700 Subject: [PATCH 16/25] Remove special summary case, more full output everywhere --- Lib/test/libregrtest/main.py | 11 ++++------- Lib/test/libregrtest/result.py | 28 +++++----------------------- Lib/test/libregrtest/testresult.py | 5 +++++ 3 files changed, 14 insertions(+), 30 deletions(-) diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index 2df99a0fa19b123..58e011ac752ec8c 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -285,7 +285,6 @@ def _rerun_failed_tests(self, runtests: RunTests) -> RunTests: fail_fast=False, match_tests_dict=match_tests_dict, output_on_failure=False, - # Annotate tests which fail again github_annotations=bool(os.environ.get("GITHUB_STEP_SUMMARY"))) self.logger.set_tests(runtests) @@ -495,11 +494,6 @@ def write_github_summary(self) -> None: return # Tests which failed in the last run (the re-run, if any) failed = self.results.rerun_results - if not self.results.rerun: - # Failed tests were not re-run (ex: --python): annotate them now. - # Otherwise, they were annotated when they failed again. - for result in failed: - result.print_github_annotations() cases = [(result.errors or []) + (result.failures or []) for result in failed] ncase = sum(map(len, cases)) @@ -561,7 +555,10 @@ def create_run_tests(self, tests: TestTuple) -> RunTests: hunt_refleak=self.hunt_refleak, test_dir=self.test_dir, use_junit=(self.junit_filename is not None), - github_annotations=False, + # Only annotate failures of the last run: the re-run, if any + github_annotations=(bool(os.environ.get("GITHUB_STEP_SUMMARY")) + and not (self.want_rerun + and not self.python_cmd)), coverage=self.coverage, memory_limit=self.memory_limit, gc_threshold=self.gc_threshold, diff --git a/Lib/test/libregrtest/result.py b/Lib/test/libregrtest/result.py index 6ed97a2148b5b4a..1bb9f3a9a48dd31 100644 --- a/Lib/test/libregrtest/result.py +++ b/Lib/test/libregrtest/result.py @@ -1,6 +1,5 @@ import dataclasses import json -import os from _colorize import get_colors # type: ignore[import-not-found] from typing import Any @@ -179,34 +178,17 @@ def __str__(self) -> str: def has_meaningful_duration(self): return State.has_meaningful_duration(self.state) - def get_github_annotations(self) -> list[tuple[str, str]]: - """(test case name, message) of each failed test case. - - Without test case details (crash, timeout, env changed, etc.), return - the test name and the test result. - """ - if not os.environ.get("GITHUB_STEP_SUMMARY"): - return [] - annotations = (self.errors or []) + (self.failures or []) - if not annotations: - message = "\n".join([str(self), *(self.env_changed_reasons or ())]) - annotations = [(self.test_name, message)] - return annotations - - def print_github_annotations(self) -> None: - """Print a GitHub Actions error annotation per failed test case.""" - for title, message in self.get_github_annotations(): - print(github_annotation(title, message), flush=True) - def print_github_crash_annotation(self, fail_env_changed: bool) -> None: """Annotate a failed test without test case failures. - Test case failures are annotated by the test runner, where they are - reported (see RegressionTestResult.printErrorList()). + For example: crash, timeout, env changed. Test case failures are + annotated by the test runner, where they are reported (see + RegressionTestResult.printErrorList()). """ if (self.is_failed(fail_env_changed) and not self.errors and not self.failures): - self.print_github_annotations() + message = "\n".join([str(self), *(self.env_changed_reasons or ())]) + print(github_annotation(self.test_name, message), flush=True) def set_env_changed(self, *reasons): if self.state is None or self.state == State.PASSED: diff --git a/Lib/test/libregrtest/testresult.py b/Lib/test/libregrtest/testresult.py index 6ad6c2acdc272db..dabd0f5870c01ee 100644 --- a/Lib/test/libregrtest/testresult.py +++ b/Lib/test/libregrtest/testresult.py @@ -154,11 +154,16 @@ def get_xml_element(self): class QuietRegressionTestRunner: def __init__(self, stream, buffer=False): + # Wrap the stream as TextTestRunner does, for printErrors() + stream = unittest.runner._WritelnDecorator(stream) self.result = RegressionTestResult(stream, None, 0) self.result.buffer = buffer def run(self, test): test(self.result) + if self.result.GITHUB_ANNOTATIONS: + # Report annotated failures, as in verbose mode + self.result.printErrors() return self.result def get_test_runner_class(verbosity, buffer=False): From c2d1d79dbc2f8247041baf525c0a45ec9f79ba32 Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Tue, 6 Oct 2026 13:58:11 -0700 Subject: [PATCH 17/25] Better message in the annotations --- Lib/test/libregrtest/testresult.py | 6 ++---- Lib/test/libregrtest/utils.py | 29 +++++++++++++++++++++++------ 2 files changed, 25 insertions(+), 10 deletions(-) diff --git a/Lib/test/libregrtest/testresult.py b/Lib/test/libregrtest/testresult.py index dabd0f5870c01ee..36ee03fc8ed9f74 100644 --- a/Lib/test/libregrtest/testresult.py +++ b/Lib/test/libregrtest/testresult.py @@ -137,10 +137,8 @@ def printErrorList(self, flavour, errors): return for test, err in errors: # Precede each failure report with a GitHub Actions annotation, - # so that the annotation links to the report in the job log. The - # report follows: don't repeat it in the annotation. - self.stream.writeln(github_annotation(str(test), err, - include_message=False)) + # so that the annotation links to the report in the job log + self.stream.writeln(github_annotation(str(test), err)) super().printErrorList(flavour, [(test, err)]) def get_xml_element(self): diff --git a/Lib/test/libregrtest/utils.py b/Lib/test/libregrtest/utils.py index 8e04ba960be45f5..95e69454b90a5ec 100644 --- a/Lib/test/libregrtest/utils.py +++ b/Lib/test/libregrtest/utils.py @@ -175,13 +175,30 @@ def traceback_location(traceback: str) -> dict[str, str | int] | None: return location -def github_annotation(title: str, message: str, *, - include_message: bool = True) -> str: +def traceback_exception(traceback: str) -> str: + """The exception which ends a traceback: type, message and notes. + + Return traceback unchanged if it has no frame. + """ + lines = traceback.splitlines() + for index in reversed(range(len(lines))): + if lines[index].startswith(' File "'): + break + else: + return traceback + # Skip the frame's indented source line and ~^ markers + index += 1 + while index < len(lines) and lines[index].startswith(" "): + index += 1 + return "\n".join(lines[index:]) + + +def github_annotation(title: str, message: str) -> str: """Format a GitHub Actions error annotation. - Locate it at the last frame of the traceback in message, if any. If - include_message is false, the annotation only has the location and the - title. + message is a traceback or a failure description. Locate the annotation at + the last frame of the traceback, if any, and only keep the exception which + ends the traceback: the job log has the full traceback. """ def escape(text: str) -> str: return (text.replace("%", "%25").replace("\r", "%0D") @@ -202,7 +219,7 @@ def escape_property(text: str) -> str: props["title"] = title # The job log only shows the message, not the properties: start the # message with the location and the title - message = f"{header}\n{message}" if include_message else header + message = f"{header}\n{traceback_exception(message).strip()}" props_text = ",".join(f"{key}={escape_property(str(value))}" for key, value in props.items()) return f"::error {props_text}::{escape(message)}" From c3cca61c6863cac97df8f0dfc87d7c618f79c80f Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Tue, 6 Oct 2026 14:25:57 -0700 Subject: [PATCH 18/25] Tweak message --- Lib/test/libregrtest/utils.py | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/Lib/test/libregrtest/utils.py b/Lib/test/libregrtest/utils.py index 95e69454b90a5ec..489acc56a2491ff 100644 --- a/Lib/test/libregrtest/utils.py +++ b/Lib/test/libregrtest/utils.py @@ -207,19 +207,12 @@ def escape(text: str) -> str: def escape_property(text: str) -> str: return escape(text).replace(":", "%3A").replace(",", "%2C") - message = decolor(message).rstrip() + message = decolor(message) props: dict[str, str | int] = {} - header = title if location := traceback_location(message): props |= location - position = ":".join(str(location[key]) - for key in ("file", "line", "col") - if key in location) - header = f"{position}: {title}" props["title"] = title - # The job log only shows the message, not the properties: start the - # message with the location and the title - message = f"{header}\n{traceback_exception(message).strip()}" + message = traceback_exception(message).strip() props_text = ",".join(f"{key}={escape_property(str(value))}" for key, value in props.items()) return f"::error {props_text}::{escape(message)}" From b125f6fe7b184cc95ab363a6696552dbebb8b650 Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Tue, 6 Oct 2026 15:14:34 -0700 Subject: [PATCH 19/25] Move to simpler regex traceback info extraction --- Lib/test/libregrtest/testresult.py | 6 ++- Lib/test/libregrtest/utils.py | 80 +++++++----------------------- 2 files changed, 24 insertions(+), 62 deletions(-) diff --git a/Lib/test/libregrtest/testresult.py b/Lib/test/libregrtest/testresult.py index 36ee03fc8ed9f74..7011120936242f8 100644 --- a/Lib/test/libregrtest/testresult.py +++ b/Lib/test/libregrtest/testresult.py @@ -138,7 +138,11 @@ def printErrorList(self, flavour, errors): for test, err in errors: # Precede each failure report with a GitHub Actions annotation, # so that the annotation links to the report in the job log - self.stream.writeln(github_annotation(str(test), err)) + # Locate the annotation in the test file + case = getattr(test, "test_case", test) # subTest() + module = sys.modules.get(type(case).__module__) + self.stream.writeln(github_annotation( + str(test), err, getattr(module, "__file__", None))) super().printErrorList(flavour, [(test, err)]) def get_xml_element(self): diff --git a/Lib/test/libregrtest/utils.py b/Lib/test/libregrtest/utils.py index 489acc56a2491ff..1eb31fd3bc7b541 100644 --- a/Lib/test/libregrtest/utils.py +++ b/Lib/test/libregrtest/utils.py @@ -1,6 +1,5 @@ import contextlib import faulthandler -import linecache import locale import math import os.path @@ -142,63 +141,13 @@ def print_warning(msg: str) -> None: orig_unraisablehook: Callable[..., None] | None = None -def traceback_location(traceback: str) -> dict[str, str | int] | None: - """Location of the last frame of a traceback, relative to the repository. - - Return a dict with file and line, plus col and endColumn when the - traceback underlines the failing expression (~~~^^^). - """ - lines = traceback.splitlines() - for index in reversed(range(len(lines))): - if match := re.match(r' File "(.+)", line (\d+)', lines[index]): - break - else: - return None - filename, lineno = match.group(1), int(match.group(2)) - # Map the stdlib directory (Lib/ in a source checkout or an installed - # lib/python3.X/) to Lib/ in the repository - stdlib_dir = os.path.dirname(os.__file__) + os.sep - if not filename.startswith(stdlib_dir): - return None - relpath = filename.removeprefix(stdlib_dir).replace(os.sep, "/") - location: dict[str, str | int] = {"file": f"Lib/{relpath}", "line": lineno} - - # The frame is followed by the dedented source line (" code") and, - # for a single-line expression, by a line of ~ and ^ markers. - markers = lines[index + 2] if index + 2 < len(lines) else "" - source = linecache.getline(filename, lineno) - if markers.strip() and not markers.strip(" ~^") and source: - indent = len(source) - len(source.lstrip()) - start = len(markers) - len(markers.lstrip()) - location["col"] = indent + start - 4 + 1 - location["endColumn"] = indent + len(markers.rstrip()) - 4 - return location - - -def traceback_exception(traceback: str) -> str: - """The exception which ends a traceback: type, message and notes. - - Return traceback unchanged if it has no frame. - """ - lines = traceback.splitlines() - for index in reversed(range(len(lines))): - if lines[index].startswith(' File "'): - break - else: - return traceback - # Skip the frame's indented source line and ~^ markers - index += 1 - while index < len(lines) and lines[index].startswith(" "): - index += 1 - return "\n".join(lines[index:]) - - -def github_annotation(title: str, message: str) -> str: +def github_annotation(title: str, message: str, + filename: str | None = None) -> str: """Format a GitHub Actions error annotation. - message is a traceback or a failure description. Locate the annotation at - the last frame of the traceback, if any, and only keep the exception which - ends the traceback: the job log has the full traceback. + message is a traceback or a failure description. Only keep the exception + which ends the traceback: the job log has the full traceback. Locate the + annotation at the last frame of the traceback in filename, if any. """ def escape(text: str) -> str: return (text.replace("%", "%25").replace("\r", "%0D") @@ -208,12 +157,21 @@ def escape_property(text: str) -> str: return escape(text).replace(":", "%3A").replace(",", "%2C") message = decolor(message) - props: dict[str, str | int] = {} - if location := traceback_location(message): - props |= location + props: dict[str, str] = {} + # Map the stdlib directory (Lib/ in a source checkout or an installed + # lib/python3.X/) to Lib/ in the repository + stdlib_dir = os.path.dirname(os.__file__) + os.sep + if filename and filename.startswith(stdlib_dir): + lines = re.findall(rf'^ File "{re.escape(filename)}", line (\d+)', + message, re.MULTILINE) + if lines: + relpath = filename.removeprefix(stdlib_dir).replace(os.sep, "/") + props |= {"file": f"Lib/{relpath}", "line": lines[-1]} props["title"] = title - message = traceback_exception(message).strip() - props_text = ",".join(f"{key}={escape_property(str(value))}" + # Strip the frames: each " File" line and its indented source lines + message = re.split(r'^ File .*\n(?: .*\n)*', message, + flags=re.MULTILINE)[-1].strip() + props_text = ",".join(f"{key}={escape_property(value)}" for key, value in props.items()) return f"::error {props_text}::{escape(message)}" From 3f320dd94958495a1d8e2edb2d958465a3e40e3b Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Tue, 6 Oct 2026 20:16:37 -0700 Subject: [PATCH 20/25] lots of review changes --- Lib/test/libregrtest/main.py | 31 ++-- Lib/test/libregrtest/result.py | 20 +- Lib/test/libregrtest/run_workers.py | 2 +- Lib/test/libregrtest/runtests.py | 2 +- Lib/test/libregrtest/setup.py | 4 +- Lib/test/libregrtest/testresult.py | 32 ++-- Lib/test/libregrtest/utils.py | 39 ++-- Lib/test/test_regrtest.py | 173 ++++++++++++++++++ ...-10-05-22-36-06.gh-issue-158895.3Dmc4J.rst | 5 +- 9 files changed, 253 insertions(+), 55 deletions(-) diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index 58e011ac752ec8c..ccac65aed1e25ea 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -1,3 +1,4 @@ +import html import os import random import re @@ -397,7 +398,7 @@ def run_test( self.results.accumulate_result(result, runtests) if runtests.github_annotations: - result.print_github_crash_annotation(runtests.fail_env_changed) + result.print_github_annotation(runtests) return result @@ -487,16 +488,16 @@ def finalize_tests(self, coverage: trace.CoverageResults | None) -> None: def write_github_summary(self) -> None: filename = os.environ.get("GITHUB_STEP_SUMMARY") + # Tests which failed in the last run (the re-run, if any) + failed = self.results.rerun_results # Only write a summary on failure: a CI run has many test jobs exitcode = self.results.get_exitcode(self.fail_env_changed, self.fail_rerun) - if not filename or not exitcode: + if not filename or not failed or not exitcode: return - # Tests which failed in the last run (the re-run, if any) - failed = self.results.rerun_results cases = [(result.errors or []) + (result.failures or []) for result in failed] - ncase = sum(map(len, cases)) + ncases = sum(map(len, cases)) with open(filename, "a", encoding="utf-8") as fp: def write(text: str) -> None: # Separate Markdown blocks with an empty line @@ -504,17 +505,18 @@ def write(text: str) -> None: write(f"## {decolor(self.get_state())}: " f"{count(len(failed), 'test file')} and " - f"{count(ncase, 'test case')} failed") + f"{count(ncases, 'test case')} failed") for result, result_cases in zip(failed, cases): write(f"### {decolor(str(result))}") if result.env_changed_reasons: write("\n".join(f"- {reason}" for reason in result.env_changed_reasons)) for name, traceback in result_cases: - # Expand short tracebacks when there are only a few - is_open = ncase <= 5 and traceback.count("\n") < 30 + # Expand the tracebacks if they fit on a screen: up to 5 + # test cases with a traceback of less than 30 lines + is_open = ncases <= 5 and traceback.count("\n") < 30 write(f"" - f"{name}") + f"{html.escape(name)}") write(f"```pytb\n{decolor(traceback).rstrip()}\n```") write("") @@ -538,6 +540,12 @@ def display_summary(self) -> None: print(f"Result: {state}") def create_run_tests(self, tests: TestTuple) -> RunTests: + # Annotate test failures in the GitHub Actions job log of the last + # run (the re-run, if any), if it reports failures + will_rerun = self.want_rerun and not self.python_cmd + github_annotations = (bool(os.environ.get("GITHUB_STEP_SUMMARY")) + and not will_rerun + and bool(self.verbose or self.output_on_failure)) return RunTests( tests, fail_fast=self.fail_fast, @@ -555,10 +563,7 @@ def create_run_tests(self, tests: TestTuple) -> RunTests: hunt_refleak=self.hunt_refleak, test_dir=self.test_dir, use_junit=(self.junit_filename is not None), - # Only annotate failures of the last run: the re-run, if any - github_annotations=(bool(os.environ.get("GITHUB_STEP_SUMMARY")) - and not (self.want_rerun - and not self.python_cmd)), + github_annotations=github_annotations, coverage=self.coverage, memory_limit=self.memory_limit, gc_threshold=self.gc_threshold, diff --git a/Lib/test/libregrtest/result.py b/Lib/test/libregrtest/result.py index 1bb9f3a9a48dd31..6cf816fad1438aa 100644 --- a/Lib/test/libregrtest/result.py +++ b/Lib/test/libregrtest/result.py @@ -1,10 +1,12 @@ import dataclasses +import importlib.util import json from _colorize import get_colors # type: ignore[import-not-found] from typing import Any +from .runtests import RunTests from .utils import ( - StrJSON, TestName, FilterTuple, + StrJSON, TestName, FilterTuple, abs_module_name, format_duration, normalize_test_name, print_warning, github_annotation) @@ -178,17 +180,21 @@ def __str__(self) -> str: def has_meaningful_duration(self): return State.has_meaningful_duration(self.state) - def print_github_crash_annotation(self, fail_env_changed: bool) -> None: + def print_github_annotation(self, runtests: RunTests) -> None: """Annotate a failed test without test case failures. - For example: crash, timeout, env changed. Test case failures are - annotated by the test runner, where they are reported (see - RegressionTestResult.printErrorList()). + For example: crash, timeout, env changed. Locate the annotation in the + test file. Test case failures are annotated by the test runner, where + they are reported (see RegressionTestResult.printErrorList()). """ - if (self.is_failed(fail_env_changed) + if (self.is_failed(runtests.fail_env_changed) and not self.errors and not self.failures): message = "\n".join([str(self), *(self.env_changed_reasons or ())]) - print(github_annotation(self.test_name, message), flush=True) + spec = importlib.util.find_spec( + abs_module_name(self.test_name, runtests.test_dir)) + filename = spec.origin if spec is not None else None + print(github_annotation(self.test_name, message, filename), + flush=True) def set_env_changed(self, *reasons): if self.state is None or self.state == State.PASSED: diff --git a/Lib/test/libregrtest/run_workers.py b/Lib/test/libregrtest/run_workers.py index 45bf9d9327fa143..9bc220e6db52094 100644 --- a/Lib/test/libregrtest/run_workers.py +++ b/Lib/test/libregrtest/run_workers.py @@ -614,7 +614,7 @@ def _process_result(self, item: QueueOutput) -> TestResult: self.results.accumulate_result(result, self.runtests) self.display_result(mp_result) if self.runtests.github_annotations: - result.print_github_crash_annotation(self.runtests.fail_env_changed) + result.print_github_annotation(self.runtests) # Display worker stdout if not self.runtests.output_on_failure: diff --git a/Lib/test/libregrtest/runtests.py b/Lib/test/libregrtest/runtests.py index a116b6e4166fc63..06b1ea34a8140d0 100644 --- a/Lib/test/libregrtest/runtests.py +++ b/Lib/test/libregrtest/runtests.py @@ -93,7 +93,7 @@ class RunTests: hunt_refleak: HuntRefleak | None test_dir: StrPath | None use_junit: bool - # Print a GitHub Actions error annotation before each failure report + # Annotate test failures in the GitHub Actions job log github_annotations: bool coverage: bool memory_limit: str | None diff --git a/Lib/test/libregrtest/setup.py b/Lib/test/libregrtest/setup.py index 7531e69d0988e64..21734461d155cc5 100644 --- a/Lib/test/libregrtest/setup.py +++ b/Lib/test/libregrtest/setup.py @@ -112,14 +112,12 @@ def setup_tests(runtests: RunTests) -> None: set_match_tests(runtests.match_tests) + from .testresult import RegressionTestResult if runtests.use_junit: support.junit_xml_list = [] - from .testresult import RegressionTestResult RegressionTestResult.USE_XML = True else: support.junit_xml_list = None - - from .testresult import RegressionTestResult RegressionTestResult.GITHUB_ANNOTATIONS = runtests.github_annotations if runtests.memory_limit is not None: diff --git a/Lib/test/libregrtest/testresult.py b/Lib/test/libregrtest/testresult.py index 7011120936242f8..625a5ea6816aa21 100644 --- a/Lib/test/libregrtest/testresult.py +++ b/Lib/test/libregrtest/testresult.py @@ -4,6 +4,7 @@ import functools import io +import re import sys import time import traceback @@ -132,19 +133,25 @@ def addUnexpectedSuccess(self, test): super().addUnexpectedSuccess(test) def printErrorList(self, flavour, errors): - if not self.GITHUB_ANNOTATIONS: - super().printErrorList(flavour, errors) - return for test, err in errors: - # Precede each failure report with a GitHub Actions annotation, - # so that the annotation links to the report in the job log - # Locate the annotation in the test file - case = getattr(test, "test_case", test) # subTest() - module = sys.modules.get(type(case).__module__) - self.stream.writeln(github_annotation( - str(test), err, getattr(module, "__file__", None))) + if self.GITHUB_ANNOTATIONS: + # Write the annotation just before the failure report, so + # that it links to the report in the job log + self.stream.writeln(github_annotation(str(test), err, + self._test_file(test))) super().printErrorList(flavour, [(test, err)]) + @staticmethod + def _test_file(test): + # Test id: "module.Class.method", "module.function" (doctest), or + # "setUpClass (module.Class)" (error in a class or module fixture) + name = getattr(test, 'test_case', test).id() # subTest() + if match := re.fullmatch(r'\w+ \((.+)\)', name): + name = match[1] + while name and name not in sys.modules: + name = name.rpartition('.')[0] + return getattr(sys.modules.get(name), '__file__', None) + def get_xml_element(self): if not self.USE_XML: raise ValueError("USE_XML is false") @@ -156,16 +163,11 @@ def get_xml_element(self): class QuietRegressionTestRunner: def __init__(self, stream, buffer=False): - # Wrap the stream as TextTestRunner does, for printErrors() - stream = unittest.runner._WritelnDecorator(stream) self.result = RegressionTestResult(stream, None, 0) self.result.buffer = buffer def run(self, test): test(self.result) - if self.result.GITHUB_ANNOTATIONS: - # Report annotated failures, as in verbose mode - self.result.printErrors() return self.result def get_test_runner_class(verbosity, buffer=False): diff --git a/Lib/test/libregrtest/utils.py b/Lib/test/libregrtest/utils.py index 1eb31fd3bc7b541..ec83a0de0d5bc46 100644 --- a/Lib/test/libregrtest/utils.py +++ b/Lib/test/libregrtest/utils.py @@ -141,14 +141,30 @@ def print_warning(msg: str) -> None: orig_unraisablehook: Callable[..., None] | None = None +def repository_path(filename: str) -> str: + """Path of a stdlib file relative to the repository. + + Map the stdlib directory (Lib/ in a source checkout or an installed + lib/python3.X/) to Lib/. Return other paths unchanged. + """ + stdlib_dir = os.path.dirname(os.__file__) + os.sep + if not filename.startswith(stdlib_dir): + return filename + return "Lib/" + filename.removeprefix(stdlib_dir).replace(os.sep, "/") + + def github_annotation(title: str, message: str, - filename: str | None = None) -> str: + filename: str | None) -> str: """Format a GitHub Actions error annotation. message is a traceback or a failure description. Only keep the exception which ends the traceback: the job log has the full traceback. Locate the - annotation at the last frame of the traceback in filename, if any. + annotation in filename, the test file, at the last frame of the traceback + in this file, if any. """ + if not filename: + raise ValueError(f"missing test file of annotation {title!r}") + def escape(text: str) -> str: return (text.replace("%", "%25").replace("\r", "%0D") .replace("\n", "%0A")) @@ -157,18 +173,15 @@ def escape_property(text: str) -> str: return escape(text).replace(":", "%3A").replace(",", "%2C") message = decolor(message) - props: dict[str, str] = {} - # Map the stdlib directory (Lib/ in a source checkout or an installed - # lib/python3.X/) to Lib/ in the repository - stdlib_dir = os.path.dirname(os.__file__) + os.sep - if filename and filename.startswith(stdlib_dir): - lines = re.findall(rf'^ File "{re.escape(filename)}", line (\d+)', - message, re.MULTILINE) - if lines: - relpath = filename.removeprefix(stdlib_dir).replace(os.sep, "/") - props |= {"file": f"Lib/{relpath}", "line": lines[-1]} + props = {"file": repository_path(filename)} + # Line of the last traceback frame (' File "filename", line 123') or + # failed doctest example (same, but not indented) in filename + if lines := re.findall(rf'^ *File "{re.escape(filename)}", line (\d+)', + message, re.MULTILINE): + props["line"] = lines[-1] props["title"] = title - # Strip the frames: each " File" line and its indented source lines + # Only keep the exception: strip the traceback frames, each one a + # ' File' line followed by its indented source lines message = re.split(r'^ File .*\n(?: .*\n)*', message, flags=re.MULTILINE)[-1].strip() props_text = ",".join(f"{key}={escape_property(value)}" diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index 53f0a2863dc4e45..ed5533bfe971ed0 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -856,6 +856,8 @@ def run_command(self, args, input=None, exitcode=0, **kw): if env is None: env = dict(os.environ) env.pop('SOURCE_DATE_EPOCH', None) + # Don't annotate the GitHub Actions job of the test suite + env.pop('GITHUB_STEP_SUMMARY', None) proc = subprocess.run(args, text=True, @@ -1639,6 +1641,177 @@ def test_env_changed(self): success=True), stats=2) + def create_marked_test(self, name, code): + # Create a test: return its name and the line number of each of its + # "# marker" comments + code = textwrap.dedent(code) + name = self.create_test(name, code) + markers = {match[1]: str(lineno) + for lineno, text in enumerate(code.splitlines(), 1) + if (match := re.search(r'# (\w+)$', text))} + return name, markers + + def run_tests_github(self, *args, exitcode=0): + # Run tests as in GitHub Actions: return the output and the job + # summary (None if regrtest didn't write it) + filename = os.path.join(self.tmptestdir, 'github_step_summary.md') + self.addCleanup(os_helper.unlink, filename) + os_helper.unlink(filename) + env = dict(os.environ) + env.pop('SOURCE_DATE_EPOCH', None) + env['GITHUB_STEP_SUMMARY'] = filename + output = self.run_tests(*args, env=env, exitcode=exitcode) + try: + with open(filename, encoding='utf-8') as fp: + return output, fp.read() + except FileNotFoundError: + return output, None + + @staticmethod + def parse_github_annotations(output): + # Return the (title, file, line) of each "::error" annotation + def unescape(text): + return (text.replace('%3A', ':').replace('%2C', ',') + .replace('%0A', '\n').replace('%0D', '\r') + .replace('%25', '%')) + + annotations = [] + for match in re.finditer(r'^::error (.*?)::', output, re.MULTILINE): + props = dict(prop.split('=', 1) for prop in match[1].split(',')) + props = {key: unescape(value) for key, value in props.items()} + annotations.append((props['title'], props['file'], + props.get('line'))) + return annotations + + def test_github_annotations(self): + # Every test failure is annotated once in the GitHub Actions job log, + # located in the test file, at the failing line of the test file if + # there is a traceback. The job summary lists the failures. + cases, lines = self.create_marked_test('github_cases', """ + import doctest, json, sys, unittest + + def load_tests(loader, tests, pattern): + tests.addTests(doctest.DocTestSuite(sys.modules[__name__])) + return tests + + def doctest_fail(): + ''' + >>> 1 + 1 # doctest + 3 + ''' + + class Tests(unittest.TestCase): + def test_error(self): + json.loads("{") # error + + def test_fail(self): + self.assertEqual(1, 2) # fail + + def test_subtest(self): + with self.subTest(x=1.5): + self.fail("subtest") # subtest + + class SetUpClassTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + raise ValueError("setUpClass") # setUpClass + + def test_never(self): + pass + """) + setup_module, setup_module_lines = self.create_marked_test( + 'github_setup_module', """ + import unittest + + def setUpModule(): + raise ValueError("setUpModule") # setUpModule + + class Tests(unittest.TestCase): + def test_never(self): + pass + """) + env_changed, _ = self.create_marked_test('github_env_changed', """ + import os, unittest + + class Tests(unittest.TestCase): + def test_env_changed(self): + os.environ["REGRTEST_GITHUB_ENV_CHANGED"] = "1" + """) + crash, _ = self.create_marked_test('github_crash', """ + import os, unittest + + class Tests(unittest.TestCase): + def test_crash(self): + os._exit(1) + """) + + def path(name): + return os.path.join(self.tmptestdir, f'{name}.py') + + # Test cases: (title, file, line) + test_cases = [ + (f'test_error ({cases}.Tests.test_error)', + path(cases), lines['error']), + (f'test_fail ({cases}.Tests.test_fail)', + path(cases), lines['fail']), + (f'test_subtest ({cases}.Tests.test_subtest) (x=1.5)', + path(cases), lines['subtest']), + # Doctest examples are subtests: "[0]" is the example index + (f'doctest_fail ({cases}) [0]', + path(cases), lines['doctest']), + (f'setUpClass ({cases}.SetUpClassTests)', + path(cases), lines['setUpClass']), + (f'setUpModule ({setup_module})', + path(setup_module), setup_module_lines['setUpModule']), + ] + # Test files: heading in the job summary + test_files = { + cases: f'### {cases} failed (2 errors, 3 failures)', + setup_module: f'### {setup_module} failed (1 error)', + env_changed: f'### {env_changed} failed (env changed)', + crash: f'### {crash} worker non-zero exit code', + } + + for args in (['-j2', '-W'], ['-j2', '--rerun'], ['-j0', '-W']): + tests = list(test_files) + if '-j0' in args: + # A crash kills the main process + tests.remove(crash) + with self.subTest(args=args): + output, summary = self.run_tests_github( + '--fail-env-changed', *args, *tests, + exitcode=EXITCODE_BAD_TEST) + + self.assertCountEqual( + self.parse_github_annotations(output), + [*test_cases, + *((name, path(name), None) + for name in (env_changed, crash) if name in tests)], + output) + + # The job summary lists the failed tests in completion order + summary_lines = summary.splitlines() + self.assertEqual(summary_lines[0], + f'## FAILURE: {len(tests)} test files and ' + f'{len(test_cases)} test cases failed') + self.assertCountEqual( + [line for line in summary_lines + if line.startswith('### ')], + [test_files[name] for name in tests]) + self.assertIn('- os.environ was modified', summary_lines) + self.assertCountEqual( + re.findall(r'(.*)', summary), + [title for title, _, _ in test_cases]) + self.assertEqual(summary.count('```pytb\n'), + len(test_cases)) + + def test_github_summary_success(self): + # No annotation and no job summary when all tests pass + testname = self.create_test() + output, summary = self.run_tests_github(testname) + self.assertNotIn('::error', output) + self.assertIsNone(summary) + def test_rerun_fail(self): # FAILURE then FAILURE code = textwrap.dedent(""" diff --git a/Misc/NEWS.d/next/Tests/2026-10-05-22-36-06.gh-issue-158895.3Dmc4J.rst b/Misc/NEWS.d/next/Tests/2026-10-05-22-36-06.gh-issue-158895.3Dmc4J.rst index ebeca7174e5903a..a1c9e648e5c744e 100644 --- a/Misc/NEWS.d/next/Tests/2026-10-05-22-36-06.gh-issue-158895.3Dmc4J.rst +++ b/Misc/NEWS.d/next/Tests/2026-10-05-22-36-06.gh-issue-158895.3Dmc4J.rst @@ -1,2 +1,3 @@ -Individual test failures and an overall summary is now written by -:mod:`test.regrtest` to the Github Action Summary. +When run in GitHub Actions, :mod:`test.regrtest` now annotates each test +failure in the job log and writes a summary of the failures to the job +summary. From 9b1ce485f33570cc756c8ca3424a0d040982de39 Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Tue, 6 Oct 2026 21:20:44 -0700 Subject: [PATCH 21/25] Add comment --- Lib/test/test_regrtest.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index ed5533bfe971ed0..25d9c0364396134 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -1772,7 +1772,10 @@ def path(name): crash: f'### {crash} worker non-zero exit code', } - for args in (['-j2', '-W'], ['-j2', '--rerun'], ['-j0', '-W']): + # Only failures of the last run are annotated: with --rerun and -W, + # failures of the first run are reported, but not annotated + for args in (['-j2', '-W'], ['-j2', '--rerun'], + ['-j2', '-W', '--rerun'], ['-j0', '-W']): tests = list(test_files) if '-j0' in args: # A crash kills the main process From b5610603c83b2ccd5f209b3a61998bd159ff31d1 Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Wed, 7 Oct 2026 09:56:03 -0700 Subject: [PATCH 22/25] Annotations after failure, better test_regrtest coverage --- Lib/test/libregrtest/testresult.py | 4 +-- Lib/test/test_regrtest.py | 55 ++++++++++++++++++++---------- 2 files changed, 39 insertions(+), 20 deletions(-) diff --git a/Lib/test/libregrtest/testresult.py b/Lib/test/libregrtest/testresult.py index 625a5ea6816aa21..45a363b155cb3d9 100644 --- a/Lib/test/libregrtest/testresult.py +++ b/Lib/test/libregrtest/testresult.py @@ -134,12 +134,12 @@ def addUnexpectedSuccess(self, test): def printErrorList(self, flavour, errors): for test, err in errors: + super().printErrorList(flavour, [(test, err)]) if self.GITHUB_ANNOTATIONS: - # Write the annotation just before the failure report, so + # Write the annotation just after the failure report, so # that it links to the report in the job log self.stream.writeln(github_annotation(str(test), err, self._test_file(test))) - super().printErrorList(flavour, [(test, err)]) @staticmethod def _test_file(test): diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index 25d9c0364396134..0cecacc4b076d29 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -1772,25 +1772,44 @@ def path(name): crash: f'### {crash} worker non-zero exit code', } - # Only failures of the last run are annotated: with --rerun and -W, - # failures of the first run are reported, but not annotated - for args in (['-j2', '-W'], ['-j2', '--rerun'], - ['-j2', '-W', '--rerun'], ['-j0', '-W']): - tests = list(test_files) - if '-j0' in args: - # A crash kills the main process - tests.remove(crash) - with self.subTest(args=args): + # Representative command lines of the CI jobs + command_lines = { + # "make ci", Windows, macOS, installed Python; the JIT jobs use + # the equivalent "-j0 --verbose2 --verbose3". Only failures of + # the re-run are annotated, not the reported first failures. + 'fast-ci': ['--fast-ci', '-j2'], + # Sanitizers and Hypothesis: env changed is not a failure + 'parallel': ['-j2', '-W'], + # iOS and WASI run tests in the main process: a crash kills it + 'single-process': ['--fast-ci', '--single-process'], + # Profile task of PGO and BOLT builds: no annotations + 'pgo': ['--pgo'], + } + for name, args in command_lines.items(): + with self.subTest(name): + tests = list(test_files) + if '--single-process' in args or '--pgo' in args: + tests.remove(crash) output, summary = self.run_tests_github( - '--fail-env-changed', *args, *tests, - exitcode=EXITCODE_BAD_TEST) - - self.assertCountEqual( - self.parse_github_annotations(output), - [*test_cases, - *((name, path(name), None) - for name in (env_changed, crash) if name in tests)], - output) + *args, *tests, exitcode=EXITCODE_BAD_TEST) + + annotations = [] + if '--pgo' not in args: + annotations += test_cases + if '--fast-ci' in args: + annotations.append((env_changed, path(env_changed), + None)) + if crash in tests: + annotations.append((crash, path(crash), None)) + self.assertCountEqual(self.parse_github_annotations(output), + annotations, output) + # Each test case is annotated right after its failure report + for title, _, _ in annotations[:len(test_cases)]: + title = re.escape(title) + self.assertRegex( + output, + rf'(?m)^(?:ERROR|FAIL): {title}\n' + rf'(?:(?!={{70}}$).*\n)*?::error .*title={title}::') # The job summary lists the failed tests in completion order summary_lines = summary.splitlines() From e506d7d940bbcb119a0d9aec20405197a6adf7f5 Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Wed, 7 Oct 2026 10:31:23 -0700 Subject: [PATCH 23/25] annotate env changed better. pgo runs --- Lib/test/libregrtest/main.py | 5 +++-- Lib/test/libregrtest/run_workers.py | 5 +++-- Lib/test/libregrtest/save_env.py | 19 +++++++++++++++- Lib/test/test_regrtest.py | 35 ++++++++++++++++++++--------- 4 files changed, 49 insertions(+), 15 deletions(-) diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index ccac65aed1e25ea..290bc628f2e7356 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -541,11 +541,12 @@ def display_summary(self) -> None: def create_run_tests(self, tests: TestTuple) -> RunTests: # Annotate test failures in the GitHub Actions job log of the last - # run (the re-run, if any), if it reports failures + # run (the re-run, if any), if it reports failures: -v, -W or --pgo will_rerun = self.want_rerun and not self.python_cmd github_annotations = (bool(os.environ.get("GITHUB_STEP_SUMMARY")) and not will_rerun - and bool(self.verbose or self.output_on_failure)) + and bool(self.verbose or self.output_on_failure + or self.pgo)) return RunTests( tests, fail_fast=self.fail_fast, diff --git a/Lib/test/libregrtest/run_workers.py b/Lib/test/libregrtest/run_workers.py index 9bc220e6db52094..4a83f8389870467 100644 --- a/Lib/test/libregrtest/run_workers.py +++ b/Lib/test/libregrtest/run_workers.py @@ -613,8 +613,6 @@ def _process_result(self, item: QueueOutput) -> TestResult: result = mp_result.result self.results.accumulate_result(result, self.runtests) self.display_result(mp_result) - if self.runtests.github_annotations: - result.print_github_annotation(self.runtests) # Display worker stdout if not self.runtests.output_on_failure: @@ -626,6 +624,9 @@ def _process_result(self, item: QueueOutput) -> TestResult: stdout = mp_result.worker_stdout if stdout: print(stdout, flush=True) + # Annotate after the output: env changed warnings, crash traceback + if self.runtests.github_annotations: + result.print_github_annotation(self.runtests) return result diff --git a/Lib/test/libregrtest/save_env.py b/Lib/test/libregrtest/save_env.py index 4149210fa173109..6a902eaf91f5f42 100644 --- a/Lib/test/libregrtest/save_env.py +++ b/Lib/test/libregrtest/save_env.py @@ -139,6 +139,18 @@ def restore_os_environ(self, saved_environ): os.environ = saved_environ[1] os.environ.clear() os.environ.update(saved_environ[2]) + @staticmethod + def describe_os_environ(original, current): + # Only list the keys: values can be secrets + before, after = original[2], current[2] + changes = ( + ('added', after.keys() - before.keys()), + ('removed', before.keys() - after.keys()), + ('changed', {key for key in before.keys() & after.keys() + if before[key] != after[key]}), + ) + return '; '.join(f'{change} {", ".join(sorted(keys))}' + for change, keys in changes if keys) def get_sys_path(self): return id(sys.path), sys.path, sys.path[:] @@ -347,7 +359,12 @@ def __exit__(self, exc_type, exc_val, exc_tb): current = get() # Check for changes to the resource's value if current != original: - support.set_environment_altered(f"{name} was modified") + reason = f"{name} was modified" + if name == 'os.environ': + delta = self.describe_os_environ(original, current) + if delta: + reason = f"{reason}: {delta}" + support.set_environment_altered(reason) restore(original) if not self.quiet and not self.pgo: print_warning( diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index 0cecacc4b076d29..a85304262a1d7e8 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -38,6 +38,7 @@ from test.libregrtest import utils from test.libregrtest.filter import get_match_tests, match_test from test.libregrtest.result import TestStats +from test.libregrtest.save_env import saved_test_environment from test.libregrtest.utils import normalize_test_name if not support.has_subprocess_support: @@ -1782,7 +1783,7 @@ def path(name): 'parallel': ['-j2', '-W'], # iOS and WASI run tests in the main process: a crash kills it 'single-process': ['--fast-ci', '--single-process'], - # Profile task of PGO and BOLT builds: no annotations + # Profile task of PGO and BOLT builds: failures stop the build 'pgo': ['--pgo'], } for name, args in command_lines.items(): @@ -1793,14 +1794,11 @@ def path(name): output, summary = self.run_tests_github( *args, *tests, exitcode=EXITCODE_BAD_TEST) - annotations = [] - if '--pgo' not in args: - annotations += test_cases - if '--fast-ci' in args: - annotations.append((env_changed, path(env_changed), - None)) - if crash in tests: - annotations.append((crash, path(crash), None)) + annotations = list(test_cases) + if '--fast-ci' in args: + annotations.append((env_changed, path(env_changed), None)) + if crash in tests: + annotations.append((crash, path(crash), None)) self.assertCountEqual(self.parse_github_annotations(output), annotations, output) # Each test case is annotated right after its failure report @@ -1810,6 +1808,13 @@ def path(name): output, rf'(?m)^(?:ERROR|FAIL): {title}\n' rf'(?:(?!={{70}}$).*\n)*?::error .*title={title}::') + # The env changed annotation comes right after the warnings + if '--fast-ci' in args: + self.assertRegex( + output, + rf'(?m)^Warning -- os\.environ was modified by ' + rf'{env_changed}\n(?:Warning -- .*\n)*' + rf'::error .*title={env_changed}::') # The job summary lists the failed tests in completion order summary_lines = summary.splitlines() @@ -1820,7 +1825,9 @@ def path(name): [line for line in summary_lines if line.startswith('### ')], [test_files[name] for name in tests]) - self.assertIn('- os.environ was modified', summary_lines) + self.assertIn('- os.environ was modified: ' + 'added REGRTEST_GITHUB_ENV_CHANGED', + summary_lines) self.assertCountEqual( re.findall(r'(.*)', summary), [title for title, _, _ in test_cases]) @@ -2950,6 +2957,14 @@ def worker(): class TestUtils(unittest.TestCase): + def test_describe_os_environ(self): + describe = saved_test_environment.describe_os_environ + before = {'A': '1', 'B': '2', 'C': '3'} + after = {'A': '1', 'B': 'secret', 'D': '4'} + self.assertEqual(describe((0, None, before), (0, None, after)), + 'added D; removed C; changed B') + self.assertEqual(describe((0, None, before), (1, None, before)), '') + def test_format_duration(self): self.assertEqual(utils.format_duration(0), '0 ms') From 2ca705b3556086a24fa82cff579a7ec021e41931 Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Wed, 7 Oct 2026 12:20:39 -0700 Subject: [PATCH 24/25] simplify, rely on env var everywhere. Stop using findspec --- Lib/test/libregrtest/main.py | 28 ++++++++----------- Lib/test/libregrtest/result.py | 21 ++++++++------ Lib/test/libregrtest/run_workers.py | 3 +- Lib/test/libregrtest/runtests.py | 2 -- Lib/test/libregrtest/save_env.py | 19 +------------ Lib/test/libregrtest/setup.py | 1 - Lib/test/libregrtest/testresult.py | 12 ++++---- Lib/test/libregrtest/utils.py | 27 +++++++++--------- Lib/test/libregrtest/worker.py | 2 -- .../test_live_collector_ui.py | 6 +++- Lib/test/test_regrtest.py | 18 +++--------- 11 files changed, 55 insertions(+), 84 deletions(-) diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index 290bc628f2e7356..20b94eff8aa3fdd 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -285,8 +285,7 @@ def _rerun_failed_tests(self, runtests: RunTests) -> RunTests: forever=False, fail_fast=False, match_tests_dict=match_tests_dict, - output_on_failure=False, - github_annotations=bool(os.environ.get("GITHUB_STEP_SUMMARY"))) + output_on_failure=False) self.logger.set_tests(runtests) msg = f"Re-running {len(tests)} failed tests in verbose mode" @@ -397,8 +396,7 @@ def run_test( result = run_single_test(test_name, runtests) self.results.accumulate_result(result, runtests) - if runtests.github_annotations: - result.print_github_annotation(runtests) + result.print_github_annotation(runtests) return result @@ -540,13 +538,6 @@ def display_summary(self) -> None: print(f"Result: {state}") def create_run_tests(self, tests: TestTuple) -> RunTests: - # Annotate test failures in the GitHub Actions job log of the last - # run (the re-run, if any), if it reports failures: -v, -W or --pgo - will_rerun = self.want_rerun and not self.python_cmd - github_annotations = (bool(os.environ.get("GITHUB_STEP_SUMMARY")) - and not will_rerun - and bool(self.verbose or self.output_on_failure - or self.pgo)) return RunTests( tests, fail_fast=self.fail_fast, @@ -564,7 +555,6 @@ def create_run_tests(self, tests: TestTuple) -> RunTests: hunt_refleak=self.hunt_refleak, test_dir=self.test_dir, use_junit=(self.junit_filename is not None), - github_annotations=github_annotations, coverage=self.coverage, memory_limit=self.memory_limit, gc_threshold=self.gc_threshold, @@ -627,10 +617,16 @@ def _run_tests(self, selected: TestTuple, tests: TestList | None) -> int: if use_load_tracker: self.logger.start_load_tracker() try: - if self.num_workers: - self._run_tests_mp(runtests, self.num_workers) - else: - self.run_tests_sequentially(runtests) + with os_helper.EnvironmentVarGuard() as env: + # In GitHub Actions, only annotate failures of the last run: + # not the first run if failed tests will be re-run (with + # --python, they are not: see rerun_failed_tests()) + if self.want_rerun and not self.python_cmd: + env.unset("GITHUB_STEP_SUMMARY") + if self.num_workers: + self._run_tests_mp(runtests, self.num_workers) + else: + self.run_tests_sequentially(runtests) coverage = self.results.get_coverage_results() self.display_result(runtests) diff --git a/Lib/test/libregrtest/result.py b/Lib/test/libregrtest/result.py index 6cf816fad1438aa..e1d05777598f2e9 100644 --- a/Lib/test/libregrtest/result.py +++ b/Lib/test/libregrtest/result.py @@ -1,13 +1,15 @@ import dataclasses -import importlib.util import json +import os from _colorize import get_colors # type: ignore[import-not-found] from typing import Any +from .findtests import findtestdir from .runtests import RunTests from .utils import ( - StrJSON, TestName, FilterTuple, abs_module_name, - format_duration, normalize_test_name, print_warning, github_annotation) + StrJSON, TestName, FilterTuple, + format_duration, normalize_test_name, print_warning, + print_github_annotation) @dataclasses.dataclass(slots=True) @@ -190,11 +192,14 @@ def print_github_annotation(self, runtests: RunTests) -> None: if (self.is_failed(runtests.fail_env_changed) and not self.errors and not self.failures): message = "\n".join([str(self), *(self.env_changed_reasons or ())]) - spec = importlib.util.find_spec( - abs_module_name(self.test_name, runtests.test_dir)) - filename = spec.origin if spec is not None else None - print(github_annotation(self.test_name, message, filename), - flush=True) + # Test file: "test_x" is test_x.py or test_x/__init__.py, + # "test.test_x.test_y" is test_x/test_y.py + name = self.test_name.removeprefix("test.") + path = os.path.join(findtestdir(runtests.test_dir), *name.split(".")) + filename = path + ".py" + if not os.path.exists(filename): + filename = os.path.join(path, "__init__.py") + print_github_annotation(self.test_name, message, filename) def set_env_changed(self, *reasons): if self.state is None or self.state == State.PASSED: diff --git a/Lib/test/libregrtest/run_workers.py b/Lib/test/libregrtest/run_workers.py index 4a83f8389870467..995543d676876cc 100644 --- a/Lib/test/libregrtest/run_workers.py +++ b/Lib/test/libregrtest/run_workers.py @@ -625,8 +625,7 @@ def _process_result(self, item: QueueOutput) -> TestResult: if stdout: print(stdout, flush=True) # Annotate after the output: env changed warnings, crash traceback - if self.runtests.github_annotations: - result.print_github_annotation(self.runtests) + result.print_github_annotation(self.runtests) return result diff --git a/Lib/test/libregrtest/runtests.py b/Lib/test/libregrtest/runtests.py index 06b1ea34a8140d0..d86b082fb34a604 100644 --- a/Lib/test/libregrtest/runtests.py +++ b/Lib/test/libregrtest/runtests.py @@ -93,8 +93,6 @@ class RunTests: hunt_refleak: HuntRefleak | None test_dir: StrPath | None use_junit: bool - # Annotate test failures in the GitHub Actions job log - github_annotations: bool coverage: bool memory_limit: str | None gc_threshold: int | None diff --git a/Lib/test/libregrtest/save_env.py b/Lib/test/libregrtest/save_env.py index 6a902eaf91f5f42..4149210fa173109 100644 --- a/Lib/test/libregrtest/save_env.py +++ b/Lib/test/libregrtest/save_env.py @@ -139,18 +139,6 @@ def restore_os_environ(self, saved_environ): os.environ = saved_environ[1] os.environ.clear() os.environ.update(saved_environ[2]) - @staticmethod - def describe_os_environ(original, current): - # Only list the keys: values can be secrets - before, after = original[2], current[2] - changes = ( - ('added', after.keys() - before.keys()), - ('removed', before.keys() - after.keys()), - ('changed', {key for key in before.keys() & after.keys() - if before[key] != after[key]}), - ) - return '; '.join(f'{change} {", ".join(sorted(keys))}' - for change, keys in changes if keys) def get_sys_path(self): return id(sys.path), sys.path, sys.path[:] @@ -359,12 +347,7 @@ def __exit__(self, exc_type, exc_val, exc_tb): current = get() # Check for changes to the resource's value if current != original: - reason = f"{name} was modified" - if name == 'os.environ': - delta = self.describe_os_environ(original, current) - if delta: - reason = f"{reason}: {delta}" - support.set_environment_altered(reason) + support.set_environment_altered(f"{name} was modified") restore(original) if not self.quiet and not self.pgo: print_warning( diff --git a/Lib/test/libregrtest/setup.py b/Lib/test/libregrtest/setup.py index 21734461d155cc5..0c7fad58fd03bf6 100644 --- a/Lib/test/libregrtest/setup.py +++ b/Lib/test/libregrtest/setup.py @@ -118,7 +118,6 @@ def setup_tests(runtests: RunTests) -> None: RegressionTestResult.USE_XML = True else: support.junit_xml_list = None - RegressionTestResult.GITHUB_ANNOTATIONS = runtests.github_annotations if runtests.memory_limit is not None: support.set_memlimit(runtests.memory_limit) diff --git a/Lib/test/libregrtest/testresult.py b/Lib/test/libregrtest/testresult.py index 45a363b155cb3d9..9c453a9ef585162 100644 --- a/Lib/test/libregrtest/testresult.py +++ b/Lib/test/libregrtest/testresult.py @@ -10,11 +10,10 @@ import traceback import unittest from test import support -from test.libregrtest.utils import github_annotation, sanitize_xml +from test.libregrtest.utils import print_github_annotation, sanitize_xml class RegressionTestResult(unittest.TextTestResult): USE_XML = False - GITHUB_ANNOTATIONS = False def __init__(self, stream, descriptions, verbosity): super().__init__(stream=stream, descriptions=descriptions, @@ -135,11 +134,10 @@ def addUnexpectedSuccess(self, test): def printErrorList(self, flavour, errors): for test, err in errors: super().printErrorList(flavour, [(test, err)]) - if self.GITHUB_ANNOTATIONS: - # Write the annotation just after the failure report, so - # that it links to the report in the job log - self.stream.writeln(github_annotation(str(test), err, - self._test_file(test))) + # Annotate just after the failure report, so that the annotation + # links to the report in the job log + print_github_annotation(str(test), err, self._test_file(test), + file=self.stream) @staticmethod def _test_file(test): diff --git a/Lib/test/libregrtest/utils.py b/Lib/test/libregrtest/utils.py index ec83a0de0d5bc46..60921d5e018a123 100644 --- a/Lib/test/libregrtest/utils.py +++ b/Lib/test/libregrtest/utils.py @@ -153,25 +153,25 @@ def repository_path(filename: str) -> str: return "Lib/" + filename.removeprefix(stdlib_dir).replace(os.sep, "/") -def github_annotation(title: str, message: str, - filename: str | None) -> str: - """Format a GitHub Actions error annotation. +# Escape the data and the properties of GitHub Actions workflow commands +GITHUB_ESCAPE_DATA = str.maketrans({"%": "%25", "\r": "%0D", "\n": "%0A"}) +GITHUB_ESCAPE_PROPERTY = GITHUB_ESCAPE_DATA | str.maketrans({":": "%3A", + ",": "%2C"}) + + +def print_github_annotation(title: str, message: str, filename: str | None, + file=None) -> None: + """Print a GitHub Actions error annotation, if run in GitHub Actions. message is a traceback or a failure description. Only keep the exception which ends the traceback: the job log has the full traceback. Locate the annotation in filename, the test file, at the last frame of the traceback in this file, if any. """ + if not os.environ.get("GITHUB_STEP_SUMMARY"): + return if not filename: raise ValueError(f"missing test file of annotation {title!r}") - - def escape(text: str) -> str: - return (text.replace("%", "%25").replace("\r", "%0D") - .replace("\n", "%0A")) - - def escape_property(text: str) -> str: - return escape(text).replace(":", "%3A").replace(",", "%2C") - message = decolor(message) props = {"file": repository_path(filename)} # Line of the last traceback frame (' File "filename", line 123') or @@ -184,9 +184,10 @@ def escape_property(text: str) -> str: # ' File' line followed by its indented source lines message = re.split(r'^ File .*\n(?: .*\n)*', message, flags=re.MULTILINE)[-1].strip() - props_text = ",".join(f"{key}={escape_property(value)}" + props_text = ",".join(f"{key}={value.translate(GITHUB_ESCAPE_PROPERTY)}" for key, value in props.items()) - return f"::error {props_text}::{escape(message)}" + print(f"::error {props_text}::{message.translate(GITHUB_ESCAPE_DATA)}", + file=file, flush=True) def regrtest_unraisable_hook(unraisable) -> None: diff --git a/Lib/test/libregrtest/worker.py b/Lib/test/libregrtest/worker.py index 24ae95ed23914f6..4e69ab9d8fad05f 100644 --- a/Lib/test/libregrtest/worker.py +++ b/Lib/test/libregrtest/worker.py @@ -28,8 +28,6 @@ def create_worker_process(runtests: WorkerRunTests, output_fd: int, cmd.extend(['-m', 'test.libregrtest.worker', worker_json]) env = dict(os.environ) - # Only the main process writes the GitHub Actions job summary - env.pop('GITHUB_STEP_SUMMARY', None) if tmp_dir is not None: env['TMPDIR'] = tmp_dir env['TEMP'] = tmp_dir diff --git a/Lib/test/test_profiling/test_sampling_profiler/test_live_collector_ui.py b/Lib/test/test_profiling/test_sampling_profiler/test_live_collector_ui.py index ae4ee969f34d8b3..57598715f8b2880 100644 --- a/Lib/test/test_profiling/test_sampling_profiler/test_live_collector_ui.py +++ b/Lib/test/test_profiling/test_sampling_profiler/test_live_collector_ui.py @@ -11,7 +11,8 @@ import time import unittest from unittest import mock -from test.support import requires, requires_remote_subprocess_debugging +from test.support import ( + os_helper, requires, requires_remote_subprocess_debugging) from test.support.import_helper import import_module # Only run these tests if curses is available @@ -856,6 +857,9 @@ def mock_init_curses_side_effect(self, n_times, mock_self, stdscr): def test_run_failed_module_live(self): """Test that running a existing module that fails exits with clean error.""" + # Don't write to the GitHub Actions job summary of the test suite + env = self.enterContext(os_helper.EnvironmentVarGuard()) + env.unset('GITHUB_STEP_SUMMARY') args = [ "profiling.sampling.cli", "run", "--live", "-m", "test", diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index a85304262a1d7e8..0de0e76665baf5a 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -38,7 +38,6 @@ from test.libregrtest import utils from test.libregrtest.filter import get_match_tests, match_test from test.libregrtest.result import TestStats -from test.libregrtest.save_env import saved_test_environment from test.libregrtest.utils import normalize_test_name if not support.has_subprocess_support: @@ -625,6 +624,9 @@ class BaseTestCase(unittest.TestCase): def setUp(self): self.testdir = os.path.realpath(os.path.dirname(__file__)) + # Don't annotate the GitHub Actions job running the test suite + env = self.enterContext(os_helper.EnvironmentVarGuard()) + env.unset('GITHUB_STEP_SUMMARY') self.tmptestdir = tempfile.mkdtemp() self.addCleanup(os_helper.rmtree, self.tmptestdir) @@ -857,8 +859,6 @@ def run_command(self, args, input=None, exitcode=0, **kw): if env is None: env = dict(os.environ) env.pop('SOURCE_DATE_EPOCH', None) - # Don't annotate the GitHub Actions job of the test suite - env.pop('GITHUB_STEP_SUMMARY', None) proc = subprocess.run(args, text=True, @@ -1825,9 +1825,7 @@ def path(name): [line for line in summary_lines if line.startswith('### ')], [test_files[name] for name in tests]) - self.assertIn('- os.environ was modified: ' - 'added REGRTEST_GITHUB_ENV_CHANGED', - summary_lines) + self.assertIn('- os.environ was modified', summary_lines) self.assertCountEqual( re.findall(r'(.*)', summary), [title for title, _, _ in test_cases]) @@ -2957,14 +2955,6 @@ def worker(): class TestUtils(unittest.TestCase): - def test_describe_os_environ(self): - describe = saved_test_environment.describe_os_environ - before = {'A': '1', 'B': '2', 'C': '3'} - after = {'A': '1', 'B': 'secret', 'D': '4'} - self.assertEqual(describe((0, None, before), (0, None, after)), - 'added D; removed C; changed B') - self.assertEqual(describe((0, None, before), (1, None, before)), '') - def test_format_duration(self): self.assertEqual(utils.format_duration(0), '0 ms') From fe082633b9111239bdc45f6da806111c7ede82dc Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Wed, 7 Oct 2026 12:45:15 -0700 Subject: [PATCH 25/25] Simplify filename extraction, use pathlib --- Lib/test/libregrtest/setup.py | 2 +- Lib/test/libregrtest/testresult.py | 27 ++++---- Lib/test/libregrtest/utils.py | 16 +++-- Lib/test/test_regrtest.py | 101 ++++++++++++++--------------- 4 files changed, 73 insertions(+), 73 deletions(-) diff --git a/Lib/test/libregrtest/setup.py b/Lib/test/libregrtest/setup.py index 0c7fad58fd03bf6..8909225d0265bf5 100644 --- a/Lib/test/libregrtest/setup.py +++ b/Lib/test/libregrtest/setup.py @@ -112,8 +112,8 @@ def setup_tests(runtests: RunTests) -> None: set_match_tests(runtests.match_tests) - from .testresult import RegressionTestResult if runtests.use_junit: + from .testresult import RegressionTestResult support.junit_xml_list = [] RegressionTestResult.USE_XML = True else: diff --git a/Lib/test/libregrtest/testresult.py b/Lib/test/libregrtest/testresult.py index 9c453a9ef585162..bdce67226fff3bf 100644 --- a/Lib/test/libregrtest/testresult.py +++ b/Lib/test/libregrtest/testresult.py @@ -4,7 +4,6 @@ import functools import io -import re import sys import time import traceback @@ -12,6 +11,16 @@ from test import support from test.libregrtest.utils import print_github_annotation, sanitize_xml +def test_to_filename(test): + test = getattr(test, 'test_case', test) # subTest() + if dt_test := getattr(test, '_dt_test', None): # doctest.DocTestCase + return dt_test.filename + module = type(test).__module__ + # Fixture errors (setUpClass, setUpModule) are not supported + if module.startswith('unittest.'): + return None + return getattr(sys.modules.get(module), '__file__', None) + class RegressionTestResult(unittest.TextTestResult): USE_XML = False @@ -136,19 +145,9 @@ def printErrorList(self, flavour, errors): super().printErrorList(flavour, [(test, err)]) # Annotate just after the failure report, so that the annotation # links to the report in the job log - print_github_annotation(str(test), err, self._test_file(test), - file=self.stream) - - @staticmethod - def _test_file(test): - # Test id: "module.Class.method", "module.function" (doctest), or - # "setUpClass (module.Class)" (error in a class or module fixture) - name = getattr(test, 'test_case', test).id() # subTest() - if match := re.fullmatch(r'\w+ \((.+)\)', name): - name = match[1] - while name and name not in sys.modules: - name = name.rpartition('.')[0] - return getattr(sys.modules.get(name), '__file__', None) + if filename := test_to_filename(test): + print_github_annotation(str(test), err, filename, + file=self.stream) def get_xml_element(self): if not self.USE_XML: diff --git a/Lib/test/libregrtest/utils.py b/Lib/test/libregrtest/utils.py index 60921d5e018a123..50de4ae9ab6d674 100644 --- a/Lib/test/libregrtest/utils.py +++ b/Lib/test/libregrtest/utils.py @@ -3,6 +3,7 @@ import locale import math import os.path +import pathlib import platform import random import re @@ -141,16 +142,21 @@ def print_warning(msg: str) -> None: orig_unraisablehook: Callable[..., None] | None = None +# Stdlib directory: Lib/ in a source checkout or an installed lib/python3.X/ +STDLIB_DIR = pathlib.Path(os.__file__).parent +# Stdlib directory relative to the repository +REPOSITORY_LIB_DIR = pathlib.Path("Lib") + + def repository_path(filename: str) -> str: """Path of a stdlib file relative to the repository. - Map the stdlib directory (Lib/ in a source checkout or an installed - lib/python3.X/) to Lib/. Return other paths unchanged. + Map STDLIB_DIR to Lib/. Return other paths unchanged. """ - stdlib_dir = os.path.dirname(os.__file__) + os.sep - if not filename.startswith(stdlib_dir): + path = pathlib.Path(filename) + if not path.is_relative_to(STDLIB_DIR): return filename - return "Lib/" + filename.removeprefix(stdlib_dir).replace(os.sep, "/") + return (REPOSITORY_LIB_DIR / path.relative_to(STDLIB_DIR)).as_posix() # Escape the data and the properties of GitHub Actions workflow commands diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index 0de0e76665baf5a..fa569207bee3366 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -1668,22 +1668,6 @@ def run_tests_github(self, *args, exitcode=0): except FileNotFoundError: return output, None - @staticmethod - def parse_github_annotations(output): - # Return the (title, file, line) of each "::error" annotation - def unescape(text): - return (text.replace('%3A', ':').replace('%2C', ',') - .replace('%0A', '\n').replace('%0D', '\r') - .replace('%25', '%')) - - annotations = [] - for match in re.finditer(r'^::error (.*?)::', output, re.MULTILINE): - props = dict(prop.split('=', 1) for prop in match[1].split(',')) - props = {key: unescape(value) for key, value in props.items()} - annotations.append((props['title'], props['file'], - props.get('line'))) - return annotations - def test_github_annotations(self): # Every test failure is annotated once in the GitHub Actions job log, # located in the test file, at the failing line of the test file if @@ -1715,17 +1699,17 @@ def test_subtest(self): class SetUpClassTests(unittest.TestCase): @classmethod def setUpClass(cls): - raise ValueError("setUpClass") # setUpClass + raise ValueError("setUpClass") def test_never(self): pass """) - setup_module, setup_module_lines = self.create_marked_test( + setup_module, _ = self.create_marked_test( 'github_setup_module', """ import unittest def setUpModule(): - raise ValueError("setUpModule") # setUpModule + raise ValueError("setUpModule") class Tests(unittest.TestCase): def test_never(self): @@ -1746,24 +1730,23 @@ def test_crash(self): os._exit(1) """) - def path(name): - return os.path.join(self.tmptestdir, f'{name}.py') - - # Test cases: (title, file, line) + # Annotated test cases: (title, file, line) test_cases = [ (f'test_error ({cases}.Tests.test_error)', - path(cases), lines['error']), + f'{cases}.py', lines['error']), (f'test_fail ({cases}.Tests.test_fail)', - path(cases), lines['fail']), + f'{cases}.py', lines['fail']), (f'test_subtest ({cases}.Tests.test_subtest) (x=1.5)', - path(cases), lines['subtest']), + f'{cases}.py', lines['subtest']), # Doctest examples are subtests: "[0]" is the example index (f'doctest_fail ({cases}) [0]', - path(cases), lines['doctest']), - (f'setUpClass ({cases}.SetUpClassTests)', - path(cases), lines['setUpClass']), - (f'setUpModule ({setup_module})', - path(setup_module), setup_module_lines['setUpModule']), + f'{cases}.py', lines['doctest']), + ] + # Test cases only listed in the job summary: fixture errors + # (setUpClass, setUpModule) are not annotated + failed_titles = [title for title, _, _ in test_cases] + [ + f'setUpClass ({cases}.SetUpClassTests)', + f'setUpModule ({setup_module})', ] # Test files: heading in the job summary test_files = { @@ -1794,33 +1777,45 @@ def path(name): output, summary = self.run_tests_github( *args, *tests, exitcode=EXITCODE_BAD_TEST) - annotations = list(test_cases) + expected = list(test_cases) if '--fast-ci' in args: - annotations.append((env_changed, path(env_changed), None)) + expected.append((env_changed, f'{env_changed}.py', None)) if crash in tests: - annotations.append((crash, path(crash), None)) - self.assertCountEqual(self.parse_github_annotations(output), - annotations, output) - # Each test case is annotated right after its failure report - for title, _, _ in annotations[:len(test_cases)]: - title = re.escape(title) - self.assertRegex( - output, - rf'(?m)^(?:ERROR|FAIL): {title}\n' - rf'(?:(?!={{70}}$).*\n)*?::error .*title={title}::') - # The env changed annotation comes right after the warnings - if '--fast-ci' in args: - self.assertRegex( - output, - rf'(?m)^Warning -- os\.environ was modified by ' - rf'{env_changed}\n(?:Warning -- .*\n)*' - rf'::error .*title={env_changed}::') + expected.append((crash, f'{crash}.py', None)) + + # "::error file=...,line=...,title=...::message" + annotations = [] + output_lines = output.splitlines() + for i, line in enumerate(output_lines): + if not line.startswith('::error '): + continue + _, props, message = line.split('::', 2) + props = props.removeprefix('error ').split(',') + props = dict(prop.split('=', 1) for prop in props) + title = props['title'] + annotations.append((title, os.path.basename(props['file']), + props.get('line'))) + before = output_lines[:i] + if title == env_changed: + # Right after the env changed warnings + self.assertStartsWith(before[-1], 'Warning -- ') + elif title != crash: + # Right after the failure report of the test, which + # ends with the exception and an empty line + report = next( + text for text in reversed(before) + if text.startswith(('ERROR: ', 'FAIL: '))) + self.assertEqual(report.split(': ', 1)[1], title) + message = message.replace('%0A', '\n').splitlines() + self.assertEqual(before[-len(message) - 1:], + message + ['']) + self.assertCountEqual(annotations, expected, output) # The job summary lists the failed tests in completion order summary_lines = summary.splitlines() self.assertEqual(summary_lines[0], f'## FAILURE: {len(tests)} test files and ' - f'{len(test_cases)} test cases failed') + f'{len(failed_titles)} test cases failed') self.assertCountEqual( [line for line in summary_lines if line.startswith('### ')], @@ -1828,9 +1823,9 @@ def path(name): self.assertIn('- os.environ was modified', summary_lines) self.assertCountEqual( re.findall(r'(.*)', summary), - [title for title, _, _ in test_cases]) + failed_titles) self.assertEqual(summary.count('```pytb\n'), - len(test_cases)) + len(failed_titles)) def test_github_summary_success(self): # No annotation and no job summary when all tests pass