drop groups, error only on final failure

Author: cmaloney

Lib/test/libregrtest/main.py

@@ -16,7 +16,7 @@
 from .logger import Logger
 from .pgo import setup_pgo_tests
 from .result import TestResult
-from .results import TestResults, EXITCODE_INTERRUPTED
+from .results import TestResults, EXITCODE_INTERRUPTED, annotate_github
 from .runtests import RunTests, HuntRefleak
 from .setup import setup_process, setup_test_dir
 from .single import run_single_test, PROGRESS_MIN_TIME
@@ -491,6 +491,10 @@ def write_github_summary(self) -> None:
             return
         # Tests which failed in the last run (the re-run, if any)
         failed = self.results.rerun_results
+        if not self.results.rerun:
+            # Failed tests were not re-run (ex: --python): annotate them now
+            for result in failed:
+                annotate_github(result)
         cases = [(result.errors or []) + (result.failures or [])
                  for result in failed]
         ncase = sum(map(len, cases))

Lib/test/libregrtest/results.py

@@ -20,14 +20,14 @@
 EXITCODE_INTERRUPTED = 130   # 128 + signal.SIGINT=2
 
 
-def annotate_github(result: TestResult, level: str) -> None:
+def annotate_github(result: TestResult) -> None:
     cases = (result.errors or []) + (result.failures or [])
     for name, traceback in cases:
-        github_annotation(level, f"{result.test_name}: {name}",
+        github_annotation("error", f"{result.test_name}: {name}",
                           decolor(traceback))
     if not cases:
         message = "\n".join([str(result), *(result.env_changed_reasons or ())])
-        github_annotation(level, result.test_name, decolor(message))
+        github_annotation("error", result.test_name, decolor(message))
 
 
 class TestResults:
@@ -139,10 +139,10 @@ def accumulate_result(self, result: TestResult, runtests: RunTests) -> None:
         if result.state == State.WORKER_BUG:
             self.worker_bug = True
 
-        if (os.environ.get("GITHUB_STEP_SUMMARY")
+        if (os.environ.get("GITHUB_STEP_SUMMARY") and rerun
                 and result.is_failed(fail_env_changed)):
-            # A failure is only an error if it fails again when re-run
-            annotate_github(result, "error" if rerun else "warning")
+            # Only annotate tests which fail again when re-run
+            annotate_github(result)
 
         if result.has_meaningful_duration() and not rerun:
             if result.duration is None:

Lib/test/libregrtest/run_workers.py

@@ -10,7 +10,6 @@
 import threading
 import time
 import traceback
-from _colorize import decolor  # type: ignore[import-not-found]
 from typing import Any, Literal, TextIO
 
 from test import support
@@ -520,11 +519,6 @@ def __init__(self, num_workers: int, runtests: RunTests,
         else:
             self.worker_timeout = None
         self.workers: list[WorkerThread] = []
-        # On GitHub Actions, fold the first run's log into groups: ranges of
-        # passing tests, and one group per failed test. Re-runs aren't folded.
-        self.github_groups = (not runtests.rerun
-                              and bool(os.environ.get("GITHUB_STEP_SUMMARY")))
-        self.in_github_group = False
 
         jobs = self.runtests.get_jobs()
         if jobs is not None:
@@ -617,16 +611,7 @@ def _process_result(self, item: QueueOutput) -> TestResult:
         self.test_index += 1
         mp_result = item[1]
         result = mp_result.result
-        failed = result.is_failed(self.runtests.fail_env_changed)
-        if self.github_groups and failed:
-            # Keep the failure annotation out of collapsed groups
-            self.github_group(None)
         self.results.accumulate_result(result, self.runtests)
-        if self.github_groups:
-            if failed:
-                self.github_group(decolor(str(result)))
-            elif not self.in_github_group:
-                self.github_group(f"Tests from #{self.test_index}")
         self.display_result(mp_result)
 
         # Display worker stdout
@@ -639,19 +624,9 @@ def _process_result(self, item: QueueOutput) -> TestResult:
             stdout = mp_result.worker_stdout
             if stdout:
                 print(stdout, flush=True)
-        if self.github_groups and failed:
-            self.github_group(None)
 
         return result
 
-    def github_group(self, title: str | None) -> None:
-        """End the current GitHub Actions log group, start a new one if title."""
-        if self.in_github_group:
-            print("::endgroup::", flush=True)
-        self.in_github_group = title is not None
-        if title is not None:
-            print(f"::group::{title}", flush=True)
-
     def get_mem_usage(self):
         usage = 0
         main_mem = get_process_memory_usage(os.getpid())
@@ -684,7 +659,6 @@ def run(self) -> None:
             print()
             self.results.interrupted = True
         finally:
-            self.github_group(None)
             if self.timeout is not None:
                 faulthandler.cancel_dump_traceback_later()
 

Lib/test/libregrtest/utils.py

@@ -154,12 +154,13 @@ def escape_property(text: str) -> str:
 
     props = f"title={escape_property(title)}"
     frames = re.findall(r'^  File "(.+)", line (\d+)', message, re.MULTILINE)
-    # Source checkout which contains Lib/
-    srcdir = os.path.dirname(os.path.dirname(os.__file__)) + os.sep
-    if frames and frames[-1][0].startswith(srcdir):
+    # Map the stdlib directory (Lib/ in a source checkout or an installed
+    # lib/python3.X/) to Lib/ in the repository
+    stdlib_dir = os.path.dirname(os.__file__) + os.sep
+    if frames and frames[-1][0].startswith(stdlib_dir):
         filename, line = frames[-1]
-        filename = filename.removeprefix(srcdir).replace(os.sep, "/")
-        props = f"file={escape_property(filename)},line={line},{props}"
+        filename = filename.removeprefix(stdlib_dir).replace(os.sep, "/")
+        props = f"file=Lib/{escape_property(filename)},line={line},{props}"
     print(f"::{level} {props}::{escape(message)}", flush=True)