Add groups more of the time

Author: cmaloney

Lib/test/libregrtest/main.py

@@ -30,15 +30,6 @@
     get_process_memory_usage, EXIT_TIMEOUT)
 
 
-def github_error(title: str, message: str) -> None:
-    # Emit a GitHub Actions error annotation (workflow command)
-    def escape(text: str) -> str:
-        return (text.replace("%", "%25").replace("\r", "%0D")
-                .replace("\n", "%0A"))
-    title = escape(title).replace(":", "%3A").replace(",", "%2C")
-    print(f"::error title={title}::{escape(message)}", flush=True)
-
-
 class Regrtest:
     """Execute a test suite.
 
@@ -513,14 +504,10 @@ def write(text: str) -> None:
                   f"{count(ncase, 'test case')} failed")
             for result, result_cases in zip(failed, cases):
                 write(f"### {decolor(str(result))}")
-                if not result_cases:
-                    github_error(result.test_name, decolor(str(result)))
                 if result.env_changed_reasons:
                     write("\n".join(f"- {reason}"
                                     for reason in result.env_changed_reasons))
                 for name, traceback in result_cases:
-                    github_error(f"{result.test_name}: {name}",
-                                 decolor(traceback))
                     # Expand short tracebacks when there are only a few
                     is_open = ncase <= 5 and traceback.count("\n") < 30
                     write(f"<details{' open' if is_open else ''}>"

Lib/test/libregrtest/results.py

@@ -1,13 +1,14 @@
+import os
 import sys
 import trace
-from _colorize import get_colors  # type: ignore[import-not-found]
+from _colorize import decolor, get_colors  # type: ignore[import-not-found]
 lazy from xml.etree.ElementTree import Element
 
 from .runtests import RunTests
 from .result import State, TestResult, TestStats, Location
 from .utils import (
     StrPath, TestName, TestTuple, TestList, FilterDict,
-    printlist, count, format_duration)
+    printlist, count, format_duration, github_annotation)
 
 
 # Python uses exit code 1 when an exception is not caught
@@ -19,6 +20,16 @@
 EXITCODE_INTERRUPTED = 130   # 128 + signal.SIGINT=2
 
 
+def annotate_github(result: TestResult, level: str) -> None:
+    cases = (result.errors or []) + (result.failures or [])
+    for name, traceback in cases:
+        github_annotation(level, f"{result.test_name}: {name}",
+                          decolor(traceback))
+    if not cases:
+        message = "\n".join([str(result), *(result.env_changed_reasons or ())])
+        github_annotation(level, result.test_name, decolor(message))
+
+
 class TestResults:
     def __init__(self) -> None:
         self.bad: TestList = []
@@ -128,6 +139,11 @@ def accumulate_result(self, result: TestResult, runtests: RunTests) -> None:
         if result.state == State.WORKER_BUG:
             self.worker_bug = True
 
+        if (os.environ.get("GITHUB_STEP_SUMMARY")
+                and result.is_failed(fail_env_changed)):
+            # A failure is only an error if it fails again when re-run
+            annotate_github(result, "error" if rerun else "warning")
+
         if result.has_meaningful_duration() and not rerun:
             if result.duration is None:
                 raise ValueError("result.duration is None")

Lib/test/libregrtest/run_workers.py

@@ -10,6 +10,7 @@
 import threading
 import time
 import traceback
+from _colorize import decolor  # type: ignore[import-not-found]
 from typing import Any, Literal, TextIO
 
 from test import support
@@ -519,6 +520,11 @@ def __init__(self, num_workers: int, runtests: RunTests,
         else:
             self.worker_timeout = None
         self.workers: list[WorkerThread] = []
+        # On GitHub Actions, fold the first run's log into groups: ranges of
+        # passing tests, and one group per failed test. Re-runs aren't folded.
+        self.github_groups = (not runtests.rerun
+                              and bool(os.environ.get("GITHUB_STEP_SUMMARY")))
+        self.in_github_group = False
 
         jobs = self.runtests.get_jobs()
         if jobs is not None:
@@ -611,7 +617,16 @@ def _process_result(self, item: QueueOutput) -> TestResult:
         self.test_index += 1
         mp_result = item[1]
         result = mp_result.result
+        failed = result.is_failed(self.runtests.fail_env_changed)
+        if self.github_groups and failed:
+            # Keep the failure annotation out of collapsed groups
+            self.github_group(None)
         self.results.accumulate_result(result, self.runtests)
+        if self.github_groups:
+            if failed:
+                self.github_group(decolor(str(result)))
+            elif not self.in_github_group:
+                self.github_group(f"Tests from #{self.test_index}")
         self.display_result(mp_result)
 
         # Display worker stdout
@@ -624,9 +639,19 @@ def _process_result(self, item: QueueOutput) -> TestResult:
             stdout = mp_result.worker_stdout
             if stdout:
                 print(stdout, flush=True)
+        if self.github_groups and failed:
+            self.github_group(None)
 
         return result
 
+    def github_group(self, title: str | None) -> None:
+        """End the current GitHub Actions log group, start a new one if title."""
+        if self.in_github_group:
+            print("::endgroup::", flush=True)
+        self.in_github_group = title is not None
+        if title is not None:
+            print(f"::group::{title}", flush=True)
+
     def get_mem_usage(self):
         usage = 0
         main_mem = get_process_memory_usage(os.getpid())
@@ -659,6 +684,7 @@ def run(self) -> None:
             print()
             self.results.interrupted = True
         finally:
+            self.github_group(None)
             if self.timeout is not None:
                 faulthandler.cancel_dump_traceback_later()
 

Lib/test/libregrtest/utils.py

@@ -140,6 +140,29 @@ def print_warning(msg: str) -> None:
 orig_unraisablehook: Callable[..., None] | None = None
 
 
+def github_annotation(level: str, title: str, message: str) -> None:
+    """Emit a GitHub Actions annotation (workflow command).
+
+    Point it at the last frame of the traceback in message, if any.
+    """
+    def escape(text: str) -> str:
+        return (text.replace("%", "%25").replace("\r", "%0D")
+                .replace("\n", "%0A"))
+
+    def escape_property(text: str) -> str:
+        return escape(text).replace(":", "%3A").replace(",", "%2C")
+
+    props = f"title={escape_property(title)}"
+    frames = re.findall(r'^  File "(.+)", line (\d+)', message, re.MULTILINE)
+    # Source checkout which contains Lib/
+    srcdir = os.path.dirname(os.path.dirname(os.__file__)) + os.sep
+    if frames and frames[-1][0].startswith(srcdir):
+        filename, line = frames[-1]
+        filename = filename.removeprefix(srcdir).replace(os.sep, "/")
+        props = f"file={escape_property(filename)},line={line},{props}"
+    print(f"::{level} {props}::{escape(message)}", flush=True)
+
+
 def regrtest_unraisable_hook(unraisable) -> None:
     global orig_unraisablehook
     support.set_environment_altered(