gh-158895: Teach libregrtest to output github job summaries

Lib/test/libregrtest/main.py

@@ -1,3 +1,4 @@
+import html
 import os
 import random
 import re
@@ -6,7 +7,7 @@
 import sysconfig
 import time
 import trace
-from _colorize import get_colors  # type: ignore[import-not-found]
+from _colorize import decolor, get_colors  # type: ignore[import-not-found]
 from typing import NoReturn
 
 from test.support import os_helper, MS_WINDOWS, flush_std_streams
@@ -395,6 +396,7 @@ def run_test(
             result = run_single_test(test_name, runtests)
 
         self.results.accumulate_result(result, runtests)
+        result.print_github_annotation(runtests)
 
         return result
 
@@ -480,6 +482,42 @@ def finalize_tests(self, coverage: trace.CoverageResults | None) -> None:
         if self.junit_filename:
             self.results.write_junit(self.junit_filename)
 
+        self.write_github_summary()
+
+    def write_github_summary(self) -> None:
+        filename = os.environ.get("GITHUB_STEP_SUMMARY")
+        # Tests which failed in the last run (the re-run, if any)
+        failed = self.results.rerun_results
+        # Only write a summary on failure: a CI run has many test jobs
+        exitcode = self.results.get_exitcode(self.fail_env_changed,
+                                             self.fail_rerun)
+        if not filename or not failed or not exitcode:
+            return
+        cases = [(result.errors or []) + (result.failures or [])
+                 for result in failed]
+        ncases = sum(map(len, cases))
+        with open(filename, "a", encoding="utf-8") as fp:
+            def write(text: str) -> None:
+                # Separate Markdown blocks with an empty line
+                print(text, end="\n\n", file=fp)
+
+            write(f"## {decolor(self.get_state())}: "
+                  f"{count(len(failed), 'test file')} and "
+                  f"{count(ncases, 'test case')} failed")
+            for result, result_cases in zip(failed, cases):
+                write(f"### {decolor(str(result))}")
+                if result.env_changed_reasons:
+                    write("\n".join(f"- {reason}"
+                                    for reason in result.env_changed_reasons))
+                for name, traceback in result_cases:
+                    # Expand the tracebacks if they fit on a screen: up to 5
+                    # test cases with a traceback of less than 30 lines
+                    is_open = ncases <= 5 and traceback.count("\n") < 30
+                    write(f"<details{' open' if is_open else ''}>"
+                          f"<summary>{html.escape(name)}</summary>")
+                    write(f"```pytb\n{decolor(traceback).rstrip()}\n```")
+                    write("</details>")
+
     def display_summary(self) -> None:
         if self.first_runtests is None:
             raise ValueError(
@@ -579,10 +617,16 @@ def _run_tests(self, selected: TestTuple, tests: TestList | None) -> int:
         if use_load_tracker:
             self.logger.start_load_tracker()
         try:
-            if self.num_workers:
-                self._run_tests_mp(runtests, self.num_workers)
-            else:
-                self.run_tests_sequentially(runtests)
+            with os_helper.EnvironmentVarGuard() as env:
+                # In GitHub Actions, only annotate failures of the last run:
+                # not the first run if failed tests will be re-run (with
+                # --python, they are not: see rerun_failed_tests())
+                if self.want_rerun and not self.python_cmd:
+                    env.unset("GITHUB_STEP_SUMMARY")
+                if self.num_workers:
+                    self._run_tests_mp(runtests, self.num_workers)
+                else:
+                    self.run_tests_sequentially(runtests)
 
             coverage = self.results.get_coverage_results()
             self.display_result(runtests)

Lib/test/libregrtest/result.py

@@ -1,11 +1,15 @@
 import dataclasses
 import json
+import os
 from _colorize import get_colors  # type: ignore[import-not-found]
 from typing import Any
 
+from .findtests import findtestdir
+from .runtests import RunTests
 from .utils import (
     StrJSON, TestName, FilterTuple,
-    format_duration, normalize_test_name, print_warning)
+    format_duration, normalize_test_name, print_warning,
+    print_github_annotation)
 
 
 @dataclasses.dataclass(slots=True)
@@ -178,6 +182,25 @@ def __str__(self) -> str:
     def has_meaningful_duration(self):
         return State.has_meaningful_duration(self.state)
 
+    def print_github_annotation(self, runtests: RunTests) -> None:
+        """Annotate a failed test without test case failures.
+
+        For example: crash, timeout, env changed. Locate the annotation in the
+        test file. Test case failures are annotated by the test runner, where
+        they are reported (see RegressionTestResult.printErrorList()).
+        """
+        if (self.is_failed(runtests.fail_env_changed)
+                and not self.errors and not self.failures):
+            message = "\n".join([str(self), *(self.env_changed_reasons or ())])
+            # Test file: "test_x" is test_x.py or test_x/__init__.py,
+            # "test.test_x.test_y" is test_x/test_y.py
+            name = self.test_name.removeprefix("test.")
+            path = os.path.join(findtestdir(runtests.test_dir), *name.split("."))
+            filename = path + ".py"
+            if not os.path.exists(filename):
+                filename = os.path.join(path, "__init__.py")
+            print_github_annotation(self.test_name, message, filename)
+
     def set_env_changed(self, *reasons):
         if self.state is None or self.state == State.PASSED:
             self.state = State.ENV_CHANGED

Lib/test/libregrtest/run_workers.py

@@ -624,6 +624,8 @@ def _process_result(self, item: QueueOutput) -> TestResult:
             stdout = mp_result.worker_stdout
             if stdout:
                 print(stdout, flush=True)
+        # Annotate after the output: env changed warnings, crash traceback
+        result.print_github_annotation(self.runtests)
 
         return result
 

Lib/test/libregrtest/setup.py

@@ -113,8 +113,8 @@ def setup_tests(runtests: RunTests) -> None:
     set_match_tests(runtests.match_tests)
 
     if runtests.use_junit:
-        support.junit_xml_list = []
         from .testresult import RegressionTestResult
+        support.junit_xml_list = []
         RegressionTestResult.USE_XML = True
     else:
         support.junit_xml_list = None

Lib/test/libregrtest/testresult.py

@@ -9,7 +9,17 @@
 import traceback
 import unittest
 from test import support
-from test.libregrtest.utils import sanitize_xml
+from test.libregrtest.utils import print_github_annotation, sanitize_xml
+
+def test_to_filename(test):
+    test = getattr(test, 'test_case', test)  # subTest()
+    if dt_test := getattr(test, '_dt_test', None):  # doctest.DocTestCase
+        return dt_test.filename
+    module = type(test).__module__
+    # Fixture errors (setUpClass, setUpModule) are not supported
+    if module.startswith('unittest.'):
+        return None
+    return getattr(sys.modules.get(module), '__file__', None)
 
 class RegressionTestResult(unittest.TextTestResult):
     USE_XML = False
@@ -130,6 +140,15 @@ def addUnexpectedSuccess(self, test):
         self._add_result(test, outcome='UNEXPECTED_SUCCESS')
         super().addUnexpectedSuccess(test)
 
+    def printErrorList(self, flavour, errors):
+        for test, err in errors:
+            super().printErrorList(flavour, [(test, err)])
+            # Annotate just after the failure report, so that the annotation
+            # links to the report in the job log
+            if filename := test_to_filename(test):
+                print_github_annotation(str(test), err, filename,
+                                        file=self.stream)
+
     def get_xml_element(self):
         if not self.USE_XML:
             raise ValueError("USE_XML is false")

Lib/test/libregrtest/utils.py

@@ -3,6 +3,7 @@
 import locale
 import math
 import os.path
+import pathlib
 import platform
 import random
 import re
@@ -13,6 +14,7 @@
 import tempfile
 import textwrap
 import types
+from _colorize import decolor  # type: ignore[import-not-found]
 from collections.abc import Callable
 _winapi: types.ModuleType | None
 try:
@@ -140,6 +142,60 @@ def print_warning(msg: str) -> None:
 orig_unraisablehook: Callable[..., None] | None = None
 
 
+# Stdlib directory: Lib/ in a source checkout or an installed lib/python3.X/
+STDLIB_DIR = pathlib.Path(os.__file__).parent
+# Stdlib directory relative to the repository
+REPOSITORY_LIB_DIR = pathlib.Path("Lib")
+
+
+def repository_path(filename: str) -> str:
+    """Path of a stdlib file relative to the repository.
+
+    Map STDLIB_DIR to Lib/. Return other paths unchanged.
+    """
+    path = pathlib.Path(filename)
+    if not path.is_relative_to(STDLIB_DIR):
+        return filename
+    return (REPOSITORY_LIB_DIR / path.relative_to(STDLIB_DIR)).as_posix()
+
+
+# Escape the data and the properties of GitHub Actions workflow commands
+GITHUB_ESCAPE_DATA = str.maketrans({"%": "%25", "\r": "%0D", "\n": "%0A"})
+GITHUB_ESCAPE_PROPERTY = GITHUB_ESCAPE_DATA | str.maketrans({":": "%3A",
+                                                             ",": "%2C"})
+
+
+def print_github_annotation(title: str, message: str, filename: str | None,
+                            file=None) -> None:
+    """Print a GitHub Actions error annotation, if run in GitHub Actions.
+
+    message is a traceback or a failure description. Only keep the exception
+    which ends the traceback: the job log has the full traceback. Locate the
+    annotation in filename, the test file, at the last frame of the traceback
+    in this file, if any.
+    """
+    if not os.environ.get("GITHUB_STEP_SUMMARY"):
+        return
+    if not filename:
+        raise ValueError(f"missing test file of annotation {title!r}")
+    message = decolor(message)
+    props = {"file": repository_path(filename)}
+    # Line of the last traceback frame ('  File "filename", line 123') or
+    # failed doctest example (same, but not indented) in filename
+    if lines := re.findall(rf'^ *File "{re.escape(filename)}", line (\d+)',
+                           message, re.MULTILINE):
+        props["line"] = lines[-1]
+    props["title"] = title
+    # Only keep the exception: strip the traceback frames, each one a
+    # '  File' line followed by its indented source lines
+    message = re.split(r'^  File .*\n(?:    .*\n)*', message,
+                       flags=re.MULTILINE)[-1].strip()
+    props_text = ",".join(f"{key}={value.translate(GITHUB_ESCAPE_PROPERTY)}"
+                          for key, value in props.items())
+    print(f"::error {props_text}::{message.translate(GITHUB_ESCAPE_DATA)}",
+          file=file, flush=True)
+
+
 def regrtest_unraisable_hook(unraisable) -> None:
     global orig_unraisablehook
     support.set_environment_altered(

Lib/test/test_bool.py

@@ -4,6 +4,7 @@
 from test.support import os_helper
 
 import os
+import sys
 
 class BoolTest(unittest.TestCase):
 
@@ -25,6 +26,11 @@
         self.assertIs(eval(repr(True)), True)
 
     def test_str(self):
+        if sys.platform == "linux":
+            # XXX: Intentional failure to test GitHub Actions annotations
+            self.assertEqual(
+                str(True),
+                'Yes')
         self.assertEqual(str(False), 'False')
         self.assertEqual(str(True), 'True')
 

Lib/test/test_json/test_decode.py

@@ -1,4 +1,5 @@
 import decimal
+import sys
 import unittest.mock
 from io import StringIO
 from collections import OrderedDict
@@ -9,6 +10,9 @@
 class TestDecode:
     def test_decimal(self):
         rval = self.loads('1.1', parse_float=decimal.Decimal)
+        if sys.platform == "linux":
+            # XXX: Intentional failure to test GitHub Actions annotations
+            self.loads('{"bad": }')
         self.assertIsInstance(rval, decimal.Decimal)
         self.assertEqual(rval, decimal.Decimal('1.1'))
 

Lib/test/test_math.py

@@ -513,6 +513,11 @@
 
     def testFloor(self):
         self.assertRaises(TypeError, math.floor)
+        if sys.platform == "linux":
+            # XXX: Intentional failure to test GitHub Actions annotations
+            for x in (0.5, 1.5):
+                with self.subTest(x=x):
+                    self.assertEqual(math.floor(x), x)
         self.assertEqual(int, type(math.floor(0.5)))
         self.assertEqual(math.floor(0.5), 0)
         self.assertEqual(math.floor(1.0), 1)

Lib/test/test_os/test_os.py

@@ -4080,7 +4080,8 @@
         self.assertFalse(os.get_inheritable(fd))
         with open(fd, "wb", closefd=False) as f:
             f.write(b'memfd_create')
-            self.assertEqual(f.tell(), 12)
+            # XXX: Intentional failure to test GitHub Actions job summaries
+            self.assertEqual(f.tell(), 13)
 
         fd2 = os.memfd_create("Hi")
         self.addCleanup(os.close, fd2)

Lib/test/test_profiling/test_sampling_profiler/test_live_collector_ui.py

@@ -11,7 +11,8 @@
 import time
 import unittest
 from unittest import mock
-from test.support import requires, requires_remote_subprocess_debugging
+from test.support import (
+    os_helper, requires, requires_remote_subprocess_debugging)
 from test.support.import_helper import import_module
 
 # Only run these tests if curses is available
@@ -856,6 +857,9 @@ def mock_init_curses_side_effect(self, n_times, mock_self, stdscr):
 
     def test_run_failed_module_live(self):
         """Test that running a existing module that fails exits with clean error."""
+        # Don't write to the GitHub Actions job summary of the test suite
+        env = self.enterContext(os_helper.EnvironmentVarGuard())
+        env.unset('GITHUB_STEP_SUMMARY')
 
         args = [
             "profiling.sampling.cli", "run", "--live", "-m", "test",