From 28074f1ed2c08435116dedaf07eac09531b2dba8 Mon Sep 17 00:00:00 2001 From: Esteban Zimanyi Date: Tue, 8 Sep 2026 01:31:16 +0200 Subject: [PATCH] Read the run counts node:test writes `node --test` is the sixth dialect, and it differs from the other five in counting the tests that never ran in TWO places. A run of two passing, one skipped and one deferred prints `tests 4`, `pass 2`, `skipped 1`, `todo 1`: the two not-run counters are disjoint from `pass` and from each other, so a reader watching `skipped` alone certifies a suite whose every remaining test is deferred. Node prints a deferred test with a tick, which is the "reads as a pass" shape this tool exists to refuse, so both counters sum into the skip count. It is read before pytest for the reason Catch2 is: a log carries whatever else the job printed, and a phrase shaped like pytest's summary sitting beside node's counters would otherwise decide the total. A test holds that order. Every string in the three new tests is copied from a real `node --test` run. Against one, the tool reads `total=4 skipped=2` and exits 1 naming two skipped tests; against a clean run it reads `total=2 skipped=0` and exits 0. 21 tests pass. --- tests/test_test_outcome.py | 30 ++++++++++++++++++++++++++ tools/check-test-outcome.py | 42 +++++++++++++++++++++++++++++++++++++ 2 files changed, 72 insertions(+) diff --git a/tests/test_test_outcome.py b/tests/test_test_outcome.py index 9c65473..eb53bd3 100644 --- a/tests/test_test_outcome.py +++ b/tests/test_test_outcome.py @@ -141,6 +141,36 @@ def test_a_vstest_run_over_an_empty_assembly_names_no_dialect(self): self.assertIsNone(outcome.read_summaries(log)[2]) +class NodeTestTests(unittest.TestCase): + """`node --test` counts the not-run tests in TWO places. + + Every string below is copied from a real `node --test` run, not composed + from the documentation.""" + + CLEAN = ("ℹ tests 2\nℹ suites 0\nℹ pass 2\nℹ fail 0\n" + "ℹ cancelled 0\nℹ skipped 0\nℹ todo 0\n" + "ℹ duration_ms 92.5\n") + + def test_a_clean_run_reads_its_total(self): + total, skipped, dialect, _ = outcome.read_summaries(self.CLEAN) + self.assertEqual(("node:test", 2, 0), (dialect, total, skipped)) + + def test_a_deferred_test_counts_as_skipped(self): + # 2 passing, 1 skipped and 1 deferred: node prints the deferred one with + # a TICK and counts it in neither `pass` nor `skipped`, so a reader + # watching `skipped` alone would report 1 over 2 tests that never ran. + log = ("ℹ tests 4\nℹ suites 1\nℹ pass 2\nℹ fail 0\n" + "ℹ cancelled 0\nℹ skipped 1\nℹ todo 1\n") + total, skipped, dialect, _ = outcome.read_summaries(log) + self.assertEqual(("node:test", 4, 2), (dialect, total, skipped)) + + def test_node_is_read_before_pytest(self): + # A node log that also carries a pytest-shaped phrase must still read as + # node:test, or the total becomes whatever that phrase names. + log = "collected: 9 passed in 0.1s\n" + self.CLEAN + self.assertEqual("node:test", outcome.read_summaries(log)[2]) + + class NoFlagPermitsASkipTests(unittest.TestCase): """No option tolerates a skip, and none may be added back. diff --git a/tools/check-test-outcome.py b/tools/check-test-outcome.py index ab31f59..8366e99 100755 --- a/tools/check-test-outcome.py +++ b/tools/check-test-outcome.py @@ -78,6 +78,20 @@ r"(?:Passed|Failed)!\s+-\s+Failed:\s*(\d+),\s*Passed:\s*(\d+)," r"\s*Skipped:\s*(\d+),\s*Total:\s*(\d+)") +# `tests 4` / `skipped 1` / `todo 1` — the counters `node --test` writes at the +# end of a run under its default reporter, each behind an information glyph. +# Mirrors the VSTEST block above: one dialect, its own constants. +# +# `tests` is the whole total, and the not-run tests are the sum of TWO +# counters. A run of 2 passing, 1 skipped and 1 deferred prints `tests 4`, +# `pass 2`, `skipped 1`, `todo 1` — the two are disjoint from `pass` and from +# each other, so a reader watching `skipped` alone certifies a suite whose +# every test is deferred. Node prints a deferred test with a TICK, which is +# exactly the "reads as a pass" shape this tool exists to refuse. +NODETEST_TESTS = re.compile(r"^\s*\S?\s*tests\s+(\d+)\s*$") +NODETEST_SKIPPED = re.compile(r"^\s*\S?\s*skipped\s+(\d+)\s*$") +NODETEST_DEFERRED = re.compile(r"^\s*\S?\s*todo\s+(\d+)\s*$") + # `All tests passed (2695 assertions in 102 test cases)` — what the Catch2 # console reporter writes when nothing failed, carrying a leading # `3 skipped tests, ` when any were. The test-case count it prints EXCLUDES the @@ -135,6 +149,34 @@ def read_summaries(text: str): if lines: return total, skipped, "vstest", lines + # node:test is read BEFORE pytest for the same reason Catch2 is: node's + # `pass 2` line does not satisfy the pytest pattern, but its `duration_ms` + # and per-test lines sit in the same log as anything else the job printed, + # and reading the counters as a block keeps a partial match from standing in + # for the summary. All three counters come from ONE run, so a log carrying + # `tests` without `skipped` is a truncated log rather than a clean one. + deferred = 0 + for raw in text.splitlines(): + line = raw.rstrip() + m = NODETEST_TESTS.search(line) + if m: + total += int(m.group(1)) + lines.append(line.strip()) + continue + m = NODETEST_SKIPPED.search(line) + if m: + skipped += int(m.group(1)) + lines.append(line.strip()) + continue + m = NODETEST_DEFERRED.search(line) + if m: + deferred += int(m.group(1)) + lines.append(line.strip()) + if lines: + # A deferred test asserts nothing and node prints it with a tick, so it + # counts as skipped rather than as a category of its own. + return total, skipped + deferred, "node:test", lines + # Catch2 is read BEFORE pytest, and the order is load-bearing: the failure # table's `101 passed` satisfies the pytest pattern, so a failing Catch2 run # read pytest-first reports the passing count as the total and misses both