diff --git a/Lib/test/test_peg_generator/test_pegen.py b/Lib/test/test_peg_generator/test_pegen.py index 64c875367cef275..67421b9fa566931 100644 --- a/Lib/test/test_peg_generator/test_pegen.py +++ b/Lib/test/test_peg_generator/test_pegen.py @@ -12,7 +12,9 @@ with test_tools.imports_under_tool("peg_generator"): from pegen.grammar_parser import GeneratedParser as GrammarParser from pegen.testutil import parse_string, generate_parser, make_parser - from pegen.grammar import GrammarVisitor, GrammarError, Grammar, RuleKind + from pegen.grammar import ( + GrammarVisitor, GrammarError, Grammar, NameLeaf, RuleKind, + ) from pegen.grammar_visualizer import ASTGrammarPrinter from pegen.parser import Parser from pegen.parser_generator import compute_nullables, compute_left_recursives @@ -751,6 +753,56 @@ def test_opt_sequence(self) -> None: # of a line in the generated source. See bpo-41044 make_parser(grammar) + def test_left_recursion_leader_order(self) -> None: + grammar = parse_string(""" + start: zeta NEWLINE + zeta: alpha '+' | NUMBER + alpha: zeta '-' | NUMBER + """, GrammarParser) + PythonParserGenerator(grammar, io.StringIO()) + self.assertTrue(grammar.rules["alpha"].leader) + self.assertFalse(grammar.rules["zeta"].leader) + + def test_large_left_recursive_grammar(self) -> None: + size = 12 + lines = ["start: r0 NEWLINE ENDMARKER"] + for i in range(size): + children = list(range(i + 1, size)) + if i: + children.append(0) + alternatives = [f"r{j} '+'" for j in children] + ["NUMBER"] + lines.append(f"r{i}: " + " | ".join(alternatives)) + parser_class = make_parser("\n".join(lines) + "\n") + node = parse_string("1\n", parser_class) + self.assertEqual(node[0].string, "1") + + def test_left_recursion_analysis_work(self) -> None: + class CountedName(str): + comparisons = 0 + __hash__ = str.__hash__ + + def __eq__(self, other): + type(self).comparisons += 1 + return super().__eq__(other) + + size = 8 + lines = ["start: r0 NEWLINE ENDMARKER"] + for i in range(size): + alternatives = [ + f"r{j} '+'" for j in range(size) if j != i + ] + ["NUMBER"] + lines.append(f"r{i}: " + " | ".join(alternatives)) + grammar = parse_string("\n".join(lines) + "\n", GrammarParser) + for rule in grammar.rules.values(): + for alt in rule.rhs.alts: + for item in alt.items: + if isinstance(item.item, NameLeaf): + item.item.value = CountedName(item.item.value) + CountedName.comparisons = 0 + with self.assertRaisesRegex(ValueError, "no leadership candidate"): + PythonParserGenerator(grammar, io.StringIO()) + self.assertLess(CountedName.comparisons, 2_000) + def test_left_recursion_too_complex(self) -> None: grammar = """ start: foo diff --git a/Misc/NEWS.d/next/Tools-Demos/2026-10-05-14-04-06.gh-issue-158847.Xbj0dg.rst b/Misc/NEWS.d/next/Tools-Demos/2026-10-05-14-04-06.gh-issue-158847.Xbj0dg.rst new file mode 100644 index 000000000000000..6a2f2df732726bc --- /dev/null +++ b/Misc/NEWS.d/next/Tools-Demos/2026-10-05-14-04-06.gh-issue-158847.Xbj0dg.rst @@ -0,0 +1,2 @@ +Pegen now picks the leader of a left-recursive rule cycle in polynomial time, +instead of enumerating every path through the cycle. diff --git a/Tools/peg_generator/pegen/grammar_analysis.py b/Tools/peg_generator/pegen/grammar_analysis.py index 5b716cf209e000a..2f95f9699c8d5fe 100644 --- a/Tools/peg_generator/pegen/grammar_analysis.py +++ b/Tools/peg_generator/pegen/grammar_analysis.py @@ -115,19 +115,15 @@ def compute_left_recursives( if len(scc) > 1: for name in scc: rules[name].left_recursive = True - # Try to find a leader such that all cycles go through it. - leaders = set(scc) - for start in scc: - for cycle in sccutils.find_cycles_in_scc(graph, scc, start): - # print("Cycle:", " -> ".join(cycle)) - leaders -= scc - set(cycle) - if not leaders: - raise ValueError( - f"SCC {scc} has no leadership candidate (no element is included in all cycles)" - ) - # print("Leaders:", leaders) - leader = min(leaders) # Pick an arbitrary leader from the candidates. - rules[leader].leader = True + # A leader lies in every cycle, so removing it must leave a DAG. + for leader in sorted(scc): + if sccutils.is_acyclic(graph, scc - {leader}): + rules[leader].leader = True + break + else: + raise ValueError( + f"SCC {scc} has no leadership candidate (no element is included in all cycles)" + ) else: name = min(scc) # The only element. if name in graph[name]: diff --git a/Tools/peg_generator/pegen/sccutils.py b/Tools/peg_generator/pegen/sccutils.py index db30fc283465b88..5362f3b0a3cf1a9 100644 --- a/Tools/peg_generator/pegen/sccutils.py +++ b/Tools/peg_generator/pegen/sccutils.py @@ -1,6 +1,6 @@ # Adapted from mypy (mypy/build.py) under the MIT license. -from collections.abc import Iterable, Iterator, Set +from collections.abc import Iterator, Set def strongly_connected_components( @@ -49,32 +49,22 @@ def dfs(v: str) -> Iterator[set[str]]: yield from dfs(v) -def find_cycles_in_scc( - graph: dict[str, Set[str]], scc: Set[str], start: str -) -> Iterable[list[str]]: - """Find cycles in SCC emanating from start. - - Yields lists of the form ['A', 'B', 'C', 'A'], which means there's - a path from A -> B -> C -> A. The first item is always the start - argument, but the last item may be another element, e.g. ['A', - 'B', 'C', 'B'] means there's a path from A to B and there's a - cycle from B to C and back. - """ - # Basic input checks. - assert start in scc, (start, scc) - assert scc <= graph.keys(), scc - graph.keys() - - # Reduce the graph to nodes in the SCC. - graph = {src: {dst for dst in dsts if dst in scc} for src, dsts in graph.items() if src in scc} - assert start in graph - - # Recursive helper that yields cycles. - def dfs(node: str, path: list[str]) -> Iterator[list[str]]: - if node in path: - yield path + [node] - return - path = path + [node] # TODO: Make this not quadratic. - for child in graph[node]: - yield from dfs(child, path) - - yield from dfs(start, []) +def is_acyclic(graph: dict[str, Set[str]], vertices: Set[str]) -> bool: + """Check the subgraph induced by vertices using a topological sort.""" + indegree = dict.fromkeys(vertices, 0) + for src in vertices: + for dst in graph[src]: + if dst in vertices: + indegree[dst] += 1 + + ready = [node for node, degree in indegree.items() if degree == 0] + processed = 0 + while ready: + src = ready.pop() + processed += 1 + for dst in graph[src]: + if dst in vertices: + indegree[dst] -= 1 + if indegree[dst] == 0: + ready.append(dst) + return processed == len(vertices)