From 46023abaeddb901652db73db61f2b571ec386669 Mon Sep 17 00:00:00 2001 From: Kyle King Date: Mon, 20 Jul 2026 22:25:35 -0600 Subject: [PATCH 1/5] fix: address confirmed regex vulnerabilities --- mdit_py_plugins/admon/index.py | 6 +- mdit_py_plugins/texmath/index.py | 10 +-- tests/test_admon.py | 7 ++ tests/test_redos.py | 108 +++++++++++++++++++++++++++++++ tests/test_texmath.py | 15 +++++ 5 files changed, 140 insertions(+), 6 deletions(-) create mode 100644 tests/test_redos.py diff --git a/mdit_py_plugins/admon/index.py b/mdit_py_plugins/admon/index.py index 6e28940..e26b554 100644 --- a/mdit_py_plugins/admon/index.py +++ b/mdit_py_plugins/admon/index.py @@ -18,10 +18,12 @@ from markdown_it.utils import EnvType, OptionsDict +TAGS_RE = re.compile(r'^\s*(?P[^"\s](?:[^"]*[^"\s])?)\s+"(?P.*)"\S*$') + + def _get_multiple_tags(params: str) -> tuple[list[str], str]: """Check for multiple tags when the title is double quoted.""" - re_tags = re.compile(r'^\s*(?P<tokens>[^"]+)\s+"(?P<title>.*)"\S*$') - match = re_tags.match(params) + match = TAGS_RE.match(params) if match: tags = match["tokens"].strip().split(" ") return [tag.lower() for tag in tags], match["title"] diff --git a/mdit_py_plugins/texmath/index.py b/mdit_py_plugins/texmath/index.py index 67372fa..8bae929 100644 --- a/mdit_py_plugins/texmath/index.py +++ b/mdit_py_plugins/texmath/index.py @@ -228,14 +228,15 @@ def render(tex: str, displayMode: bool, macros: Any) -> str: { "name": "math_block_eqno", "rex": re.compile( - r"^`{3}math\s+?([^`]+?)\s+?`{3}\s*?\(([^)$\r\n]+?)\)", re.M + r"^`{3}math\s+([^`\s](?:[^`]*[^`\s])?)\s+`{3}\s*?\(([^)$\r\n]+?)\)", + re.M, ), "tmpl": '<section class="eqno">\n<eqn>{0}</eqn><span>({1})</span>\n</section>\n', "tag": "```math", }, { "name": "math_block", - "rex": re.compile(r"^`{3}math\s+?([^`]+?)\s+?`{3}", re.M), + "rex": re.compile(r"^`{3}math\s+([^`\s](?:[^`]*[^`\s])?)\s+`{3}", re.M), "tmpl": "<section>\n<eqn>{0}</eqn>\n</section>\n", "tag": "```math", }, @@ -270,14 +271,15 @@ def render(tex: str, displayMode: bool, macros: Any) -> str: { "name": "math_block_eqno", "rex": re.compile( - r"^`{3}math\s+?([^`]+?)\s+?`{3}\s*?\(([^)$\r\n]+?)\)", re.M + r"^`{3}math\s+([^`\s](?:[^`]*[^`\s])?)\s+`{3}\s*?\(([^)$\r\n]+?)\)", + re.M, ), "tmpl": '<section class="eqno"><eqn>{0}</eqn><span>({1})</span></section>', "tag": "```math", }, { "name": "math_block", - "rex": re.compile(r"^`{3}math\s+?([^`]+?)\s+?`{3}", re.M), + "rex": re.compile(r"^`{3}math\s+([^`\s](?:[^`]*[^`\s])?)\s+`{3}", re.M), "tmpl": "<section><eqn>{0}</eqn></section>", "tag": "```math", }, diff --git a/tests/test_admon.py b/tests/test_admon.py index f0ef479..a9f36bb 100644 --- a/tests/test_admon.py +++ b/tests/test_admon.py @@ -34,3 +34,10 @@ def test_plugin_parse(data_regression, text_idx): md = MarkdownIt().use(admon_plugin) tokens = md.parse(dedent(texts[text_idx])) data_regression.check([t.as_dict() for t in tokens]) + + +@pytest.mark.timeout(2) +def test_title_parsing_redos(): + """A long unquoted admonition header must not trigger catastrophic backtracking.""" + md = MarkdownIt("commonmark").use(admon_plugin) + md.render("!!! note " + "a " * 50000 + "b\n content") diff --git a/tests/test_redos.py b/tests/test_redos.py new file mode 100644 index 0000000..4993229 --- /dev/null +++ b/tests/test_redos.py @@ -0,0 +1,108 @@ +"""Categorical ReDoS guard for every regex shipped by the package. + +New plugins add regexes over time. Rather than trust review to catch catastrophic +backtracking, this test discovers every compiled pattern reachable from the +package and asserts it stays fast against adversarial homogeneous input (the +class of input that makes overlapping or nested quantifiers blow up). +""" + +from __future__ import annotations + +import importlib +import pkgutil +import re +import time + +import pytest + +import mdit_py_plugins + +# Homogeneous runs of a single class of character are what drive overlapping +# quantifiers (e.g. `\s+ [^x]+ \s+`) into super-linear backtracking. +_ADVERSARIAL_CHARS = [" ", "\t", "a", "1", "$", "`", "\\", '"', ".", "-"] +_LENGTH = 20_000 +_BUDGET_SECONDS = 0.5 + +# Backslash escapes that denote a character *class*, not a literal, so prefix +# extraction must stop rather than emit the letter that follows the backslash. +_CLASS_ESCAPES = set("sSdDwWbBAZ") + + +def _literal_prefix(source: str) -> str: + """Best-effort leading literal of a regex, so anchored patterns get exercised. + + Anchored rules like ``^```math\\s+...`` only backtrack once their literal + lead-in matches, so an adversarial run must be prefixed with that literal. + """ + if source.startswith("^"): + source = source[1:] + out: list[str] = [] + i = 0 + while i < len(source): + char = source[i] + if char == "\\" and i + 1 < len(source): + nxt = source[i + 1] + if nxt in _CLASS_ESCAPES: + break + out.append(nxt) + i += 2 + elif char in ".^$*+?()[]{}|": + break + else: + out.append(char) + i += 1 + if i < len(source) and source[i] == "{": + end = source.find("}", i) + if end == -1 or not out: + break + count = source[i + 1 : end].split(",")[0] + if not count.isdigit(): + break + out.append(out[-1] * (int(count) - 1)) + i = end + 1 + return "".join(out) + + +def _iter_patterns() -> list[tuple[str, re.Pattern[str]]]: + found: dict[int, tuple[str, re.Pattern[str]]] = {} + + def _collect(where: str, value: object) -> None: + if isinstance(value, re.Pattern): + found.setdefault(id(value), (where, value)) + elif isinstance(value, dict): + for item in value.values(): + _collect(where, item) + elif isinstance(value, (list, tuple)): + for item in value: + _collect(where, item) + + for info in pkgutil.walk_packages( + mdit_py_plugins.__path__, prefix=f"{mdit_py_plugins.__name__}." + ): + module = importlib.import_module(info.name) + for name, value in vars(module).items(): + _collect(f"{info.name}.{name}", value) + + return sorted(found.values(), key=lambda item: item[0]) + + +_PATTERNS = _iter_patterns() + + +@pytest.mark.timeout(10) +@pytest.mark.parametrize("where,pattern", _PATTERNS, ids=[w for w, _ in _PATTERNS]) +def test_regex_is_redos_safe(where: str, pattern: re.Pattern[str]) -> None: + prefix = _literal_prefix(pattern.pattern) + for char in _ADVERSARIAL_CHARS: + payload = prefix + char * _LENGTH + start = time.perf_counter() + pattern.search(payload) + elapsed = time.perf_counter() - start + assert elapsed < _BUDGET_SECONDS, ( + f"{where} took {elapsed:.3f}s on {prefix!r}+{_LENGTH}x{char!r}; " + "likely catastrophic backtracking (ReDoS)" + ) + + +def test_patterns_were_discovered() -> None: + assert len(_PATTERNS) > 10 diff --git a/tests/test_texmath.py b/tests/test_texmath.py index 42f7d9b..da601b6 100644 --- a/tests/test_texmath.py +++ b/tests/test_texmath.py @@ -106,3 +106,18 @@ def test_bracket_fixtures(line, title, input, expected): text = md.render(input) print(text) assert text.rstrip() == expected.rstrip() + + +@pytest.mark.parametrize("delimiters", ["gitlab", "julia"]) +def test_fenced_math_block(delimiters): + md = MarkdownIt("commonmark").use(texmath_plugin, delimiters=delimiters) + text = md.render("```math\n\\alpha = 1\n```\n") + assert "<eqn>\\alpha = 1</eqn>" in text + + +@pytest.mark.parametrize("delimiters", ["gitlab", "julia"]) +@pytest.mark.timeout(2) +def test_fenced_math_block_redos(delimiters): + """An unclosed ``` ```math ``` fence must not trigger catastrophic backtracking.""" + md = MarkdownIt("commonmark").use(texmath_plugin, delimiters=delimiters) + md.render("```math\n" + " " * 20000) From 8309e76e40268dbdc88361c4350d9eeffde5674f Mon Sep 17 00:00:00 2001 From: Kyle King <KyleKing@users.noreply.github.com> Date: Mon, 20 Jul 2026 22:46:43 -0600 Subject: [PATCH 2/5] refactor: check inline regex too Hoist inline and string regexes to module-level compiled patterns (amsmath, tasklists, dollarmath, anchors) so the ReDoS guard discovers them. Compiling is also faster for these hot paths. Inline re.match(str, ...) still compiles once and caches, but pays a cache lookup on every call, so a compiled object runs about 2.2x faster per call (102 vs 235 ns on py3.14). The one-time compile (5 to 59 us) is paid once either way, and module globals sidestep re._cache's 512-entry eviction. Benched with timeit (best-of-5, 2M calls) comparing compiled.match(text) against a cache-warm re.match(pattern, text). --- mdit_py_plugins/amsmath/__init__.py | 4 ++-- mdit_py_plugins/anchors/index.py | 5 +++- mdit_py_plugins/dollarmath/index.py | 5 +++- mdit_py_plugins/tasklists/__init__.py | 3 ++- tests/test_redos.py | 34 +++++++++++++++++---------- 5 files changed, 33 insertions(+), 18 deletions(-) diff --git a/mdit_py_plugins/amsmath/__init__.py b/mdit_py_plugins/amsmath/__init__.py index bad1b21..c5a9927 100644 --- a/mdit_py_plugins/amsmath/__init__.py +++ b/mdit_py_plugins/amsmath/__init__.py @@ -55,7 +55,7 @@ # whose total width is the actual width of the contents; # thus they can be used as a component in a containing expression -RE_OPEN = r"\\begin\{(" + "|".join(ENVIRONMENTS) + r")([\*]?)\}" +RE_OPEN = re.compile(r"\\begin\{(" + "|".join(ENVIRONMENTS) + r")([\*]?)\}") def amsmath_plugin( @@ -112,7 +112,7 @@ def amsmath_block( first_end = state.eMarks[startLine] first_text = state.src[first_start:first_end] - if not (match_open := re.match(RE_OPEN, first_text)): + if not (match_open := RE_OPEN.match(first_text)): return False # construct the closing tag diff --git a/mdit_py_plugins/anchors/index.py b/mdit_py_plugins/anchors/index.py index 66bf0aa..f00eb6d 100644 --- a/mdit_py_plugins/anchors/index.py +++ b/mdit_py_plugins/anchors/index.py @@ -115,8 +115,11 @@ def _anchor_func(state: StateCore) -> None: return _anchor_func +_NON_SLUG_CHARS_RE = re.compile(r"[^\w\u4e00-\u9fff\- ]") + + def slugify(title: str) -> str: - return re.sub(r"[^\w\u4e00-\u9fff\- ]", "", title.strip().lower().replace(" ", "-")) + return _NON_SLUG_CHARS_RE.sub("", title.strip().lower().replace(" ", "-")) def unique_slug(slug: str, slugs: set[str]) -> str: diff --git a/mdit_py_plugins/dollarmath/index.py b/mdit_py_plugins/dollarmath/index.py index 57ba590..7af9a6a 100644 --- a/mdit_py_plugins/dollarmath/index.py +++ b/mdit_py_plugins/dollarmath/index.py @@ -17,6 +17,9 @@ from markdown_it.utils import EnvType, OptionsDict +_LABEL_WHITESPACE_RE = re.compile(r"\s+") + + def dollarmath_plugin( md: MarkdownIt, *, @@ -54,7 +57,7 @@ def dollarmath_plugin( """ if label_normalizer is None: - label_normalizer = lambda label: re.sub(r"\s+", "-", label) # noqa: E731 + label_normalizer = lambda label: _LABEL_WHITESPACE_RE.sub("-", label) # noqa: E731 md.inline.ruler.before( "escape", diff --git a/mdit_py_plugins/tasklists/__init__.py b/mdit_py_plugins/tasklists/__init__.py index d80f475..d2ef09f 100644 --- a/mdit_py_plugins/tasklists/__init__.py +++ b/mdit_py_plugins/tasklists/__init__.py @@ -28,6 +28,7 @@ # https://github.github.com/gfm/#whitespace-character # (spec version 0.29-gfm (2019-04-06)) _GFM_WHITESPACE_RE = r"[ \t\n\v\f\r]" +_TASK_LIST_ITEM_RE = re.compile(rf"\[[ xX]]{_GFM_WHITESPACE_RE}+") def tasklists_plugin( @@ -148,4 +149,4 @@ def is_list_item(token: Token) -> bool: def starts_with_todo_markdown(token: Token) -> bool: # leading whitespace in a list item is already trimmed off by markdown-it - return re.match(rf"\[[ xX]]{_GFM_WHITESPACE_RE}+", token.content) is not None + return _TASK_LIST_ITEM_RE.match(token.content) is not None diff --git a/tests/test_redos.py b/tests/test_redos.py index 4993229..d86d517 100644 --- a/tests/test_redos.py +++ b/tests/test_redos.py @@ -23,6 +23,8 @@ class of input that makes overlapping or nested quantifiers blow up). _LENGTH = 20_000 _BUDGET_SECONDS = 0.5 +# Metacharacters that end the run of literal leading characters. +_METACHARS = set(".^$*+?()[]{}|") # Backslash escapes that denote a character *class*, not a literal, so prefix # extraction must stop rather than emit the letter that follows the backslash. _CLASS_ESCAPES = set("sSdDwWbBAZ") @@ -33,9 +35,11 @@ def _literal_prefix(source: str) -> str: Anchored rules like ``^```math\\s+...`` only backtrack once their literal lead-in matches, so an adversarial run must be prefixed with that literal. + Parsing walks the source until the first metacharacter or character class, + treating ``\\x`` as the literal ``x`` and expanding a ``{n}`` repeat of the + preceding literal (so ```` `{3} ```` yields three backticks). """ - if source.startswith("^"): - source = source[1:] + source = source.removeprefix("^") out: list[str] = [] i = 0 while i < len(source): @@ -46,26 +50,26 @@ def _literal_prefix(source: str) -> str: break out.append(nxt) i += 2 - elif char in ".^$*+?()[]{}|": + elif char == "{" and out: + end = source.find("}", i) + count = source[i + 1 : end] if end != -1 else "" + if not (repeats := count.split(",")[0]).isdigit(): + break + out.append(out[-1] * (int(repeats) - 1)) + i = end + 1 + elif char in _METACHARS: break else: out.append(char) i += 1 - if i < len(source) and source[i] == "{": - end = source.find("}", i) - if end == -1 or not out: - break - count = source[i + 1 : end].split(",")[0] - if not count.isdigit(): - break - out.append(out[-1] * (int(count) - 1)) - i = end + 1 return "".join(out) def _iter_patterns() -> list[tuple[str, re.Pattern[str]]]: found: dict[int, tuple[str, re.Pattern[str]]] = {} + # Patterns may sit directly in a module global or nested inside a global + # container (e.g. texmath's ``rules`` dict of delimiter flavors), so recurse. def _collect(where: str, value: object) -> None: if isinstance(value, re.Pattern): found.setdefault(id(value), (where, value)) @@ -88,9 +92,13 @@ def _collect(where: str, value: object) -> None: _PATTERNS = _iter_patterns() +# Several patterns share a ``where`` (e.g. every texmath flavor lives in one +# ``rules`` dict), so include the regex source to keep failing ids unambiguous. +_IDS = [f"{where} {pattern.pattern[:48]}" for where, pattern in _PATTERNS] + @pytest.mark.timeout(10) -@pytest.mark.parametrize("where,pattern", _PATTERNS, ids=[w for w, _ in _PATTERNS]) +@pytest.mark.parametrize("where,pattern", _PATTERNS, ids=_IDS) def test_regex_is_redos_safe(where: str, pattern: re.Pattern[str]) -> None: prefix = _literal_prefix(pattern.pattern) for char in _ADVERSARIAL_CHARS: From c290fd8f0309f11066f820892edb2513783636f4 Mon Sep 17 00:00:00 2001 From: Kyle King <KyleKing@users.noreply.github.com> Date: Tue, 21 Jul 2026 08:11:47 -0600 Subject: [PATCH 3/5] fix: address review comments/re-scope --- CHANGELOG.md | 17 ++ mdit_py_plugins/admon/index.py | 6 +- mdit_py_plugins/amsmath/__init__.py | 4 +- mdit_py_plugins/anchors/index.py | 5 +- mdit_py_plugins/dollarmath/index.py | 5 +- mdit_py_plugins/tasklists/__init__.py | 3 +- tests/test_admon.py | 7 - tests/test_redos.py | 238 ++++++++++++++++---------- 8 files changed, 167 insertions(+), 118 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a1b78b3..92c8da6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,22 @@ # Change Log +## Unreleased + +- 🐛 FIX: Resolve regex denial-of-service (ReDoS) in `texmath` and `admon` (#143, #148) + + Two shipped patterns backtracked super-linearly on adversarial input with no + closing delimiter, so a single crafted document could hang the parser: + + - `texmath`'s `` ```math `` block rules (all `` ```math `` flavors, i.e. the + `gitlab` and `julia` delimiters) blew up on a `` ```math `` fence followed by a + long whitespace run with no closing fence. + - `admon`'s multi-tag title parser blew up on an unquoted header containing a long + interior whitespace run. + + Both patterns were rewritten to anchor the capture on non-whitespace at both ends, + which stays flat on the same input. A behavioral guard (`tests/test_redos.py`) now + renders adversarial payloads through every plugin to catch the next such pattern. + ## 0.7.0 - 2026-07-19 - ✨ NEW: Add section reference plugin (`section_ref`) (#144) diff --git a/mdit_py_plugins/admon/index.py b/mdit_py_plugins/admon/index.py index e26b554..d603e1a 100644 --- a/mdit_py_plugins/admon/index.py +++ b/mdit_py_plugins/admon/index.py @@ -18,12 +18,10 @@ from markdown_it.utils import EnvType, OptionsDict -TAGS_RE = re.compile(r'^\s*(?P<tokens>[^"\s](?:[^"]*[^"\s])?)\s+"(?P<title>.*)"\S*$') - - def _get_multiple_tags(params: str) -> tuple[list[str], str]: """Check for multiple tags when the title is double quoted.""" - match = TAGS_RE.match(params) + re_tags = re.compile(r'^\s*(?P<tokens>[^"\s](?:[^"]*[^"\s])?)\s+"(?P<title>.*)"\S*$') + match = re_tags.match(params) if match: tags = match["tokens"].strip().split(" ") return [tag.lower() for tag in tags], match["title"] diff --git a/mdit_py_plugins/amsmath/__init__.py b/mdit_py_plugins/amsmath/__init__.py index c5a9927..bad1b21 100644 --- a/mdit_py_plugins/amsmath/__init__.py +++ b/mdit_py_plugins/amsmath/__init__.py @@ -55,7 +55,7 @@ # whose total width is the actual width of the contents; # thus they can be used as a component in a containing expression -RE_OPEN = re.compile(r"\\begin\{(" + "|".join(ENVIRONMENTS) + r")([\*]?)\}") +RE_OPEN = r"\\begin\{(" + "|".join(ENVIRONMENTS) + r")([\*]?)\}" def amsmath_plugin( @@ -112,7 +112,7 @@ def amsmath_block( first_end = state.eMarks[startLine] first_text = state.src[first_start:first_end] - if not (match_open := RE_OPEN.match(first_text)): + if not (match_open := re.match(RE_OPEN, first_text)): return False # construct the closing tag diff --git a/mdit_py_plugins/anchors/index.py b/mdit_py_plugins/anchors/index.py index f00eb6d..66bf0aa 100644 --- a/mdit_py_plugins/anchors/index.py +++ b/mdit_py_plugins/anchors/index.py @@ -115,11 +115,8 @@ def _anchor_func(state: StateCore) -> None: return _anchor_func -_NON_SLUG_CHARS_RE = re.compile(r"[^\w\u4e00-\u9fff\- ]") - - def slugify(title: str) -> str: - return _NON_SLUG_CHARS_RE.sub("", title.strip().lower().replace(" ", "-")) + return re.sub(r"[^\w\u4e00-\u9fff\- ]", "", title.strip().lower().replace(" ", "-")) def unique_slug(slug: str, slugs: set[str]) -> str: diff --git a/mdit_py_plugins/dollarmath/index.py b/mdit_py_plugins/dollarmath/index.py index 7af9a6a..57ba590 100644 --- a/mdit_py_plugins/dollarmath/index.py +++ b/mdit_py_plugins/dollarmath/index.py @@ -17,9 +17,6 @@ from markdown_it.utils import EnvType, OptionsDict -_LABEL_WHITESPACE_RE = re.compile(r"\s+") - - def dollarmath_plugin( md: MarkdownIt, *, @@ -57,7 +54,7 @@ def dollarmath_plugin( """ if label_normalizer is None: - label_normalizer = lambda label: _LABEL_WHITESPACE_RE.sub("-", label) # noqa: E731 + label_normalizer = lambda label: re.sub(r"\s+", "-", label) # noqa: E731 md.inline.ruler.before( "escape", diff --git a/mdit_py_plugins/tasklists/__init__.py b/mdit_py_plugins/tasklists/__init__.py index d2ef09f..d80f475 100644 --- a/mdit_py_plugins/tasklists/__init__.py +++ b/mdit_py_plugins/tasklists/__init__.py @@ -28,7 +28,6 @@ # https://github.github.com/gfm/#whitespace-character # (spec version 0.29-gfm (2019-04-06)) _GFM_WHITESPACE_RE = r"[ \t\n\v\f\r]" -_TASK_LIST_ITEM_RE = re.compile(rf"\[[ xX]]{_GFM_WHITESPACE_RE}+") def tasklists_plugin( @@ -149,4 +148,4 @@ def is_list_item(token: Token) -> bool: def starts_with_todo_markdown(token: Token) -> bool: # leading whitespace in a list item is already trimmed off by markdown-it - return _TASK_LIST_ITEM_RE.match(token.content) is not None + return re.match(rf"\[[ xX]]{_GFM_WHITESPACE_RE}+", token.content) is not None diff --git a/tests/test_admon.py b/tests/test_admon.py index a9f36bb..f0ef479 100644 --- a/tests/test_admon.py +++ b/tests/test_admon.py @@ -34,10 +34,3 @@ def test_plugin_parse(data_regression, text_idx): md = MarkdownIt().use(admon_plugin) tokens = md.parse(dedent(texts[text_idx])) data_regression.check([t.as_dict() for t in tokens]) - - -@pytest.mark.timeout(2) -def test_title_parsing_redos(): - """A long unquoted admonition header must not trigger catastrophic backtracking.""" - md = MarkdownIt("commonmark").use(admon_plugin) - md.render("!!! note " + "a " * 50000 + "b\n content") diff --git a/tests/test_redos.py b/tests/test_redos.py index d86d517..22f9650 100644 --- a/tests/test_redos.py +++ b/tests/test_redos.py @@ -1,116 +1,164 @@ -"""Categorical ReDoS guard for every regex shipped by the package. +"""Behavioral ReDoS guard: render adversarial markdown through each plugin. New plugins add regexes over time. Rather than trust review to catch catastrophic -backtracking, this test discovers every compiled pattern reachable from the -package and asserts it stays fast against adversarial homogeneous input (the -class of input that makes overlapping or nested quantifiers blow up). +backtracking, this drives each plugin end-to-end (``MarkdownIt().use(plugin)`` then +``md.render(payload)``) against adversarial input and asserts it stays fast. + +Two properties make this faithful to the real attack surface without introspecting +any pattern: + +- The payload embeds a long *homogeneous run* of a single character class, which is + what pushes overlapping or nested quantifiers (e.g. ``\\s+ [^x]+ \\s+``) into + super-linear backtracking. We fuzz every run character, so we never need to know + which regex is vulnerable or how it is structured. +- Rendering exercises the *whole* rule chain (the parser re-invoking a block rule + across offsets), so it also catches the O(n^2) amplification an isolated-regex + check would miss, and never false-positives on a pattern adversarial input can't + actually reach. + +The only per-plugin cost is one hand-written activation payload: the plugin's public +markdown syntax with a ``{run}`` placeholder for the homogeneous run. """ from __future__ import annotations -import importlib -import pkgutil -import re +from collections.abc import Callable +from dataclasses import dataclass import time +from markdown_it import MarkdownIt import pytest -import mdit_py_plugins - -# Homogeneous runs of a single class of character are what drive overlapping -# quantifiers (e.g. `\s+ [^x]+ \s+`) into super-linear backtracking. -_ADVERSARIAL_CHARS = [" ", "\t", "a", "1", "$", "`", "\\", '"', ".", "-"] -_LENGTH = 20_000 +from mdit_py_plugins.admon import admon_plugin +from mdit_py_plugins.amsmath import amsmath_plugin +from mdit_py_plugins.anchors import anchors_plugin +from mdit_py_plugins.colon_fence import colon_fence_plugin +from mdit_py_plugins.deflist import deflist_plugin +from mdit_py_plugins.dollarmath import dollarmath_plugin +from mdit_py_plugins.field_list import fieldlist_plugin +from mdit_py_plugins.footnote import footnote_plugin +from mdit_py_plugins.myst_blocks import myst_block_plugin +from mdit_py_plugins.myst_role import myst_role_plugin +from mdit_py_plugins.tasklists import tasklists_plugin +from mdit_py_plugins.texmath import texmath_plugin + +# Homogeneous runs of one character class are what drive overlapping/nested +# quantifiers into catastrophic backtracking; fuzz every plugin with each. +_FILLERS = [" ", "\t", "a", "1", "$", "`", "\\", '"', ".", "-"] +_RUN = 50_000 _BUDGET_SECONDS = 0.5 -# Metacharacters that end the run of literal leading characters. -_METACHARS = set(".^$*+?()[]{}|") -# Backslash escapes that denote a character *class*, not a literal, so prefix -# extraction must stop rather than emit the letter that follows the backslash. -_CLASS_ESCAPES = set("sSdDwWbBAZ") - - -def _literal_prefix(source: str) -> str: - """Best-effort leading literal of a regex, so anchored patterns get exercised. - - Anchored rules like ``^```math\\s+...`` only backtrack once their literal - lead-in matches, so an adversarial run must be prefixed with that literal. - Parsing walks the source until the first metacharacter or character class, - treating ``\\x`` as the literal ``x`` and expanding a ``{n}`` repeat of the - preceding literal (so ```` `{3} ```` yields three backticks). - """ - source = source.removeprefix("^") - out: list[str] = [] - i = 0 - while i < len(source): - char = source[i] - if char == "\\" and i + 1 < len(source): - nxt = source[i + 1] - if nxt in _CLASS_ESCAPES: - break - out.append(nxt) - i += 2 - elif char == "{" and out: - end = source.find("}", i) - count = source[i + 1 : end] if end != -1 else "" - if not (repeats := count.split(",")[0]).isdigit(): - break - out.append(out[-1] * (int(repeats) - 1)) - i = end + 1 - elif char in _METACHARS: - break - else: - out.append(char) - i += 1 - return "".join(out) - - -def _iter_patterns() -> list[tuple[str, re.Pattern[str]]]: - found: dict[int, tuple[str, re.Pattern[str]]] = {} - - # Patterns may sit directly in a module global or nested inside a global - # container (e.g. texmath's ``rules`` dict of delimiter flavors), so recurse. - def _collect(where: str, value: object) -> None: - if isinstance(value, re.Pattern): - found.setdefault(id(value), (where, value)) - elif isinstance(value, dict): - for item in value.values(): - _collect(where, item) - elif isinstance(value, (list, tuple)): - for item in value: - _collect(where, item) - for info in pkgutil.walk_packages( - mdit_py_plugins.__path__, prefix=f"{mdit_py_plugins.__name__}." - ): - module = importlib.import_module(info.name) - for name, value in vars(module).items(): - _collect(f"{info.name}.{name}", value) - - return sorted(found.values(), key=lambda item: item[0]) - - -_PATTERNS = _iter_patterns() - -# Several patterns share a ``where`` (e.g. every texmath flavor lives in one -# ``rules`` dict), so include the regex source to keep failing ids unambiguous. -_IDS = [f"{where} {pattern.pattern[:48]}" for where, pattern in _PATTERNS] +@dataclass(frozen=True) +class Case: + id: str + configure: Callable[[MarkdownIt], None] + payload: Callable[[str], str] + + +_CASES = [ + Case( + # An interior run with no closing quote: ``[^"]+\s+"`` backtracks the run + # exhaustively looking for a ``"`` that never comes. A trailing run would be + # stripped off before the regex sees it, so the run must sit between tokens. + "admon-header", + lambda md: md.use(admon_plugin), + lambda run: f"!!! note a{run}b\n content", + ), + Case( + "amsmath-open", + lambda md: md.use(amsmath_plugin), + lambda run: f"\\begin{{{run}", + ), + Case( + "amsmath-unclosed", + lambda md: md.use(amsmath_plugin), + lambda run: f"\\begin{{equation}}\n{run}", + ), + Case( + "anchors-heading", + lambda md: md.use(anchors_plugin), + lambda run: f"# {run}", + ), + Case( + "colon-fence", + lambda md: md.use(colon_fence_plugin), + lambda run: f"::: {run}\ncontent\n:::", + ), + Case( + "deflist", + lambda md: md.use(deflist_plugin), + lambda run: f"term\n:{run}", + ), + Case( + "dollarmath-inline", + lambda md: md.use(dollarmath_plugin), + lambda run: f"${run}$", + ), + Case( + "dollarmath-block", + lambda md: md.use(dollarmath_plugin, allow_labels=True), + lambda run: f"$$\n{run}", + ), + Case( + "dollarmath-label", + lambda md: md.use(dollarmath_plugin, allow_labels=True), + lambda run: f"$$a$$ ({run})", + ), + Case( + "fieldlist", + lambda md: md.use(fieldlist_plugin), + lambda run: f":name:{run}", + ), + Case( + "footnote-ref", + lambda md: md.use(footnote_plugin), + lambda run: f"[^{run}]: body", + ), + Case( + "myst-block-target", + lambda md: md.use(myst_block_plugin), + lambda run: f"({run})=", + ), + Case( + "myst-role", + lambda md: md.use(myst_role_plugin), + lambda run: f"{{role}}`{run}`", + ), + Case( + "tasklist", + lambda md: md.use(tasklists_plugin), + lambda run: f"- [ ]{run}x", + ), + Case( + "texmath-fence", + lambda md: md.use(texmath_plugin, delimiters="gitlab"), + lambda run: f"```math\n{run}", + ), + Case( + "texmath-fence-eqno", + lambda md: md.use(texmath_plugin, delimiters="gitlab"), + lambda run: f"```math\n{run}``` (1)", + ), + Case( + "texmath-dollar", + lambda md: md.use(texmath_plugin, delimiters="dollars"), + lambda run: f"${run}$", + ), +] @pytest.mark.timeout(10) -@pytest.mark.parametrize("where,pattern", _PATTERNS, ids=_IDS) -def test_regex_is_redos_safe(where: str, pattern: re.Pattern[str]) -> None: - prefix = _literal_prefix(pattern.pattern) - for char in _ADVERSARIAL_CHARS: - payload = prefix + char * _LENGTH +@pytest.mark.parametrize("case", _CASES, ids=lambda case: case.id) +def test_plugin_is_redos_safe(case: Case) -> None: + for filler in _FILLERS: + md = MarkdownIt("commonmark") + case.configure(md) + source = case.payload(filler * _RUN) start = time.perf_counter() - pattern.search(payload) + md.render(source) elapsed = time.perf_counter() - start assert elapsed < _BUDGET_SECONDS, ( - f"{where} took {elapsed:.3f}s on {prefix!r}+{_LENGTH}x{char!r}; " + f"{case.id} took {elapsed:.3f}s on a {_RUN}x{filler!r} run; " "likely catastrophic backtracking (ReDoS)" ) - - -def test_patterns_were_discovered() -> None: - assert len(_PATTERNS) > 10 From bdc7f414e6e184a2efc1127b3eb477bb9a183573 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 21 Jul 2026 15:54:39 +0000 Subject: [PATCH 4/5] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- mdit_py_plugins/admon/index.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mdit_py_plugins/admon/index.py b/mdit_py_plugins/admon/index.py index d603e1a..e54c1ae 100644 --- a/mdit_py_plugins/admon/index.py +++ b/mdit_py_plugins/admon/index.py @@ -20,7 +20,9 @@ def _get_multiple_tags(params: str) -> tuple[list[str], str]: """Check for multiple tags when the title is double quoted.""" - re_tags = re.compile(r'^\s*(?P<tokens>[^"\s](?:[^"]*[^"\s])?)\s+"(?P<title>.*)"\S*$') + re_tags = re.compile( + r'^\s*(?P<tokens>[^"\s](?:[^"]*[^"\s])?)\s+"(?P<title>.*)"\S*$' + ) match = re_tags.match(params) if match: tags = match["tokens"].strip().split(" ") From b254e54ce9d1510073ac4665fd0d5197f8c48224 Mon Sep 17 00:00:00 2001 From: Kyle King <KyleKing@users.noreply.github.com> Date: Tue, 21 Jul 2026 10:18:46 -0600 Subject: [PATCH 5/5] ci: relax time limit in CI and speed up --- tests/test_redos.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_redos.py b/tests/test_redos.py index 22f9650..cfc1320 100644 --- a/tests/test_redos.py +++ b/tests/test_redos.py @@ -43,10 +43,10 @@ from mdit_py_plugins.texmath import texmath_plugin # Homogeneous runs of one character class are what drive overlapping/nested -# quantifiers into catastrophic backtracking; fuzz every plugin with each. -_FILLERS = [" ", "\t", "a", "1", "$", "`", "\\", '"', ".", "-"] -_RUN = 50_000 -_BUDGET_SECONDS = 0.5 +# quantifiers into catastrophic backtracking; fuzz every plugin with each +_FILLERS = [" ", "\t", "$", "`", "\\", '"', "-"] +_RUN = 15_000 +_BUDGET_SECONDS = 1.0 # Safe threshold relaxed for CI @dataclass(frozen=True)