✨ NEW: Support configurable front matter markers

mdit_py_plugins/front_matter/index.py

@@ -1,36 +1,76 @@
 """Process front matter."""
 
+from functools import partial
+
 from markdown_it import MarkdownIt
 from markdown_it.rules_block import StateBlock
 
 from mdit_py_plugins.utils import is_code_block
 
 
-def front_matter_plugin(md: MarkdownIt) -> None:
+def front_matter_plugin(md: MarkdownIt, *, marker: str = "-") -> None:
     """Plugin ported from
     `markdown-it-front-matter <https://github.com/ParkSB/markdown-it-front-matter>`__.
 
-    It parses initial metadata, stored between opening/closing dashes:
+    It parses initial metadata, stored between opening/closing markers:
 
     .. code-block:: md
 
         ---
         valid-front-matter: true
         ---
 
+    For example, to extract TOML-style front matter:
+
+    .. code-block:: python
+
+        md = MarkdownIt().use(front_matter_plugin, marker="+")
+        tokens = md.parse('+++\\ntitle = "Hello"\\n+++\\n# Heading')
+        assert tokens[0].content == 'title = "Hello"'
+
+    :param marker: Single non-whitespace character used for the delimiters,
+        excluding NUL (which Markdown normalizes before parsing).
+        At least three repetitions are required; the closing delimiter must
+        be at least as long as the opening delimiter. Defaults to ``-`` for
+        YAML-style front matter. Use ``+`` for TOML-style ``+++`` delimiters
+        or ``;`` for JSON-style ``;;;`` delimiters. The content is returned
+        as raw text, without decoding YAML, TOML, or JSON. The YAML ``...``
+        terminator is recognized only with the default marker.
+
+        The plugin is meant to be used once per parser, with one marker.
+        Calling ``use`` again with another marker adds a second rule with the
+        same ``front_matter`` name, which rule-name operations such as
+        ``md.disable("front_matter")`` do not reach.
+
+        Choose a character that does not start other Markdown block syntax
+        (for example not ``#``, ``>``, ``~`` or a backtick): a document whose
+        first line starts with three or more of the marker is otherwise taken
+        as front matter when a later line closes it.
+    :raises ValueError: If the marker is not a single non-whitespace character,
+        or is NUL.
+
     """
+    if len(marker) != 1 or marker.isspace() or marker == "\x00":
+        raise ValueError(
+            "marker must be a single non-whitespace character other than NUL"
+        )
+
     md.block.ruler.before(
         "table",
         "front_matter",
-        _front_matter_rule,
+        partial(_front_matter_rule, marker_chr=marker),
         {"alt": ["paragraph", "reference", "blockquote", "list"]},
     )
 
 
 def _front_matter_rule(
-    state: StateBlock, startLine: int, endLine: int, silent: bool
+    state: StateBlock,
+    startLine: int,
+    endLine: int,
+    silent: bool,
+    *,
+    marker_chr: str = "-",
 ) -> bool:
-    marker_chr = "-"
     min_markers = 3
 
     auto_closed = False
@@ -69,12 +109,20 @@ def _front_matter_rule(
             # unclosed block should be autoclosed by end of document.
             return False
 
-        if state.src[start:maximum] == "...":
-            break
-
         start = state.bMarks[nextLine] + state.tShift[nextLine]
         maximum = state.eMarks[nextLine]
 
+        if (
+            marker_chr == "-"
+            and state.sCount[nextLine] == 0
+            and state.src.startswith("...", start)
+            and state.skipSpaces(start + 3) >= maximum
+        ):
+            # YAML document-end marker (default marker only): `...` at column 0,
+            # trailing spaces allowed; consumed here, so it is not content.
+            auto_closed = True
+            break
+
         if start < maximum and state.sCount[nextLine] < state.blkIndent:
             # non-empty line with negative indent should stop the list:
             # - ```

tests/fixtures/front_matter.md

@@ -32,6 +32,61 @@ x: 1
 <h1>Head</h1>
 .
 
+should parse triple dots at end of document:
+.
+---
+x: 1
+...
+.
+.
+
+should ignore spaces after triple dots:
+.
+---
+x: 1
+...   
+# Head
+.
+
+<h1>Head</h1>
+.
+
+should not close on indented triple dots:
+.
+---
+text: |
+  ...
+---
+# Head
+.
+
+<h1>Head</h1>
+.
+
+should not close on four dots:
+.
+---
+x: 1
+....
+---
+# Head
+.
+
+<h1>Head</h1>
+.
+
+should not close on triple dots followed by text:
+.
+---
+x: 1
+...x
+---
+# Head
+.
+
+<h1>Head</h1>
+.
+
 should parse front matter with indentation:
 .
 ---

tests/test_front_matter.py

@@ -42,8 +42,110 @@ def test_token():
     ]
 
 
+@pytest.mark.parametrize("closer", ["---", "...", "...   "])
+def test_token_content_excludes_the_closer(closer):
+    """The closing marker must not be left inside the extracted content.
+
+    ``...`` is the YAML document-end marker and is exercised by the fixture file,
+    so it has to be consumed like the dashed form rather than kept in the block.
+    """
+    md = MarkdownIt("commonmark").use(front_matter_plugin)
+    tokens = md.parse(f"---\na: 1\n{closer}")
+    assert [token.type for token in tokens] == ["front_matter"]
+    assert tokens[0].content == "a: 1"
+
+
+def test_dots_closer_on_the_last_line():
+    """A block closed by ``...`` on the final line is still front matter.
+
+    ``test_all`` cannot see this: the front_matter token is hidden, so only the
+    rendered output is asserted, and a block that is dropped rather than closed
+    renders as visible text instead of disappearing.
+    """
+    md = MarkdownIt("commonmark").use(front_matter_plugin)
+    assert md.render("---\na: 1\n...\n") == md.render("---\na: 1\n---\n") == ""
+
+
 def test_short_source():
     md = MarkdownIt("commonmark").use(front_matter_plugin)
 
     # The code should not raise an IndexError.
     assert md.parse("-")
+
+
+@pytest.mark.parametrize(
+    "marker,content", [("+", 'title = "Hello"'), (";", '{"title": "Hello"}')]
+)
+def test_custom_marker(marker, content):
+    md = MarkdownIt("commonmark").use(front_matter_plugin, marker=marker)
+    source = f"{marker * 3}\n{content}\n{marker * 3}\n# Head"
+    tokens = md.parse(source)
+
+    assert tokens[0].type == "front_matter"
+    assert tokens[0].content == content
+    assert tokens[0].markup == marker * 3
+    assert tokens[0].map == [0, 3]
+    assert tokens[0].hidden
+    assert tokens[0].block
+    assert md.render(source) == "\n<h1>Head</h1>\n"
+
+
+@pytest.mark.parametrize(
+    "opening,closing", [("+++", "+++"), ("++++", "+++++"), ("+++  ", "+++\t ")]
+)
+def test_custom_marker_empty_metadata(opening, closing):
+    md = MarkdownIt().use(front_matter_plugin, marker="+")
+    tokens = md.parse(f"{opening}\n{closing}")
+    assert len(tokens) == 1
+    assert tokens[0].type == "front_matter"
+    assert tokens[0].content == ""
+    assert tokens[0].map == [0, 2]
+
+
+@pytest.mark.parametrize(
+    "source",
+    [
+        "",
+        "+",
+        "++",
+        "++\na: 1\n++",
+        "+++\na: 1",
+        "+++\na: 1\n---",
+        "++++\na: 1\n+++",
+        "+++\na: 1\n+++ text",
+        " +++\na: 1\n+++",
+        "\n+++\na: 1\n+++",
+        "# Head\n+++\na: 1\n+++",
+    ],
+)
+def test_custom_marker_not_front_matter(source):
+    md = MarkdownIt().use(front_matter_plugin, marker="+")
+    assert all(token.type != "front_matter" for token in md.parse(source))
+
+
+def test_custom_marker_keeps_yaml_terminator_as_content():
+    md = MarkdownIt().use(front_matter_plugin, marker="+")
+    content = 'text = """\n...\n"""'
+    source = f"+++\n{content}\n+++\n# Head"
+    tokens = md.parse(source)
+    assert tokens[0].content == content
+    assert tokens[0].map == [0, 5]
+    assert md.render(source) == "\n<h1>Head</h1>\n"
+
+
+def test_marker_configuration_is_per_parser():
+    default = MarkdownIt().use(front_matter_plugin)
+    custom = MarkdownIt().use(front_matter_plugin, marker="+")
+    yaml_source = "---\nx: 1\n---"
+    toml_source = "+++\nx = 1\n+++"
+
+    assert default.parse(yaml_source)[0].type == "front_matter"
+    assert custom.parse(toml_source)[0].type == "front_matter"
+    assert all(token.type != "front_matter" for token in default.parse(toml_source))
+    assert all(token.type != "front_matter" for token in custom.parse(yaml_source))
+
+
+@pytest.mark.parametrize("marker", ["", "++", " ", "\t", "\n", "\x00"])
+def test_invalid_marker(marker):
+    with pytest.raises(ValueError, match="single non-whitespace character"):
+        MarkdownIt().use(front_matter_plugin, marker=marker)