diff --git a/mdit_py_plugins/front_matter/index.py b/mdit_py_plugins/front_matter/index.py index 1551d76..e90eca0 100644 --- a/mdit_py_plugins/front_matter/index.py +++ b/mdit_py_plugins/front_matter/index.py @@ -1,16 +1,18 @@ """Process front matter.""" +from functools import partial + from markdown_it import MarkdownIt from markdown_it.rules_block import StateBlock from mdit_py_plugins.utils import is_code_block -def front_matter_plugin(md: MarkdownIt) -> None: +def front_matter_plugin(md: MarkdownIt, *, marker: str = "-") -> None: """Plugin ported from `markdown-it-front-matter `__. - It parses initial metadata, stored between opening/closing dashes: + It parses initial metadata, stored between opening/closing markers: .. code-block:: md @@ -18,19 +20,57 @@ def front_matter_plugin(md: MarkdownIt) -> None: valid-front-matter: true --- + For example, to extract TOML-style front matter: + + .. code-block:: python + + md = MarkdownIt().use(front_matter_plugin, marker="+") + tokens = md.parse('+++\\ntitle = "Hello"\\n+++\\n# Heading') + assert tokens[0].content == 'title = "Hello"' + + :param marker: Single non-whitespace character used for the delimiters, + excluding NUL (which Markdown normalizes before parsing). + At least three repetitions are required; the closing delimiter must + be at least as long as the opening delimiter. Defaults to ``-`` for + YAML-style front matter. Use ``+`` for TOML-style ``+++`` delimiters + or ``;`` for JSON-style ``;;;`` delimiters. The content is returned + as raw text, without decoding YAML, TOML, or JSON. The YAML ``...`` + terminator is recognized only with the default marker. + + The plugin is meant to be used once per parser, with one marker. + Calling ``use`` again with another marker adds a second rule with the + same ``front_matter`` name, which rule-name operations such as + ``md.disable("front_matter")`` do not reach. + + Choose a character that does not start other Markdown block syntax + (for example not ``#``, ``>``, ``~`` or a backtick): a document whose + first line starts with three or more of the marker is otherwise taken + as front matter when a later line closes it. + :raises ValueError: If the marker is not a single non-whitespace character, + or is NUL. + """ + if len(marker) != 1 or marker.isspace() or marker == "\x00": + raise ValueError( + "marker must be a single non-whitespace character other than NUL" + ) + md.block.ruler.before( "table", "front_matter", - _front_matter_rule, + partial(_front_matter_rule, marker_chr=marker), {"alt": ["paragraph", "reference", "blockquote", "list"]}, ) def _front_matter_rule( - state: StateBlock, startLine: int, endLine: int, silent: bool + state: StateBlock, + startLine: int, + endLine: int, + silent: bool, + *, + marker_chr: str = "-", ) -> bool: - marker_chr = "-" min_markers = 3 auto_closed = False @@ -69,12 +109,20 @@ def _front_matter_rule( # unclosed block should be autoclosed by end of document. return False - if state.src[start:maximum] == "...": - break - start = state.bMarks[nextLine] + state.tShift[nextLine] maximum = state.eMarks[nextLine] + if ( + marker_chr == "-" + and state.sCount[nextLine] == 0 + and state.src.startswith("...", start) + and state.skipSpaces(start + 3) >= maximum + ): + # YAML document-end marker (default marker only): `...` at column 0, + # trailing spaces allowed; consumed here, so it is not content. + auto_closed = True + break + if start < maximum and state.sCount[nextLine] < state.blkIndent: # non-empty line with negative indent should stop the list: # - ``` diff --git a/tests/fixtures/front_matter.md b/tests/fixtures/front_matter.md index e01fd4e..5cff35b 100644 --- a/tests/fixtures/front_matter.md +++ b/tests/fixtures/front_matter.md @@ -32,6 +32,61 @@ x: 1

Head

. +should parse triple dots at end of document: +. +--- +x: 1 +... +. +. + +should ignore spaces after triple dots: +. +--- +x: 1 +... +# Head +. + +

Head

+. + +should not close on indented triple dots: +. +--- +text: | + ... +--- +# Head +. + +

Head

+. + +should not close on four dots: +. +--- +x: 1 +.... +--- +# Head +. + +

Head

+. + +should not close on triple dots followed by text: +. +--- +x: 1 +...x +--- +# Head +. + +

Head

+. + should parse front matter with indentation: . --- diff --git a/tests/test_front_matter.py b/tests/test_front_matter.py index 627cb89..892f683 100644 --- a/tests/test_front_matter.py +++ b/tests/test_front_matter.py @@ -42,8 +42,110 @@ def test_token(): ] +@pytest.mark.parametrize("closer", ["---", "...", "... "]) +def test_token_content_excludes_the_closer(closer): + """The closing marker must not be left inside the extracted content. + + ``...`` is the YAML document-end marker and is exercised by the fixture file, + so it has to be consumed like the dashed form rather than kept in the block. + """ + md = MarkdownIt("commonmark").use(front_matter_plugin) + tokens = md.parse(f"---\na: 1\n{closer}") + assert [token.type for token in tokens] == ["front_matter"] + assert tokens[0].content == "a: 1" + + +def test_dots_closer_on_the_last_line(): + """A block closed by ``...`` on the final line is still front matter. + + ``test_all`` cannot see this: the front_matter token is hidden, so only the + rendered output is asserted, and a block that is dropped rather than closed + renders as visible text instead of disappearing. + """ + md = MarkdownIt("commonmark").use(front_matter_plugin) + assert md.render("---\na: 1\n...\n") == md.render("---\na: 1\n---\n") == "" + + def test_short_source(): md = MarkdownIt("commonmark").use(front_matter_plugin) # The code should not raise an IndexError. assert md.parse("-") + + +@pytest.mark.parametrize( + "marker,content", [("+", 'title = "Hello"'), (";", '{"title": "Hello"}')] +) +def test_custom_marker(marker, content): + md = MarkdownIt("commonmark").use(front_matter_plugin, marker=marker) + source = f"{marker * 3}\n{content}\n{marker * 3}\n# Head" + tokens = md.parse(source) + + assert tokens[0].type == "front_matter" + assert tokens[0].content == content + assert tokens[0].markup == marker * 3 + assert tokens[0].map == [0, 3] + assert tokens[0].hidden + assert tokens[0].block + assert md.render(source) == "\n

Head

\n" + + +@pytest.mark.parametrize( + "opening,closing", [("+++", "+++"), ("++++", "+++++"), ("+++ ", "+++\t ")] +) +def test_custom_marker_empty_metadata(opening, closing): + md = MarkdownIt().use(front_matter_plugin, marker="+") + tokens = md.parse(f"{opening}\n{closing}") + assert len(tokens) == 1 + assert tokens[0].type == "front_matter" + assert tokens[0].content == "" + assert tokens[0].map == [0, 2] + + +@pytest.mark.parametrize( + "source", + [ + "", + "+", + "++", + "++\na: 1\n++", + "+++\na: 1", + "+++\na: 1\n---", + "++++\na: 1\n+++", + "+++\na: 1\n+++ text", + " +++\na: 1\n+++", + "\n+++\na: 1\n+++", + "# Head\n+++\na: 1\n+++", + ], +) +def test_custom_marker_not_front_matter(source): + md = MarkdownIt().use(front_matter_plugin, marker="+") + assert all(token.type != "front_matter" for token in md.parse(source)) + + +def test_custom_marker_keeps_yaml_terminator_as_content(): + md = MarkdownIt().use(front_matter_plugin, marker="+") + content = 'text = """\n...\n"""' + source = f"+++\n{content}\n+++\n# Head" + tokens = md.parse(source) + assert tokens[0].content == content + assert tokens[0].map == [0, 5] + assert md.render(source) == "\n

Head

\n" + + +def test_marker_configuration_is_per_parser(): + default = MarkdownIt().use(front_matter_plugin) + custom = MarkdownIt().use(front_matter_plugin, marker="+") + yaml_source = "---\nx: 1\n---" + toml_source = "+++\nx = 1\n+++" + + assert default.parse(yaml_source)[0].type == "front_matter" + assert custom.parse(toml_source)[0].type == "front_matter" + assert all(token.type != "front_matter" for token in default.parse(toml_source)) + assert all(token.type != "front_matter" for token in custom.parse(yaml_source)) + + +@pytest.mark.parametrize("marker", ["", "++", " ", "\t", "\n", "\x00"]) +def test_invalid_marker(marker): + with pytest.raises(ValueError, match="single non-whitespace character"): + MarkdownIt().use(front_matter_plugin, marker=marker)