From 07d3a4f7317ed95a5ffad1e5da72c88fc9100b15 Mon Sep 17 00:00:00 2001 From: Gonghan-Princess <268298391+Gonghan-Princess@users.noreply.github.com> Date: Sun, 13 Sep 2026 19:26:45 +0800 Subject: [PATCH 1/9] =?UTF-8?q?=E2=9C=A8=20NEW:=20Configure=20front=20matt?= =?UTF-8?q?er=20delimiter=20characters?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGELOG.md | 4 ++ mdit_py_plugins/front_matter/index.py | 42 ++++++++++++--- tests/test_front_matter.py | 78 +++++++++++++++++++++++++++ 3 files changed, 118 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a1b78b3..a649e9a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Change Log +## Unreleased + +- ✨ NEW: Allow configuring the front matter delimiter character with `marker` (#134). + ## 0.7.0 - 2026-07-19 - ✨ NEW: Add section reference plugin (`section_ref`) (#144) diff --git a/mdit_py_plugins/front_matter/index.py b/mdit_py_plugins/front_matter/index.py index 1551d76..a785d5f 100644 --- a/mdit_py_plugins/front_matter/index.py +++ b/mdit_py_plugins/front_matter/index.py @@ -1,16 +1,18 @@ """Process front matter.""" +from functools import partial + from markdown_it import MarkdownIt from markdown_it.rules_block import StateBlock from mdit_py_plugins.utils import is_code_block -def front_matter_plugin(md: MarkdownIt) -> None: +def front_matter_plugin(md: MarkdownIt, *, marker: str = "-") -> None: """Plugin ported from `markdown-it-front-matter `__. - It parses initial metadata, stored between opening/closing dashes: + It parses initial metadata, stored between opening/closing markers: .. code-block:: md @@ -18,19 +20,47 @@ def front_matter_plugin(md: MarkdownIt) -> None: valid-front-matter: true --- + For example, to extract TOML-style front matter: + + .. code-block:: python + + md = MarkdownIt().use(front_matter_plugin, marker="+") + tokens = md.parse('+++\\ntitle = "Hello"\\n+++\\n# Heading') + assert tokens[0].content == 'title = "Hello"' + + :param marker: Single non-whitespace character used for the delimiters, + excluding NUL (which Markdown normalizes before parsing). + At least three repetitions are required; the closing delimiter must + be at least as long as the opening delimiter. Defaults to ``-`` for + YAML-style front matter. Use ``+`` for TOML-style ``+++`` delimiters + or ``;`` for JSON-style ``;;;`` delimiters. The content is returned + as raw text, without decoding YAML, TOML, or JSON. The YAML ``...`` + terminator is recognized only with the default marker. + :raises ValueError: If the marker is not a single non-whitespace character, + or is NUL. + """ + if len(marker) != 1 or marker.isspace() or marker == "\x00": + raise ValueError( + "marker must be a single non-whitespace character other than NUL" + ) + md.block.ruler.before( "table", "front_matter", - _front_matter_rule, + partial(_front_matter_rule, marker_chr=marker), {"alt": ["paragraph", "reference", "blockquote", "list"]}, ) def _front_matter_rule( - state: StateBlock, startLine: int, endLine: int, silent: bool + state: StateBlock, + startLine: int, + endLine: int, + silent: bool, + *, + marker_chr: str = "-", ) -> bool: - marker_chr = "-" min_markers = 3 auto_closed = False @@ -69,7 +99,7 @@ def _front_matter_rule( # unclosed block should be autoclosed by end of document. return False - if state.src[start:maximum] == "...": + if marker_chr == "-" and state.src[start:maximum] == "...": break start = state.bMarks[nextLine] + state.tShift[nextLine] diff --git a/tests/test_front_matter.py b/tests/test_front_matter.py index 627cb89..79c0843 100644 --- a/tests/test_front_matter.py +++ b/tests/test_front_matter.py @@ -47,3 +47,81 @@ def test_short_source(): # The code should not raise an IndexError. assert md.parse("-") + + +@pytest.mark.parametrize( + "marker,content", [("+", 'title = "Hello"'), (";", '{"title": "Hello"}')] +) +def test_custom_marker(marker, content): + md = MarkdownIt("commonmark").use(front_matter_plugin, marker=marker) + source = f"{marker * 3}\n{content}\n{marker * 3}\n# Head" + tokens = md.parse(source) + + assert tokens[0].type == "front_matter" + assert tokens[0].content == content + assert tokens[0].markup == marker * 3 + assert tokens[0].map == [0, 3] + assert tokens[0].hidden + assert tokens[0].block + assert md.render(source) == "\n

Head

\n" + + +@pytest.mark.parametrize( + "opening,closing", [("+++", "+++"), ("++++", "+++++"), ("+++ ", "+++\t ")] +) +def test_custom_marker_empty_metadata(opening, closing): + md = MarkdownIt().use(front_matter_plugin, marker="+") + tokens = md.parse(f"{opening}\n{closing}") + assert len(tokens) == 1 + assert tokens[0].type == "front_matter" + assert tokens[0].content == "" + assert tokens[0].map == [0, 2] + + +@pytest.mark.parametrize( + "source", + [ + "", + "+", + "++", + "++\na: 1\n++", + "+++\na: 1", + "+++\na: 1\n---", + "++++\na: 1\n+++", + "+++\na: 1\n+++ text", + " +++\na: 1\n+++", + "\n+++\na: 1\n+++", + "# Head\n+++\na: 1\n+++", + ], +) +def test_custom_marker_not_front_matter(source): + md = MarkdownIt().use(front_matter_plugin, marker="+") + assert all(token.type != "front_matter" for token in md.parse(source)) + + +def test_custom_marker_keeps_yaml_terminator_as_content(): + md = MarkdownIt().use(front_matter_plugin, marker="+") + content = 'text = """\n...\n"""' + source = f"+++\n{content}\n+++\n# Head" + tokens = md.parse(source) + assert tokens[0].content == content + assert tokens[0].map == [0, 5] + assert md.render(source) == "\n

Head

\n" + + +def test_marker_configuration_is_per_parser(): + default = MarkdownIt().use(front_matter_plugin) + custom = MarkdownIt().use(front_matter_plugin, marker="+") + yaml_source = "---\nx: 1\n---" + toml_source = "+++\nx = 1\n+++" + + assert default.parse(yaml_source)[0].type == "front_matter" + assert custom.parse(toml_source)[0].type == "front_matter" + assert all(token.type != "front_matter" for token in default.parse(toml_source)) + assert all(token.type != "front_matter" for token in custom.parse(yaml_source)) + + +@pytest.mark.parametrize("marker", ["", "++", " ", "\t", "\n", "\x00"]) +def test_invalid_marker(marker): + with pytest.raises(ValueError, match="single non-whitespace character"): + MarkdownIt().use(front_matter_plugin, marker=marker) From 107d69c7cb116205731dcf53136b174f587e7b34 Mon Sep 17 00:00:00 2001 From: feiiiiii5 <204683769+feiiiiii5@users.noreply.github.com> Date: Fri, 2 Oct 2026 11:06:29 +0800 Subject: [PATCH 2/9] =?UTF-8?q?=F0=9F=90=9B=20FIX:=20parse=20front=20matte?= =?UTF-8?q?r=20closed=20by=20the=20YAML=20`...`=20marker?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The `...` document-end marker was tested against the *previous* line, because `start`/`maximum` are only moved onto the current line after the test. Two things followed, both silent: A block whose last line was `...` was dropped entirely, so the front matter leaked into the document as visible text: MarkdownIt().use(front_matter_plugin).render("---\ntitle: a\n...\n") before '
\n

title: a\n...

\n' after '' And where it did parse, the marker stayed in the token content, so `content` was `'title: a\n...'` rather than `'title: a'`, unlike the dashed form. Move the test to after the per-line refresh and mark the line consumed. `auto_closed` is only read at `state.line = nextLine + (1 if auto_closed else 0)`, so setting it is what stops the marker being re-parsed as content. `test_all` could not catch either half: the front_matter token is hidden, so only the rendered output is asserted, and a dropped block renders as visible text rather than failing. The new tests assert the token itself. pytest tests -q 514 passed --- CHANGELOG.md | 16 ++++++++++++++++ mdit_py_plugins/front_matter/index.py | 11 ++++++++--- tests/test_front_matter.py | 24 ++++++++++++++++++++++++ 3 files changed, 48 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a1b78b3..324134e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,21 @@ # Change Log +## Unreleased + +- 🐛 FIX: front matter closed by `...` is parsed correctly + + The YAML document-end marker was tested against the previous line rather than + the current one, because `start`/`maximum` were only moved onto the current + line after the test. Two things followed: a block whose last line was `...` was + dropped entirely, so the front matter leaked into the document as visible text, + and where it did parse, the marker itself was included in the token content. + + ```python + MarkdownIt().use(front_matter_plugin).render("---\ntitle: a\n...\n") + # before: '
\n

title: a\n...

\n' + # after: '' + ``` + ## 0.7.0 - 2026-07-19 - ✨ NEW: Add section reference plugin (`section_ref`) (#144) diff --git a/mdit_py_plugins/front_matter/index.py b/mdit_py_plugins/front_matter/index.py index 1551d76..3e8ee9b 100644 --- a/mdit_py_plugins/front_matter/index.py +++ b/mdit_py_plugins/front_matter/index.py @@ -69,12 +69,17 @@ def _front_matter_rule( # unclosed block should be autoclosed by end of document. return False - if state.src[start:maximum] == "...": - break - start = state.bMarks[nextLine] + state.tShift[nextLine] maximum = state.eMarks[nextLine] + if state.src[start:maximum] == "...": + # The YAML document-end marker closes the block on this line, so it must + # be consumed here: checking it before start/maximum are moved onto the + # current line tests the previous one, which both missed a `...` that + # ends the document and leaked the marker into the extracted content. + auto_closed = True + break + if start < maximum and state.sCount[nextLine] < state.blkIndent: # non-empty line with negative indent should stop the list: # - ``` diff --git a/tests/test_front_matter.py b/tests/test_front_matter.py index 627cb89..8401e75 100644 --- a/tests/test_front_matter.py +++ b/tests/test_front_matter.py @@ -42,6 +42,30 @@ def test_token(): ] +@pytest.mark.parametrize("closer", ["---", "..."]) +def test_token_content_excludes_the_closer(closer): + """The closing marker must not be left inside the extracted content. + + ``...`` is the YAML document-end marker and is exercised by the fixture file, + so it has to be consumed like the dashed form rather than kept in the block. + """ + md = MarkdownIt("commonmark").use(front_matter_plugin) + tokens = md.parse(f"---\na: 1\n{closer}") + assert [token.type for token in tokens] == ["front_matter"] + assert tokens[0].content == "a: 1" + + +def test_dots_closer_on_the_last_line(): + """A block closed by ``...`` on the final line is still front matter. + + ``test_all`` cannot see this: the front_matter token is hidden, so only the + rendered output is asserted, and a block that is dropped rather than closed + renders as visible text instead of disappearing. + """ + md = MarkdownIt("commonmark").use(front_matter_plugin) + assert md.render("---\na: 1\n...\n") == md.render("---\na: 1\n---\n") == "" + + def test_short_source(): md = MarkdownIt("commonmark").use(front_matter_plugin) From 161f0f240e19cd558b9f130c1d3b4e65bd7a4952 Mon Sep 17 00:00:00 2001 From: Chris Sewell Date: Mon, 9 Nov 2026 20:25:11 +0100 Subject: [PATCH 3/9] =?UTF-8?q?=F0=9F=94=A7=20MAINTAIN:=20drop=20changelog?= =?UTF-8?q?=20entry=20(written=20at=20release)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGELOG.md | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 324134e..a1b78b3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,21 +1,5 @@ # Change Log -## Unreleased - -- 🐛 FIX: front matter closed by `...` is parsed correctly - - The YAML document-end marker was tested against the previous line rather than - the current one, because `start`/`maximum` were only moved onto the current - line after the test. Two things followed: a block whose last line was `...` was - dropped entirely, so the front matter leaked into the document as visible text, - and where it did parse, the marker itself was included in the token content. - - ```python - MarkdownIt().use(front_matter_plugin).render("---\ntitle: a\n...\n") - # before: '
\n

title: a\n...

\n' - # after: '' - ``` - ## 0.7.0 - 2026-07-19 - ✨ NEW: Add section reference plugin (`section_ref`) (#144) From 1f624d0bc7f42e11b42e0196f586eef8385afc8a Mon Sep 17 00:00:00 2001 From: Chris Sewell Date: Mon, 9 Nov 2026 20:26:03 +0100 Subject: [PATCH 4/9] =?UTF-8?q?=F0=9F=90=9B=20FIX:=20front=20matter=20`...?= =?UTF-8?q?`=20closer=20must=20be=20at=20column=200,=20trailing=20spaces?= =?UTF-8?q?=20allowed?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit YAML's document-end marker is column-0 only, so an indented `...` (for example inside a block scalar) no longer closes the front matter. Trailing spaces after it are accepted, as they already are after the `---` closer. --- mdit_py_plugins/front_matter/index.py | 12 +++++++----- tests/test_front_matter.py | 2 +- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/mdit_py_plugins/front_matter/index.py b/mdit_py_plugins/front_matter/index.py index 3e8ee9b..28a176e 100644 --- a/mdit_py_plugins/front_matter/index.py +++ b/mdit_py_plugins/front_matter/index.py @@ -72,11 +72,13 @@ def _front_matter_rule( start = state.bMarks[nextLine] + state.tShift[nextLine] maximum = state.eMarks[nextLine] - if state.src[start:maximum] == "...": - # The YAML document-end marker closes the block on this line, so it must - # be consumed here: checking it before start/maximum are moved onto the - # current line tests the previous one, which both missed a `...` that - # ends the document and leaked the marker into the extracted content. + if ( + state.sCount[nextLine] == 0 + and state.src.startswith("...", start) + and state.skipSpaces(start + 3) >= maximum + ): + # YAML document-end marker: `...` at column 0, trailing spaces allowed. + # It is checked on the current line and consumed, so it is not content. auto_closed = True break diff --git a/tests/test_front_matter.py b/tests/test_front_matter.py index 8401e75..00b714c 100644 --- a/tests/test_front_matter.py +++ b/tests/test_front_matter.py @@ -42,7 +42,7 @@ def test_token(): ] -@pytest.mark.parametrize("closer", ["---", "..."]) +@pytest.mark.parametrize("closer", ["---", "...", "... "]) def test_token_content_excludes_the_closer(closer): """The closing marker must not be left inside the extracted content. From 96fd4e5ea2283aaa698611e7b5ece19d42459919 Mon Sep 17 00:00:00 2001 From: Chris Sewell Date: Mon, 9 Nov 2026 20:26:03 +0100 Subject: [PATCH 5/9] =?UTF-8?q?=F0=9F=A7=AA=20TEST:=20front=20matter=20fix?= =?UTF-8?q?tures=20for=20the=20`...`=20closer?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tests/fixtures/front_matter.md | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/tests/fixtures/front_matter.md b/tests/fixtures/front_matter.md index e01fd4e..d1d5307 100644 --- a/tests/fixtures/front_matter.md +++ b/tests/fixtures/front_matter.md @@ -32,6 +32,37 @@ x: 1

Head

. +should parse triple dots at end of document: +. +--- +x: 1 +... +. +. + +should ignore spaces after triple dots: +. +--- +x: 1 +... +# Head +. + +

Head

+. + +should not close on indented triple dots: +. +--- +text: | + ... +--- +# Head +. + +

Head

+. + should parse front matter with indentation: . --- From cf169ce5c2d2dd4e83a4f0e78dd607a80111b958 Mon Sep 17 00:00:00 2001 From: Chris Sewell Date: Mon, 9 Nov 2026 20:26:43 +0100 Subject: [PATCH 6/9] =?UTF-8?q?=F0=9F=94=A7=20MAINTAIN:=20drop=20changelog?= =?UTF-8?q?=20entry=20(written=20at=20release)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGELOG.md | 4 ---- 1 file changed, 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a649e9a..a1b78b3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,9 +1,5 @@ # Change Log -## Unreleased - -- ✨ NEW: Allow configuring the front matter delimiter character with `marker` (#134). - ## 0.7.0 - 2026-07-19 - ✨ NEW: Add section reference plugin (`section_ref`) (#144) From 15b0cdaca8c44fdd8573b4e2576edac4b9cc2b4f Mon Sep 17 00:00:00 2001 From: Chris Sewell Date: Mon, 9 Nov 2026 20:27:15 +0100 Subject: [PATCH 7/9] =?UTF-8?q?=F0=9F=93=9A=20DOCS:=20front=20matter=20`ma?= =?UTF-8?q?rker`=20caveats?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Say that the plugin takes one marker per parser (a second `use` adds a second rule under the same name), and that the marker should not be a character that starts other block syntax, since a first line of three or more of it is taken as front matter. --- mdit_py_plugins/front_matter/index.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/mdit_py_plugins/front_matter/index.py b/mdit_py_plugins/front_matter/index.py index 206422d..9d0cfc0 100644 --- a/mdit_py_plugins/front_matter/index.py +++ b/mdit_py_plugins/front_matter/index.py @@ -36,6 +36,16 @@ def front_matter_plugin(md: MarkdownIt, *, marker: str = "-") -> None: or ``;`` for JSON-style ``;;;`` delimiters. The content is returned as raw text, without decoding YAML, TOML, or JSON. The YAML ``...`` terminator is recognized only with the default marker. + + The plugin is meant to be used once per parser, with one marker. + Calling ``use`` again with another marker adds a second rule with the + same ``front_matter`` name, which rule-name operations such as + ``md.disable("front_matter")`` do not reach. + + Choose a character that does not start other Markdown block syntax + (for example not ``#``, ``>``, ``~`` or a backtick): a document whose + first line starts with three or more of the marker is otherwise taken + as front matter. :raises ValueError: If the marker is not a single non-whitespace character, or is NUL. From f61fa23570c341a03198f1c1374d9f8b90ac8bf7 Mon Sep 17 00:00:00 2001 From: Chris Sewell Date: Mon, 9 Nov 2026 21:33:01 +0100 Subject: [PATCH 8/9] =?UTF-8?q?=F0=9F=A7=AA=20TEST:=20front=20matter=20`..?= =?UTF-8?q?.`=20must=20be=20alone=20on=20its=20line=20to=20close?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two fixtures pin the tail check on the closer: `....` and `...x` are content, so the block runs on to the following `---`. --- tests/fixtures/front_matter.md | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/tests/fixtures/front_matter.md b/tests/fixtures/front_matter.md index d1d5307..5cff35b 100644 --- a/tests/fixtures/front_matter.md +++ b/tests/fixtures/front_matter.md @@ -63,6 +63,30 @@ text: |

Head

. +should not close on four dots: +. +--- +x: 1 +.... +--- +# Head +. + +

Head

+. + +should not close on triple dots followed by text: +. +--- +x: 1 +...x +--- +# Head +. + +

Head

+. + should parse front matter with indentation: . --- From 725cc2564b86c2fcc8c9364b17f6717de465466a Mon Sep 17 00:00:00 2001 From: Chris Sewell Date: Mon, 9 Nov 2026 21:33:56 +0100 Subject: [PATCH 9/9] =?UTF-8?q?=F0=9F=93=9A=20DOCS:=20front=20matter=20`ma?= =?UTF-8?q?rker`=20caveat=20needs=20a=20closing=20line?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A first line of three or more of the marker is taken as front matter only when a later line closes it; an unclosed block parses as ordinary Markdown. --- mdit_py_plugins/front_matter/index.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mdit_py_plugins/front_matter/index.py b/mdit_py_plugins/front_matter/index.py index 9d0cfc0..e90eca0 100644 --- a/mdit_py_plugins/front_matter/index.py +++ b/mdit_py_plugins/front_matter/index.py @@ -45,7 +45,7 @@ def front_matter_plugin(md: MarkdownIt, *, marker: str = "-") -> None: Choose a character that does not start other Markdown block syntax (for example not ``#``, ``>``, ``~`` or a backtick): a document whose first line starts with three or more of the marker is otherwise taken - as front matter. + as front matter when a later line closes it. :raises ValueError: If the marker is not a single non-whitespace character, or is NUL.