From 39e11f1ee43fdc3e7c88cfc341d0b7f6b3eae662 Mon Sep 17 00:00:00 2001 From: Chris Sewell Date: Mon, 9 Nov 2026 23:11:55 +0100 Subject: [PATCH 1/2] =?UTF-8?q?=F0=9F=91=8C=20Copy=20edited=20tokens=20in?= =?UTF-8?q?=20SyntaxTreeNode.to=5Ftokens=20instead=20of=20writing=20in=20p?= =?UTF-8?q?lace?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit #453 made to_tokens() keep edits to the children of inline and image nodes by assigning the rebuilt child list to the shared Token objects. That also changed the token stream the tree was built from, and a list returned earlier could change again after another tree built from the same tokens called to_tokens(). A token is now replaced by token.copy(children=...) only when its rebuilt children differ by identity from its current ones. Edits still reach the output, the original tokens are left unchanged, and an unedited round trip still returns the same Token objects and child lists. A copied image token makes the enclosing inline token's children differ, so it is copied too. The test that pinned the in-place write is replaced by tests for each of these properties, and the docstring and docs/using.md no longer need the shared-state caveats or the deepcopy advice. --- docs/using.md | 9 ++-- markdown_it/tree.py | 23 +++++---- tests/test_tree.py | 111 ++++++++++++++++++++++++++++++++++++++++---- 3 files changed, 117 insertions(+), 26 deletions(-) diff --git a/docs/using.md b/docs/using.md index 816bd75c..efe2acd5 100644 --- a/docs/using.md +++ b/docs/using.md @@ -287,12 +287,9 @@ node[0].next_sibling ``` Use `node.to_tokens()` to recover the linear token stream after modifying the -tree's `children`, including those of inline and image nodes. This reuses the -original token objects and updates their child lists to match the tree. -The child lists are replaced in place, so the token stream the tree was built -from sees the same edits. Another tree built from the same tokens keeps its own -nodes, and its `to_tokens()` writes its own children back. -Build the tree from `copy.deepcopy(tokens)` to avoid sharing. +tree's `children`, including those of inline and image nodes. Unchanged tokens +are reused, and a token whose children were edited is returned as a copy, +so the token stream the tree was built from is not modified. ## Renderers diff --git a/markdown_it/tree.py b/markdown_it/tree.py index dab29ae1..8e147232 100644 --- a/markdown_it/tree.py +++ b/markdown_it/tree.py @@ -93,15 +93,10 @@ def __getitem__(self, item: int | slice) -> Self | list[Self]: def to_tokens(self) -> list[Token]: """Recover the linear token stream, including changes to the tree. - The original tokens are reused, with their child lists updated to match - the tree. - - The ``children`` of inline and image tokens are replaced in place. - These ``Token`` objects are shared with the token stream the tree was - built from, so that stream sees the edits too. - Another tree built from the same tokens keeps its own nodes, - and its ``to_tokens()`` writes its own children back. - Build the tree from ``copy.deepcopy(tokens)`` to avoid sharing. + Unchanged tokens are reused as they are. + A token whose children were edited, such as an inline or image token, + is returned as a shallow copy with the new children, + so the tokens the tree was built from are not modified. """ def recursive_collect_tokens(node: Self, token_list: list[Token]) -> None: @@ -109,12 +104,16 @@ def recursive_collect_tokens(node: Self, token_list: list[Token]) -> None: for child in node.children: recursive_collect_tokens(child, token_list) elif node.token: - if node.token.children is not None: + token = node.token + if token.children is not None: children: list[Token] = [] for child in node.children: recursive_collect_tokens(child, children) - node.token.children = children - token_list.append(node.token) + if len(children) != len(token.children) or any( + a is not b for a, b in zip(children, token.children) + ): + token = token.copy(children=children) + token_list.append(token) else: assert node.nester_tokens token_list.append(node.nester_tokens.opening) diff --git a/tests/test_tree.py b/tests/test_tree.py index b06a80ba..17196e86 100644 --- a/tests/test_tree.py +++ b/tests/test_tree.py @@ -84,22 +84,117 @@ def test_tree_to_tokens_replaced_empty_inline_children(): assert md.renderer.render(tree.to_tokens(), md.options, {}) == "new" -def test_tree_to_tokens_updates_original_tokens(): - """Inline child edits are written in place on the shared tokens. +def test_tree_to_tokens_leaves_original_tokens_unchanged(): + """Edits reach the output of ``to_tokens``, not the tokens the tree came from. - This pins the documented behaviour of ``to_tokens``: the token stream the - tree was built from sees the edit as well. + An inline token whose children were edited is copied, so the original stream + still renders the source, while unedited tokens are reused as they are. """ md = MarkdownIt() tokens = md.parse("before *nested* after") + original_children = tokens[1].children + original_dicts = [token.as_dict() for token in tokens] tree = SyntaxTreeNode(tokens) tree[0][0].children.clear() - tree.to_tokens() + result = tree.to_tokens() + + assert md.renderer.render(result, md.options, {}) == "

\n" + assert result[0] is tokens[0] + assert result[1] is not tokens[1] + assert result[1].children == [] + assert result[2] is tokens[2] + assert tokens[1].children is original_children + assert [token.as_dict() for token in tokens] == original_dicts + assert ( + md.renderer.render(tokens, md.options, {}) + == "

before nested after

\n" + ) + + +def test_tree_to_tokens_output_is_stable(): + """A returned token list is not changed by a later ``to_tokens`` call. + + Two trees built from the same tokens are edited differently; + converting the second must not undo the edit in the first one's output. + """ + md = MarkdownIt() + tokens = md.parse("before *nested* after") + first = SyntaxTreeNode(tokens) + first[0][0].children.pop() + result = first.to_tokens() + + second = SyntaxTreeNode(tokens) + second[0][0].children.reverse() + second.to_tokens() + unedited = SyntaxTreeNode(tokens) + unedited.to_tokens() + + expected = "

before nested

\n" + assert md.renderer.render(result, md.options, {}) == expected + assert md.renderer.render(first.to_tokens(), md.options, {}) == expected + + +def test_tree_to_tokens_unedited_reuses_tokens(): + """An unedited round trip returns the original tokens and child lists. + + Copies are only made for edited children, so nothing is copied or written + here, including the nested inline and image tokens. + """ + + def flatten(tokens): + for token in tokens: + yield token, token.children + if token.children is not None: + yield from flatten(token.children) - assert tokens[1].type == "inline" - assert tokens[1].children == [] - assert md.renderer.render(tokens, md.options, {}) == "

\n" + md = MarkdownIt() + tokens = md.parse("before *nested* after\n\n![alt *x*](image.png)\n") + before = list(flatten(tokens)) + + after = list(flatten(SyntaxTreeNode(tokens).to_tokens())) + + assert len(after) == len(before) + for (token_before, children_before), (token_after, children_after) in zip( + before, after, strict=True + ): + assert token_after is token_before + assert children_after is children_before + assert token_before.children is children_before + + +def test_tree_to_tokens_nested_image_edit(): + """An edit inside an image nested in an inline token reaches the output. + + The image token is copied, which changes the inline token's children, + so the inline token is copied too; the original tokens are untouched. + """ + md = MarkdownIt() + tokens = md.parse("before ![alt *x*](image.png)") + inline_children = tokens[1].children + assert inline_children is not None + text, image = inline_children + assert image.type == "image" + tree = SyntaxTreeNode(tokens) + tree[0][0][1].children.reverse() + + result = tree.to_tokens() + + assert ( + md.renderer.render(result, md.options, {}) + == '

before xalt

\n' + ) + assert result[1] is not tokens[1] + result_children = result[1].children + assert result_children is not None + assert result_children[0] is text + assert result_children[1] is not image + assert tokens[1].children is inline_children + assert inline_children == [text, image] + assert ( + md.renderer.render(tokens, md.options, {}) + == '

before alt x

\n' + ) def test_property_passthrough(): From 0338102037b892dee17f908f5b7a7a73ff4ab34d Mon Sep 17 00:00:00 2001 From: Chris Sewell Date: Mon, 9 Nov 2026 23:21:05 +0100 Subject: [PATCH 2/2] =?UTF-8?q?=F0=9F=93=9A=20Say=20what=20to=5Ftokens()?= =?UTF-8?q?=20reads=20and=20leaves=20alone,=20and=20pin=20the=20identity?= =?UTF-8?q?=20check?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The docstring and docs/using.md now say that the tree's children are the source of truth, so changes made directly to a token's children list do not appear in the output, and that to_tokens() itself does not modify the token stream, although nodes share their Token objects with it. The edit check uses all(map(operator.is_, ...)), which halves its overhead on an unedited round trip. A new test replaces a child with an equal but distinct token, which fails if the identity check becomes an equality check, and another pins that direct changes to a token's children do not reach the output. --- docs/using.md | 10 ++++++--- markdown_it/tree.py | 13 ++++++++--- tests/test_tree.py | 53 +++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 70 insertions(+), 6 deletions(-) diff --git a/docs/using.md b/docs/using.md index efe2acd5..fd242313 100644 --- a/docs/using.md +++ b/docs/using.md @@ -287,9 +287,13 @@ node[0].next_sibling ``` Use `node.to_tokens()` to recover the linear token stream after modifying the -tree's `children`, including those of inline and image nodes. Unchanged tokens -are reused, and a token whose children were edited is returned as a copy, -so the token stream the tree was built from is not modified. +tree's `children`, including those of inline and image nodes. The tree's +`children` are the source of truth: changes made directly to a token's +`children` list do not appear in the output. Unchanged tokens are reused, and a +token whose children were edited is returned as a copy, so `to_tokens()` does +not modify the token stream the tree was built from. Nodes do share their +`Token` objects with that stream, though, so setting a field such as +`node.token.content` changes both. ## Renderers diff --git a/markdown_it/tree.py b/markdown_it/tree.py index 8e147232..50957bc4 100644 --- a/markdown_it/tree.py +++ b/markdown_it/tree.py @@ -6,6 +6,7 @@ from __future__ import annotations from collections.abc import Generator, Sequence +import operator import textwrap from typing import TYPE_CHECKING, Any, NamedTuple, TypeVar, overload @@ -93,10 +94,16 @@ def __getitem__(self, item: int | slice) -> Self | list[Self]: def to_tokens(self) -> list[Token]: """Recover the linear token stream, including changes to the tree. + The tree's ``children`` are the source of truth: + changes made directly to a token's ``children`` list + do not appear in the output. + Unchanged tokens are reused as they are. A token whose children were edited, such as an inline or image token, is returned as a shallow copy with the new children, - so the tokens the tree was built from are not modified. + so this method does not modify the tokens the tree was built from. + Nodes do share their ``Token`` objects with that token stream, though, + so setting a field such as ``node.token.content`` changes both. """ def recursive_collect_tokens(node: Self, token_list: list[Token]) -> None: @@ -109,8 +116,8 @@ def recursive_collect_tokens(node: Self, token_list: list[Token]) -> None: children: list[Token] = [] for child in node.children: recursive_collect_tokens(child, children) - if len(children) != len(token.children) or any( - a is not b for a, b in zip(children, token.children) + if len(children) != len(token.children) or not all( + map(operator.is_, children, token.children) ): token = token.copy(children=children) token_list.append(token) diff --git a/tests/test_tree.py b/tests/test_tree.py index 17196e86..347adc67 100644 --- a/tests/test_tree.py +++ b/tests/test_tree.py @@ -197,6 +197,59 @@ def test_tree_to_tokens_nested_image_edit(): ) +def test_tree_to_tokens_replaced_child_with_equal_token(): + """A child node replaced by one wrapping an equal but distinct token is kept. + + Edits are detected by identity, not equality, + so the output holds the new token rather than the original one. + """ + md = MarkdownIt() + tokens = md.parse("before *nested* after") + original_children = tokens[1].children + assert original_children is not None + original_text = original_children[0] + replacement = original_text.copy() + assert replacement == original_text + tree = SyntaxTreeNode(tokens) + tree[0][0].children[0] = SyntaxTreeNode([replacement], create_root=False) + + result = tree.to_tokens() + + assert result[1] is not tokens[1] + result_children = result[1].children + assert result_children is not None + assert result_children[0] is replacement + assert tokens[1].children is original_children + assert original_children[0] is original_text + + +def test_tree_to_tokens_tree_children_are_source_of_truth(): + """Changes made directly to a token's ``children`` do not reach the output. + + The output follows the tree's ``children``; + the token stream the tree was built from keeps the direct change. + """ + md = MarkdownIt() + tokens = md.parse("before *nested* after") + tree = SyntaxTreeNode(tokens) + inline_token = tree[0][0].token + assert inline_token is tokens[1] + assert inline_token.children is not None + appended = inline_token.children[-1].copy(content=" appended") + inline_token.children.append(appended) + + result = tree.to_tokens() + + assert ( + md.renderer.render(result, md.options, {}) + == "

before nested after

\n" + ) + assert ( + md.renderer.render(tokens, md.options, {}) + == "

before nested after appended

\n" + ) + + def test_property_passthrough(): tokens = MarkdownIt().parse(EXAMPLE_MARKDOWN) heading_open = tokens[0]