diff --git a/docs/using.md b/docs/using.md
index 816bd75c..fd242313 100644
--- a/docs/using.md
+++ b/docs/using.md
@@ -287,12 +287,13 @@ node[0].next_sibling
```
Use `node.to_tokens()` to recover the linear token stream after modifying the
-tree's `children`, including those of inline and image nodes. This reuses the
-original token objects and updates their child lists to match the tree.
-The child lists are replaced in place, so the token stream the tree was built
-from sees the same edits. Another tree built from the same tokens keeps its own
-nodes, and its `to_tokens()` writes its own children back.
-Build the tree from `copy.deepcopy(tokens)` to avoid sharing.
+tree's `children`, including those of inline and image nodes. The tree's
+`children` are the source of truth: changes made directly to a token's
+`children` list do not appear in the output. Unchanged tokens are reused, and a
+token whose children were edited is returned as a copy, so `to_tokens()` does
+not modify the token stream the tree was built from. Nodes do share their
+`Token` objects with that stream, though, so setting a field such as
+`node.token.content` changes both.
## Renderers
diff --git a/markdown_it/tree.py b/markdown_it/tree.py
index dab29ae1..50957bc4 100644
--- a/markdown_it/tree.py
+++ b/markdown_it/tree.py
@@ -6,6 +6,7 @@
from __future__ import annotations
from collections.abc import Generator, Sequence
+import operator
import textwrap
from typing import TYPE_CHECKING, Any, NamedTuple, TypeVar, overload
@@ -93,15 +94,16 @@ def __getitem__(self, item: int | slice) -> Self | list[Self]:
def to_tokens(self) -> list[Token]:
"""Recover the linear token stream, including changes to the tree.
- The original tokens are reused, with their child lists updated to match
- the tree.
+ The tree's ``children`` are the source of truth:
+ changes made directly to a token's ``children`` list
+ do not appear in the output.
- The ``children`` of inline and image tokens are replaced in place.
- These ``Token`` objects are shared with the token stream the tree was
- built from, so that stream sees the edits too.
- Another tree built from the same tokens keeps its own nodes,
- and its ``to_tokens()`` writes its own children back.
- Build the tree from ``copy.deepcopy(tokens)`` to avoid sharing.
+ Unchanged tokens are reused as they are.
+ A token whose children were edited, such as an inline or image token,
+ is returned as a shallow copy with the new children,
+ so this method does not modify the tokens the tree was built from.
+ Nodes do share their ``Token`` objects with that token stream, though,
+ so setting a field such as ``node.token.content`` changes both.
"""
def recursive_collect_tokens(node: Self, token_list: list[Token]) -> None:
@@ -109,12 +111,16 @@ def recursive_collect_tokens(node: Self, token_list: list[Token]) -> None:
for child in node.children:
recursive_collect_tokens(child, token_list)
elif node.token:
- if node.token.children is not None:
+ token = node.token
+ if token.children is not None:
children: list[Token] = []
for child in node.children:
recursive_collect_tokens(child, children)
- node.token.children = children
- token_list.append(node.token)
+ if len(children) != len(token.children) or not all(
+ map(operator.is_, children, token.children)
+ ):
+ token = token.copy(children=children)
+ token_list.append(token)
else:
assert node.nester_tokens
token_list.append(node.nester_tokens.opening)
diff --git a/tests/test_tree.py b/tests/test_tree.py
index b06a80ba..347adc67 100644
--- a/tests/test_tree.py
+++ b/tests/test_tree.py
@@ -84,22 +84,170 @@ def test_tree_to_tokens_replaced_empty_inline_children():
assert md.renderer.render(tree.to_tokens(), md.options, {}) == "new"
-def test_tree_to_tokens_updates_original_tokens():
- """Inline child edits are written in place on the shared tokens.
+def test_tree_to_tokens_leaves_original_tokens_unchanged():
+ """Edits reach the output of ``to_tokens``, not the tokens the tree came from.
- This pins the documented behaviour of ``to_tokens``: the token stream the
- tree was built from sees the edit as well.
+ An inline token whose children were edited is copied, so the original stream
+ still renders the source, while unedited tokens are reused as they are.
"""
md = MarkdownIt()
tokens = md.parse("before *nested* after")
+ original_children = tokens[1].children
+ original_dicts = [token.as_dict() for token in tokens]
tree = SyntaxTreeNode(tokens)
tree[0][0].children.clear()
- tree.to_tokens()
+ result = tree.to_tokens()
+
+ assert md.renderer.render(result, md.options, {}) == "\n"
+ assert result[0] is tokens[0]
+ assert result[1] is not tokens[1]
+ assert result[1].children == []
+ assert result[2] is tokens[2]
+ assert tokens[1].children is original_children
+ assert [token.as_dict() for token in tokens] == original_dicts
+ assert (
+ md.renderer.render(tokens, md.options, {})
+ == "before nested after
\n"
+ )
+
+
+def test_tree_to_tokens_output_is_stable():
+ """A returned token list is not changed by a later ``to_tokens`` call.
+
+ Two trees built from the same tokens are edited differently;
+ converting the second must not undo the edit in the first one's output.
+ """
+ md = MarkdownIt()
+ tokens = md.parse("before *nested* after")
+ first = SyntaxTreeNode(tokens)
+ first[0][0].children.pop()
+ result = first.to_tokens()
+
+ second = SyntaxTreeNode(tokens)
+ second[0][0].children.reverse()
+ second.to_tokens()
+ unedited = SyntaxTreeNode(tokens)
+ unedited.to_tokens()
+
+ expected = "before nested
\n"
+ assert md.renderer.render(result, md.options, {}) == expected
+ assert md.renderer.render(first.to_tokens(), md.options, {}) == expected
+
+
+def test_tree_to_tokens_unedited_reuses_tokens():
+ """An unedited round trip returns the original tokens and child lists.
+
+ Copies are only made for edited children, so nothing is copied or written
+ here, including the nested inline and image tokens.
+ """
+
+ def flatten(tokens):
+ for token in tokens:
+ yield token, token.children
+ if token.children is not None:
+ yield from flatten(token.children)
+
+ md = MarkdownIt()
+ tokens = md.parse("before *nested* after\n\n\n")
+ before = list(flatten(tokens))
+
+ after = list(flatten(SyntaxTreeNode(tokens).to_tokens()))
+
+ assert len(after) == len(before)
+ for (token_before, children_before), (token_after, children_after) in zip(
+ before, after, strict=True
+ ):
+ assert token_after is token_before
+ assert children_after is children_before
+ assert token_before.children is children_before
+
+
+def test_tree_to_tokens_nested_image_edit():
+ """An edit inside an image nested in an inline token reaches the output.
+
+ The image token is copied, which changes the inline token's children,
+ so the inline token is copied too; the original tokens are untouched.
+ """
+ md = MarkdownIt()
+ tokens = md.parse("before ")
+ inline_children = tokens[1].children
+ assert inline_children is not None
+ text, image = inline_children
+ assert image.type == "image"
+ tree = SyntaxTreeNode(tokens)
+ tree[0][0][1].children.reverse()
- assert tokens[1].type == "inline"
- assert tokens[1].children == []
- assert md.renderer.render(tokens, md.options, {}) == "\n"
+ result = tree.to_tokens()
+
+ assert (
+ md.renderer.render(result, md.options, {})
+ == 'before 
\n'
+ )
+ assert result[1] is not tokens[1]
+ result_children = result[1].children
+ assert result_children is not None
+ assert result_children[0] is text
+ assert result_children[1] is not image
+ assert tokens[1].children is inline_children
+ assert inline_children == [text, image]
+ assert (
+ md.renderer.render(tokens, md.options, {})
+ == 'before
\n'
+ )
+
+
+def test_tree_to_tokens_replaced_child_with_equal_token():
+ """A child node replaced by one wrapping an equal but distinct token is kept.
+
+ Edits are detected by identity, not equality,
+ so the output holds the new token rather than the original one.
+ """
+ md = MarkdownIt()
+ tokens = md.parse("before *nested* after")
+ original_children = tokens[1].children
+ assert original_children is not None
+ original_text = original_children[0]
+ replacement = original_text.copy()
+ assert replacement == original_text
+ tree = SyntaxTreeNode(tokens)
+ tree[0][0].children[0] = SyntaxTreeNode([replacement], create_root=False)
+
+ result = tree.to_tokens()
+
+ assert result[1] is not tokens[1]
+ result_children = result[1].children
+ assert result_children is not None
+ assert result_children[0] is replacement
+ assert tokens[1].children is original_children
+ assert original_children[0] is original_text
+
+
+def test_tree_to_tokens_tree_children_are_source_of_truth():
+ """Changes made directly to a token's ``children`` do not reach the output.
+
+ The output follows the tree's ``children``;
+ the token stream the tree was built from keeps the direct change.
+ """
+ md = MarkdownIt()
+ tokens = md.parse("before *nested* after")
+ tree = SyntaxTreeNode(tokens)
+ inline_token = tree[0][0].token
+ assert inline_token is tokens[1]
+ assert inline_token.children is not None
+ appended = inline_token.children[-1].copy(content=" appended")
+ inline_token.children.append(appended)
+
+ result = tree.to_tokens()
+
+ assert (
+ md.renderer.render(result, md.options, {})
+ == "before nested after
\n"
+ )
+ assert (
+ md.renderer.render(tokens, md.options, {})
+ == "before nested after appended
\n"
+ )
def test_property_passthrough():