👌 Copy edited tokens in SyntaxTreeNode.to_tokens instead of writing in place

docs/using.md

@@ -287,12 +287,13 @@ node[0].next_sibling
 ```
 
 Use `node.to_tokens()` to recover the linear token stream after modifying the
-tree's `children`, including those of inline and image nodes. This reuses the
-original token objects and updates their child lists to match the tree.
-The child lists are replaced in place, so the token stream the tree was built
-from sees the same edits. Another tree built from the same tokens keeps its own
-nodes, and its `to_tokens()` writes its own children back.
-Build the tree from `copy.deepcopy(tokens)` to avoid sharing.
+tree's `children`, including those of inline and image nodes. The tree's
+`children` are the source of truth: changes made directly to a token's
+`children` list do not appear in the output. Unchanged tokens are reused, and a
+token whose children were edited is returned as a copy, so `to_tokens()` does
+not modify the token stream the tree was built from. Nodes do share their
+`Token` objects with that stream, though, so setting a field such as
+`node.token.content` changes both.
 
 ## Renderers
 

markdown_it/tree.py

@@ -6,6 +6,7 @@
 from __future__ import annotations
 
 from collections.abc import Generator, Sequence
+import operator
 import textwrap
 from typing import TYPE_CHECKING, Any, NamedTuple, TypeVar, overload
 
@@ -93,28 +94,33 @@ def __getitem__(self, item: int | slice) -> Self | list[Self]:
     def to_tokens(self) -> list[Token]:
         """Recover the linear token stream, including changes to the tree.
 
-        The original tokens are reused, with their child lists updated to match
-        the tree.
+        The tree's ``children`` are the source of truth:
+        changes made directly to a token's ``children`` list
+        do not appear in the output.
 
-        The ``children`` of inline and image tokens are replaced in place.
-        These ``Token`` objects are shared with the token stream the tree was
-        built from, so that stream sees the edits too.
-        Another tree built from the same tokens keeps its own nodes,
-        and its ``to_tokens()`` writes its own children back.
-        Build the tree from ``copy.deepcopy(tokens)`` to avoid sharing.
+        Unchanged tokens are reused as they are.
+        A token whose children were edited, such as an inline or image token,
+        is returned as a shallow copy with the new children,
+        so this method does not modify the tokens the tree was built from.
+        Nodes do share their ``Token`` objects with that token stream, though,
+        so setting a field such as ``node.token.content`` changes both.
         """
 
         def recursive_collect_tokens(node: Self, token_list: list[Token]) -> None:
             if node.type == "root":
                 for child in node.children:
                     recursive_collect_tokens(child, token_list)
             elif node.token:
-                if node.token.children is not None:
+                token = node.token
+                if token.children is not None:
                     children: list[Token] = []
                     for child in node.children:
                         recursive_collect_tokens(child, children)
-                    node.token.children = children
-                token_list.append(node.token)
+                    if len(children) != len(token.children) or not all(
+                        map(operator.is_, children, token.children)
+                    ):
+                        token = token.copy(children=children)
+                token_list.append(token)
             else:
                 assert node.nester_tokens
                 token_list.append(node.nester_tokens.opening)

tests/test_tree.py

@@ -84,22 +84,170 @@ def test_tree_to_tokens_replaced_empty_inline_children():
     assert md.renderer.render(tree.to_tokens(), md.options, {}) == "<em>new</em>"
 
 
-def test_tree_to_tokens_updates_original_tokens():
-    """Inline child edits are written in place on the shared tokens.
+def test_tree_to_tokens_leaves_original_tokens_unchanged():
+    """Edits reach the output of ``to_tokens``, not the tokens the tree came from.
 
-    This pins the documented behaviour of ``to_tokens``: the token stream the
-    tree was built from sees the edit as well.
+    An inline token whose children were edited is copied, so the original stream
+    still renders the source, while unedited tokens are reused as they are.
     """
     md = MarkdownIt()
     tokens = md.parse("before *nested* after")
+    original_children = tokens[1].children
+    original_dicts = [token.as_dict() for token in tokens]
     tree = SyntaxTreeNode(tokens)
     tree[0][0].children.clear()
 
-    tree.to_tokens()
+    result = tree.to_tokens()
+
+    assert md.renderer.render(result, md.options, {}) == "<p></p>\n"
+    assert result[0] is tokens[0]
+    assert result[1] is not tokens[1]
+    assert result[1].children == []
+    assert result[2] is tokens[2]
+    assert tokens[1].children is original_children
+    assert [token.as_dict() for token in tokens] == original_dicts
+    assert (
+        md.renderer.render(tokens, md.options, {})
+        == "<p>before <em>nested</em> after</p>\n"
+    )
+
+
+def test_tree_to_tokens_output_is_stable():
+    """A returned token list is not changed by a later ``to_tokens`` call.
+
+    Two trees built from the same tokens are edited differently;
+    converting the second must not undo the edit in the first one's output.
+    """
+    md = MarkdownIt()
+    tokens = md.parse("before *nested* after")
+    first = SyntaxTreeNode(tokens)
+    first[0][0].children.pop()
+    result = first.to_tokens()
+
+    second = SyntaxTreeNode(tokens)
+    second[0][0].children.reverse()
+    second.to_tokens()
+    unedited = SyntaxTreeNode(tokens)
+    unedited.to_tokens()
+
+    expected = "<p>before <em>nested</em></p>\n"
+    assert md.renderer.render(result, md.options, {}) == expected
+    assert md.renderer.render(first.to_tokens(), md.options, {}) == expected
+
+
+def test_tree_to_tokens_unedited_reuses_tokens():
+    """An unedited round trip returns the original tokens and child lists.
+
+    Copies are only made for edited children, so nothing is copied or written
+    here, including the nested inline and image tokens.
+    """
+
+    def flatten(tokens):
+        for token in tokens:
+            yield token, token.children
+            if token.children is not None:
+                yield from flatten(token.children)
+
+    md = MarkdownIt()
+    tokens = md.parse("before *nested* after\n\n![alt *x*](image.png)\n")
+    before = list(flatten(tokens))
+
+    after = list(flatten(SyntaxTreeNode(tokens).to_tokens()))
+
+    assert len(after) == len(before)
+    for (token_before, children_before), (token_after, children_after) in zip(
+        before, after, strict=True
+    ):
+        assert token_after is token_before
+        assert children_after is children_before
+        assert token_before.children is children_before
+
+
+def test_tree_to_tokens_nested_image_edit():
+    """An edit inside an image nested in an inline token reaches the output.
+
+    The image token is copied, which changes the inline token's children,
+    so the inline token is copied too; the original tokens are untouched.
+    """
+    md = MarkdownIt()
+    tokens = md.parse("before ![alt *x*](image.png)")
+    inline_children = tokens[1].children
+    assert inline_children is not None
+    text, image = inline_children
+    assert image.type == "image"
+    tree = SyntaxTreeNode(tokens)
+    tree[0][0][1].children.reverse()
 
-    assert tokens[1].type == "inline"
-    assert tokens[1].children == []
-    assert md.renderer.render(tokens, md.options, {}) == "<p></p>\n"
+    result = tree.to_tokens()
+
+    assert (
+        md.renderer.render(result, md.options, {})
+        == '<p>before <img src="image.png" alt="xalt " /></p>\n'
+    )
+    assert result[1] is not tokens[1]
+    result_children = result[1].children
+    assert result_children is not None
+    assert result_children[0] is text
+    assert result_children[1] is not image
+    assert tokens[1].children is inline_children
+    assert inline_children == [text, image]
+    assert (
+        md.renderer.render(tokens, md.options, {})
+        == '<p>before <img src="image.png" alt="alt x" /></p>\n'
+    )
+
+
+def test_tree_to_tokens_replaced_child_with_equal_token():
+    """A child node replaced by one wrapping an equal but distinct token is kept.
+
+    Edits are detected by identity, not equality,
+    so the output holds the new token rather than the original one.
+    """
+    md = MarkdownIt()
+    tokens = md.parse("before *nested* after")
+    original_children = tokens[1].children
+    assert original_children is not None
+    original_text = original_children[0]
+    replacement = original_text.copy()
+    assert replacement == original_text
+    tree = SyntaxTreeNode(tokens)
+    tree[0][0].children[0] = SyntaxTreeNode([replacement], create_root=False)
+
+    result = tree.to_tokens()
+
+    assert result[1] is not tokens[1]
+    result_children = result[1].children
+    assert result_children is not None
+    assert result_children[0] is replacement
+    assert tokens[1].children is original_children
+    assert original_children[0] is original_text
+
+
+def test_tree_to_tokens_tree_children_are_source_of_truth():
+    """Changes made directly to a token's ``children`` do not reach the output.
+
+    The output follows the tree's ``children``;
+    the token stream the tree was built from keeps the direct change.
+    """
+    md = MarkdownIt()
+    tokens = md.parse("before *nested* after")
+    tree = SyntaxTreeNode(tokens)
+    inline_token = tree[0][0].token
+    assert inline_token is tokens[1]
+    assert inline_token.children is not None
+    appended = inline_token.children[-1].copy(content=" appended")
+    inline_token.children.append(appended)
+
+    result = tree.to_tokens()
+
+    assert (
+        md.renderer.render(result, md.options, {})
+        == "<p>before <em>nested</em> after</p>\n"
+    )
+    assert (
+        md.renderer.render(tokens, md.options, {})
+        == "<p>before <em>nested</em> after appended</p>\n"
+    )
 
 
 def test_property_passthrough():