fix: drop zero-length text tokens in fragments_join

markdown_it/rules_inline/fragments_join.py

@@ -52,3 +52,12 @@ def fragments_join(state: StateInline) -> None:
 
     if curr != last:
         del state.tokens[last:]
+
+    # Drop zero-length text tokens.  Emphasis post-processing merges a ``**``
+    # delimiter run into a single ``strong`` token pair and blanks out the
+    # content of the two "inner" marker tokens instead of removing them.  They
+    # render as nothing, but would surface as empty ``text`` nodes in
+    # ``SyntaxTreeNode`` (see executablebooks/markdown-it-py#374).
+    state.tokens[:] = [
+        token for token in state.tokens if token.type != "text" or token.content
+    ]

tests/test_port/test_misc.py

@@ -1,4 +1,5 @@
 from markdown_it import MarkdownIt, presets
+from markdown_it.tree import SyntaxTreeNode
 
 
 def test_highlight_arguments():
@@ -42,3 +43,37 @@ def type_filter(tokens, type_):
     assert tokens[2].markup == "."
     assert tokens[3].info == "199"
     assert tokens[3].markup == "."
+
+
+def test_no_empty_text_tokens_after_nested_emphasis():
+    # Regression test for #374.
+    #
+    # `emphasis.postProcess` only turns the *first* marker of a `**` pair into
+    # `strong_open` (and the *second* of the closing pair into `strong_close`),
+    # blanking the content of the duplicate markers while leaving them as
+    # `text` tokens. `fragments_join` folds most of that residue into adjacent
+    # text, but a residue with no text neighbour survives: in `_li**ne**_` the
+    # closing residue sits between `strong_close` and `em_close`, and in
+    # `a **b**` the residue sits at the end of the token stream. The syntax
+    # tree then exposes such residue as an extra empty `<text>` node.
+    md = MarkdownIt("gfm-like").disable(["linkify"])
+    sources = (
+        "new _li**ne**_",
+        "new _l**i**ne_",
+        "new _**line**_",
+        "new _li**ne**_\na",
+    )
+    for source in sources:
+        tokens = md.parse(source)
+        [inline] = [t for t in tokens if t.type == "inline"]
+        empties = [
+            c for c in (inline.children or []) if c.type == "text" and not c.content
+        ]
+        assert empties == [], f"empty text tokens left in stream for {source!r}"
+
+        empty_tree_nodes = [
+            node
+            for node in SyntaxTreeNode(tokens).walk(include_self=False)
+            if node.type == "text" and not node.content
+        ]
+        assert empty_tree_nodes == [], f"empty text nodes in tree for {source!r}"

tests/test_tree.py

@@ -83,6 +83,30 @@ def test_pretty_text_special(file_regression):
     file_regression.check(tree.pretty(show_text=True), extension=".xml")
 
 
+def test_no_empty_text_node():
+    """Regression test for https://github.com/executablebooks/markdown-it-py/issues/374
+
+    Emphasis post-processing turns a ``**`` delimiter run into a single
+    ``strong`` token pair, leaving the two "inner" marker tokens behind with
+    empty content.  These zero-length text tokens must not show up as empty
+    ``text`` nodes in the syntax tree.
+    """
+    tokens = MarkdownIt("commonmark").parse("new _li**ne**_\na")
+    tree = SyntaxTreeNode(tokens)
+    inline = tree[0].children[0]
+    assert [child.type for child in inline.children] == [
+        "text",
+        "em",
+        "softbreak",
+        "text",
+    ]
+    assert [child.type for child in inline[1].children] == ["text", "strong"]
+    # no empty text nodes anywhere in the tree
+    assert all(node.content for node in tree.walk() if node.type == "text")
+    # dropping the empty tokens must not break token round-tripping
+    assert tree.to_tokens() == tokens
+
+
 def test_walk():
     tokens = MarkdownIt().parse(EXAMPLE_MARKDOWN)
     tree = SyntaxTreeNode(tokens)

tests/test_tree/test_pretty.xml

@@ -19,7 +19,6 @@
           <strong>
             <text>
               list
-          <text>
   <blockquote>
     <paragraph>
       <inline>