From 77cfacd615fbff2bd800ee0b4d2332d50c497610 Mon Sep 17 00:00:00 2001 From: kevindurant735rocket-creator Date: Sun, 18 Oct 2026 14:44:19 +0800 Subject: [PATCH] fix: drop zero-length text tokens in fragments_join Emphasis post-processing merges a '**' delimiter run into a single strong token pair and blanks out the content of the two inner marker tokens instead of removing them. They render as nothing, but surfaced as empty text nodes in SyntaxTreeNode. Drop zero-length text tokens after the join so tree consumers never see empty text nodes. Fixes #374 --- markdown_it/rules_inline/fragments_join.py | 9 ++++++ tests/test_port/test_misc.py | 35 ++++++++++++++++++++++ tests/test_tree.py | 24 +++++++++++++++ tests/test_tree/test_pretty.xml | 1 - 4 files changed, 68 insertions(+), 1 deletion(-) diff --git a/markdown_it/rules_inline/fragments_join.py b/markdown_it/rules_inline/fragments_join.py index 5eb88a14..c5022695 100644 --- a/markdown_it/rules_inline/fragments_join.py +++ b/markdown_it/rules_inline/fragments_join.py @@ -52,3 +52,12 @@ def fragments_join(state: StateInline) -> None: if curr != last: del state.tokens[last:] + + # Drop zero-length text tokens. Emphasis post-processing merges a ``**`` + # delimiter run into a single ``strong`` token pair and blanks out the + # content of the two "inner" marker tokens instead of removing them. They + # render as nothing, but would surface as empty ``text`` nodes in + # ``SyntaxTreeNode`` (see executablebooks/markdown-it-py#374). + state.tokens[:] = [ + token for token in state.tokens if token.type != "text" or token.content + ] diff --git a/tests/test_port/test_misc.py b/tests/test_port/test_misc.py index 62b5bf85..a55f7567 100644 --- a/tests/test_port/test_misc.py +++ b/tests/test_port/test_misc.py @@ -1,4 +1,5 @@ from markdown_it import MarkdownIt, presets +from markdown_it.tree import SyntaxTreeNode def test_highlight_arguments(): @@ -42,3 +43,37 @@ def type_filter(tokens, type_): assert tokens[2].markup == "." assert tokens[3].info == "199" assert tokens[3].markup == "." + + +def test_no_empty_text_tokens_after_nested_emphasis(): + # Regression test for #374. + # + # `emphasis.postProcess` only turns the *first* marker of a `**` pair into + # `strong_open` (and the *second* of the closing pair into `strong_close`), + # blanking the content of the duplicate markers while leaving them as + # `text` tokens. `fragments_join` folds most of that residue into adjacent + # text, but a residue with no text neighbour survives: in `_li**ne**_` the + # closing residue sits between `strong_close` and `em_close`, and in + # `a **b**` the residue sits at the end of the token stream. The syntax + # tree then exposes such residue as an extra empty `` node. + md = MarkdownIt("gfm-like").disable(["linkify"]) + sources = ( + "new _li**ne**_", + "new _l**i**ne_", + "new _**line**_", + "new _li**ne**_\na", + ) + for source in sources: + tokens = md.parse(source) + [inline] = [t for t in tokens if t.type == "inline"] + empties = [ + c for c in (inline.children or []) if c.type == "text" and not c.content + ] + assert empties == [], f"empty text tokens left in stream for {source!r}" + + empty_tree_nodes = [ + node + for node in SyntaxTreeNode(tokens).walk(include_self=False) + if node.type == "text" and not node.content + ] + assert empty_tree_nodes == [], f"empty text nodes in tree for {source!r}" diff --git a/tests/test_tree.py b/tests/test_tree.py index 36bd0b67..cc37ea13 100644 --- a/tests/test_tree.py +++ b/tests/test_tree.py @@ -83,6 +83,30 @@ def test_pretty_text_special(file_regression): file_regression.check(tree.pretty(show_text=True), extension=".xml") +def test_no_empty_text_node(): + """Regression test for https://github.com/executablebooks/markdown-it-py/issues/374 + + Emphasis post-processing turns a ``**`` delimiter run into a single + ``strong`` token pair, leaving the two "inner" marker tokens behind with + empty content. These zero-length text tokens must not show up as empty + ``text`` nodes in the syntax tree. + """ + tokens = MarkdownIt("commonmark").parse("new _li**ne**_\na") + tree = SyntaxTreeNode(tokens) + inline = tree[0].children[0] + assert [child.type for child in inline.children] == [ + "text", + "em", + "softbreak", + "text", + ] + assert [child.type for child in inline[1].children] == ["text", "strong"] + # no empty text nodes anywhere in the tree + assert all(node.content for node in tree.walk() if node.type == "text") + # dropping the empty tokens must not break token round-tripping + assert tree.to_tokens() == tokens + + def test_walk(): tokens = MarkdownIt().parse(EXAMPLE_MARKDOWN) tree = SyntaxTreeNode(tokens) diff --git a/tests/test_tree/test_pretty.xml b/tests/test_tree/test_pretty.xml index 41d399ef..ce223d30 100644 --- a/tests/test_tree/test_pretty.xml +++ b/tests/test_tree/test_pretty.xml @@ -19,7 +19,6 @@ list -