diff --git a/markdown_it/rules_inline/fragments_join.py b/markdown_it/rules_inline/fragments_join.py index 5eb88a14..c5022695 100644 --- a/markdown_it/rules_inline/fragments_join.py +++ b/markdown_it/rules_inline/fragments_join.py @@ -52,3 +52,12 @@ def fragments_join(state: StateInline) -> None: if curr != last: del state.tokens[last:] + + # Drop zero-length text tokens. Emphasis post-processing merges a ``**`` + # delimiter run into a single ``strong`` token pair and blanks out the + # content of the two "inner" marker tokens instead of removing them. They + # render as nothing, but would surface as empty ``text`` nodes in + # ``SyntaxTreeNode`` (see executablebooks/markdown-it-py#374). + state.tokens[:] = [ + token for token in state.tokens if token.type != "text" or token.content + ] diff --git a/tests/test_port/test_misc.py b/tests/test_port/test_misc.py index 62b5bf85..a55f7567 100644 --- a/tests/test_port/test_misc.py +++ b/tests/test_port/test_misc.py @@ -1,4 +1,5 @@ from markdown_it import MarkdownIt, presets +from markdown_it.tree import SyntaxTreeNode def test_highlight_arguments(): @@ -42,3 +43,37 @@ def type_filter(tokens, type_): assert tokens[2].markup == "." assert tokens[3].info == "199" assert tokens[3].markup == "." + + +def test_no_empty_text_tokens_after_nested_emphasis(): + # Regression test for #374. + # + # `emphasis.postProcess` only turns the *first* marker of a `**` pair into + # `strong_open` (and the *second* of the closing pair into `strong_close`), + # blanking the content of the duplicate markers while leaving them as + # `text` tokens. `fragments_join` folds most of that residue into adjacent + # text, but a residue with no text neighbour survives: in `_li**ne**_` the + # closing residue sits between `strong_close` and `em_close`, and in + # `a **b**` the residue sits at the end of the token stream. The syntax + # tree then exposes such residue as an extra empty `` node. + md = MarkdownIt("gfm-like").disable(["linkify"]) + sources = ( + "new _li**ne**_", + "new _l**i**ne_", + "new _**line**_", + "new _li**ne**_\na", + ) + for source in sources: + tokens = md.parse(source) + [inline] = [t for t in tokens if t.type == "inline"] + empties = [ + c for c in (inline.children or []) if c.type == "text" and not c.content + ] + assert empties == [], f"empty text tokens left in stream for {source!r}" + + empty_tree_nodes = [ + node + for node in SyntaxTreeNode(tokens).walk(include_self=False) + if node.type == "text" and not node.content + ] + assert empty_tree_nodes == [], f"empty text nodes in tree for {source!r}" diff --git a/tests/test_tree.py b/tests/test_tree.py index 36bd0b67..cc37ea13 100644 --- a/tests/test_tree.py +++ b/tests/test_tree.py @@ -83,6 +83,30 @@ def test_pretty_text_special(file_regression): file_regression.check(tree.pretty(show_text=True), extension=".xml") +def test_no_empty_text_node(): + """Regression test for https://github.com/executablebooks/markdown-it-py/issues/374 + + Emphasis post-processing turns a ``**`` delimiter run into a single + ``strong`` token pair, leaving the two "inner" marker tokens behind with + empty content. These zero-length text tokens must not show up as empty + ``text`` nodes in the syntax tree. + """ + tokens = MarkdownIt("commonmark").parse("new _li**ne**_\na") + tree = SyntaxTreeNode(tokens) + inline = tree[0].children[0] + assert [child.type for child in inline.children] == [ + "text", + "em", + "softbreak", + "text", + ] + assert [child.type for child in inline[1].children] == ["text", "strong"] + # no empty text nodes anywhere in the tree + assert all(node.content for node in tree.walk() if node.type == "text") + # dropping the empty tokens must not break token round-tripping + assert tree.to_tokens() == tokens + + def test_walk(): tokens = MarkdownIt().parse(EXAMPLE_MARKDOWN) tree = SyntaxTreeNode(tokens) diff --git a/tests/test_tree/test_pretty.xml b/tests/test_tree/test_pretty.xml index 41d399ef..ce223d30 100644 --- a/tests/test_tree/test_pretty.xml +++ b/tests/test_tree/test_pretty.xml @@ -19,7 +19,6 @@ list -