diff --git a/markdown_it/common/utils.py b/markdown_it/common/utils.py index 11bda644..07f89b0b 100644 --- a/markdown_it/common/utils.py +++ b/markdown_it/common/utils.py @@ -191,6 +191,27 @@ def isWhiteSpace(code: int) -> bool: return code in MD_WHITESPACE +#: The characters ``String.prototype.trim`` removes in JavaScript, which is +#: what upstream markdown-it strips, minus U+FEFF. +#: +#: ``str.strip()`` without an argument uses :py:meth:`str.isspace`, which also +#: removes U+001C, U+001D, U+001E, U+001F and U+0085. Those are not whitespace +#: in CommonMark and are not removed by ``trim``, so relying on it drops them +#: from the output and makes two different reference labels compare equal. +#: +#: U+FEFF is deliberately excluded: ``trim`` does remove it, and that has the +#: very label folding effect this constant exists to avoid. +MD_TRIM_CHARS = "".join( + chr(code) + for code in sorted(MD_WHITESPACE | set(range(0x2000, 0x200B)) | {0x2028, 0x2029}) +) + + +def mdTrim(string: str) -> str: + """Strip leading and trailing whitespace, using the CommonMark set.""" + return string.strip(MD_TRIM_CHARS) + + # ////////////////////////////////////////////////////////////////////////////// @@ -254,7 +275,7 @@ def normalizeReference(string: str) -> str: """Helper to unify [reference labels].""" # Trim and collapse whitespace # - string = re.sub(r"\s+", " ", string.strip()) + string = re.sub("[" + re.escape(MD_TRIM_CHARS) + "]+", " ", mdTrim(string)) # In node v10 'ẞ'.toLowerCase() === 'Ṿ', which is presumed to be a bug # fixed in v12 (couldn't find any details). diff --git a/markdown_it/renderer.py b/markdown_it/renderer.py index f690b091..397efbe1 100644 --- a/markdown_it/renderer.py +++ b/markdown_it/renderer.py @@ -10,9 +10,10 @@ class Renderer from collections.abc import Sequence import inspect +import re from typing import Any, ClassVar, Protocol -from .common.utils import escapeHtml, unescapeAll +from .common.utils import MD_TRIM_CHARS, escapeHtml, mdTrim, unescapeAll from .token import Token from .utils import EnvType, OptionsDict @@ -266,12 +267,14 @@ def fence( env: EnvType, ) -> str: token = tokens[idx] - info = unescapeAll(token.info).strip() if token.info else "" + info = mdTrim(unescapeAll(token.info)) if token.info else "" langName = "" langAttrs = "" if info: - arr = info.split(maxsplit=1) + # Not ``str.split()``: it splits on the Python whitespace set, + # which is wider than the one upstream uses here. + arr = re.split("[" + re.escape(MD_TRIM_CHARS) + "]+", info, maxsplit=1) langName = arr[0] if len(arr) == 2: langAttrs = arr[1] diff --git a/markdown_it/rules_block/heading.py b/markdown_it/rules_block/heading.py index afcf9ed4..e768151a 100644 --- a/markdown_it/rules_block/heading.py +++ b/markdown_it/rules_block/heading.py @@ -4,7 +4,7 @@ import logging -from ..common.utils import isStrSpace +from ..common.utils import isStrSpace, mdTrim from .state_block import StateBlock LOGGER = logging.getLogger(__name__) @@ -59,7 +59,7 @@ def heading(state: StateBlock, startLine: int, endLine: int, silent: bool) -> bo token.map = [startLine, state.line] token = state.push("inline", "", 0) - token.content = state.src[pos:maximum].strip() + token.content = mdTrim(state.src[pos:maximum]) token.map = [startLine, state.line] token.children = [] diff --git a/markdown_it/rules_block/lheading.py b/markdown_it/rules_block/lheading.py index 3522207a..57138039 100644 --- a/markdown_it/rules_block/lheading.py +++ b/markdown_it/rules_block/lheading.py @@ -1,6 +1,7 @@ # lheading (---, ==) import logging +from ..common.utils import mdTrim from .state_block import StateBlock LOGGER = logging.getLogger(__name__) @@ -65,7 +66,7 @@ def lheading(state: StateBlock, startLine: int, endLine: int, silent: bool) -> b # Didn't find valid underline return False - content = state.getLines(startLine, nextLine, state.blkIndent, False).strip() + content = mdTrim(state.getLines(startLine, nextLine, state.blkIndent, False)) state.line = nextLine + 1 diff --git a/markdown_it/rules_block/paragraph.py b/markdown_it/rules_block/paragraph.py index 30ba8777..d27b07b1 100644 --- a/markdown_it/rules_block/paragraph.py +++ b/markdown_it/rules_block/paragraph.py @@ -2,6 +2,7 @@ import logging +from ..common.utils import mdTrim from .state_block import StateBlock LOGGER = logging.getLogger(__name__) @@ -47,7 +48,7 @@ def paragraph(state: StateBlock, startLine: int, endLine: int, silent: bool) -> nextLine += 1 - content = state.getLines(startLine, nextLine, state.blkIndent, False).strip() + content = mdTrim(state.getLines(startLine, nextLine, state.blkIndent, False)) state.line = nextLine diff --git a/markdown_it/rules_block/table.py b/markdown_it/rules_block/table.py index c52553d8..c07e52c2 100644 --- a/markdown_it/rules_block/table.py +++ b/markdown_it/rules_block/table.py @@ -3,7 +3,7 @@ import re -from ..common.utils import charStrAt, isStrSpace +from ..common.utils import charStrAt, isStrSpace, mdTrim from .state_block import StateBlock headerLineRe = re.compile(r"^:?-+:?$") @@ -108,7 +108,7 @@ def table(state: StateBlock, startLine: int, endLine: int, silent: bool) -> bool columns = lineText.split("|") aligns = [] for i in range(len(columns)): - t = columns[i].strip() + t = mdTrim(columns[i]) if not t: # allow empty columns before and after table, but not in between columns; # e.g. allow ` |---| `, disallow ` ---||--- ` @@ -126,7 +126,7 @@ def table(state: StateBlock, startLine: int, endLine: int, silent: bool) -> bool else: aligns.append("") - lineText = getLine(state, startLine).strip() + lineText = mdTrim(getLine(state, startLine)) if "|" not in lineText: return False if state.is_code_block(startLine): @@ -171,7 +171,7 @@ def table(state: StateBlock, startLine: int, endLine: int, silent: bool) -> bool # note in markdown-it this map was removed in v12.0.0 however, we keep it, # since it is helpful to propagate to children tokens token.map = [startLine, startLine + 1] - token.content = columns[i].strip() + token.content = mdTrim(columns[i]) token.children = [] token = state.push("th_close", "th", -1) @@ -193,7 +193,7 @@ def table(state: StateBlock, startLine: int, endLine: int, silent: bool) -> bool if terminate: break - lineText = getLine(state, nextLine).strip() + lineText = mdTrim(getLine(state, nextLine)) if not lineText: break if state.is_code_block(nextLine): @@ -227,7 +227,7 @@ def table(state: StateBlock, startLine: int, endLine: int, silent: bool) -> bool # since it is helpful to propagate to children tokens token.map = [nextLine, nextLine + 1] try: - token.content = columns[i].strip() if columns[i] else "" + token.content = mdTrim(columns[i]) if columns[i] else "" except IndexError: token.content = "" token.children = [] diff --git a/tests/test_port/test_whitespace.py b/tests/test_port/test_whitespace.py new file mode 100644 index 00000000..dee1b196 --- /dev/null +++ b/tests/test_port/test_whitespace.py @@ -0,0 +1,81 @@ +"""Whitespace handling must match upstream markdown-it, not Python's ``str``. + +``str.strip()`` and ``str.split()`` without arguments use :py:meth:`str.isspace`, +which additionally treats U+001C, U+001D, U+001E, U+001F and U+0085 as +whitespace. CommonMark and ``String.prototype.trim`` do not, so relying on them +drops those characters from the output and folds two distinct reference labels +into one. + +The expected values below are what ``markdown-it@14.1.0`` produces for the same +input. +""" + +import pytest + +from markdown_it import MarkdownIt + +#: Characters Python calls whitespace and CommonMark does not. +EXTRA = ["\x1c", "\x1d", "\x1e", "\x1f", "\x85"] + + +@pytest.mark.parametrize("char", EXTRA) +def test_reference_label_is_not_folded(char): + """A definition whose label differs must not supply a different usage. + + Before this was fixed, ``[ab]`` and ``[a b]`` normalized to the same + label, so the definition resolved a usage that does not name it. + """ + md = MarkdownIt("js-default") + assert md.render(f"[a b]\n\n[a{char}b]: http://example.com") == "

[a b]

\n" + assert ( + md.render(f"[a{char}b]\n\n[a b]: http://example.com") == f"

[a{char}b]

\n" + ) + + +def test_reference_label_still_collapses_real_whitespace(): + """Control: labels differing only in real whitespace still match.""" + md = MarkdownIt("js-default") + assert ( + md.render("[a b]\n\n[a\tb]: http://example.com") + == '

a b

\n' + ) + + +@pytest.mark.parametrize("char", EXTRA) +def test_character_survives_in_output(char): + """The character is content, so it has to reach the output.""" + md = MarkdownIt("js-default") + assert md.render(f"x{char}") == f"

x{char}

\n" + assert md.render(f"{char}x") == f"

{char}x

\n" + assert md.render(f"# h{char}") == f"

h{char}

\n" + assert md.render(f"h{char}\n===") == f"

h{char}

\n" + + +def test_real_whitespace_is_still_trimmed(): + """Control: what CommonMark does call whitespace is still removed.""" + md = MarkdownIt("js-default") + for char in ["\t", " ", "\xa0", " ", "　"]: + assert md.render(f"# h{char}") == "

h

\n", repr(char) + + +def test_fence_info_is_not_split_on_it(): + """``str.split()`` would split the info string on U+0085.""" + md = MarkdownIt("js-default") + assert ( + md.render("```py\x85rest\nx\n```") + == '
x\n
\n' + ) + + +def test_fence_info_still_splits_on_whitespace(): + """Control: a real space still separates language from attributes.""" + md = MarkdownIt("js-default") + assert ( + md.render("```py extra\nx\n```") + == '
x\n
\n' + ) + + +def test_table_cell_keeps_the_character(): + md = MarkdownIt("js-default").enable("table") + assert "c\x85" in md.render("|a|\n|---|\n|c\x85|")