From 71c98fab542e53c574a3e19caee46d6c81394ba5 Mon Sep 17 00:00:00 2001 From: Mark Scherer Date: Wed, 11 Nov 2026 13:25:41 +0100 Subject: [PATCH 1/2] Match djot.js on empty-label references, leftover attributes, trailing cell tabs and whitespace-only item code lines - A reference definition with an empty label (`[]: p`) in a quote leaves no open paragraph, so an unquoted line after it ends the quote. The container trackers required a non-empty label, unlike the definition parser. - Attributes an item leaves without a target (an empty item holding only `{...}`, or a trailing `{...}` after a blank line) go to the next item's `
  • `, as in djot.js. A blank line before the next item drops them. Before, they reached the first block inside the next item. - A footnote definition consumes the block attributes before it. Before, they passed through to the block after the note. - Only spaces are trimmed at the end of a table cell. A trailing tab is cell text, as djot.js keeps it, also after an escaped space. - A whitespace-only line in a code or raw block inside a list item keeps the spaces past the content column instead of becoming an empty line. --- src/Parser/Block/TableParser.php | 4 +- src/Parser/BlockParser.php | 28 ++++--- tests/TestCase/ConformanceFollowupsTest.php | 88 +++++++++++++++++++++ 3 files changed, 109 insertions(+), 11 deletions(-) create mode 100644 tests/TestCase/ConformanceFollowupsTest.php diff --git a/src/Parser/Block/TableParser.php b/src/Parser/Block/TableParser.php index 5671b7f..1087aa8 100644 --- a/src/Parser/Block/TableParser.php +++ b/src/Parser/Block/TableParser.php @@ -713,11 +713,13 @@ public function lineEndsWithPipeOutsideCodeSpan(string $line): bool /** * Trim a cell's content, keeping a space that a backslash escapes (`| C:\\ |`). * + * Like djot.js, only spaces are trimmed at the end; a trailing tab is cell text. + * * @param string $content */ public function trimCellContent(string $content): string { - $trimmed = rtrim($content, " \t"); + $trimmed = rtrim($content, ' '); $backslashes = strlen($trimmed) - strlen(rtrim($trimmed, '\\')); if ($backslashes % 2 === 1 && strlen($trimmed) < strlen($content)) { $trimmed .= $content[strlen($trimmed)]; diff --git a/src/Parser/BlockParser.php b/src/Parser/BlockParser.php index dd3e1a0..07a4f97 100644 --- a/src/Parser/BlockParser.php +++ b/src/Parser/BlockParser.php @@ -1670,7 +1670,7 @@ protected function startsHeadingBlock(string $line): bool return false; } if ( - preg_match('/^\[[^\]]+\]:(?:[ \t]+\S*)?[ \t]*$/', $line) + preg_match('/^\[[^\]]*\]:(?:[ \t]+\S*)?[ \t]*$/', $line) || $this->fencedBlockParser->parseCodeFenceOpener($line) !== null || $this->fencedBlockParser->parseDivFenceOpener($line) !== null || $this->listParser->parseListItemMarker($line) !== null @@ -1948,7 +1948,7 @@ private function trackBlockQuoteLine(string $line, array &$state, array &$contai $rest = substr($line, $pos); if ( $reference !== null && $pos > $reference['indent'] && $rest !== '' - && !preg_match('/^\[[^\]]+\]:(?=[ \t]|$)/', $rest) && !$this->startsNewBlock($rest) + && !preg_match('/^\[[^\]]*\]:(?=[ \t]|$)/', $rest) && !$this->startsNewBlock($rest) ) { $containers[] = $reference; @@ -1986,7 +1986,7 @@ private function trackBlockQuoteLine(string $line, array &$state, array &$contai return true; } - if (preg_match('/^\[[^\]]+\]:(?:[ \t]+\S*)?[ \t]*$/', $rest)) { + if (preg_match('/^\[[^\]]*\]:(?:[ \t]+\S*)?[ \t]*$/', $rest)) { // A footnote definition holds blocks of its own, and the list parser reads a // definition's indented lines its own way; leave both to the flat tracker. if (str_starts_with($rest, '[^') || in_array('item', array_column($containers, 'type'), true)) { @@ -2054,7 +2054,7 @@ private function opensBlock(string $line): bool return $this->isQuotePrefixAt($trimmed, 0) || preg_match('/^#{1,6}(?:[ \t]|$)/', $trimmed) === 1 || preg_match('/^(?:\*\[|\[\^)[^\]]+\]:(?=[ \t]|$)/', $trimmed) === 1 - || preg_match('/^\[[^\]]+\]:(?:[ \t]+\S*)?[ \t]*$/', $trimmed) === 1 + || preg_match('/^\[[^\]]*\]:(?:[ \t]+\S*)?[ \t]*$/', $trimmed) === 1 || $this->fencedBlockParser->parseCodeFenceOpener($trimmed) !== null || $this->fencedBlockParser->parseDivFenceOpener($trimmed) !== null || $this->leavesNoParagraph($trimmed) @@ -2547,7 +2547,7 @@ protected function tryParseList(Node $parent, array $lines, int $start, ?array $ if (IndentationHelper::isBlankLine($subLine)) { $subLines[] = ''; $subLineMap[] = $this->sourceLineFor($lineMap, $i); - $subRaw[] = ''; + $subRaw[] = $subLine; $sawBlankLine = true; $i++; @@ -2661,6 +2661,13 @@ protected function tryParseList(Node $parent, array $lines, int $start, ?array $ /** @var string|null $taskMarker */ $taskMarker = $itemInfo['taskMarker'] ?? null; $listItem = new ListItem($taskMarker); + // Attributes the previous item left without a target go to this item (djot.js); + // a blank line before it drops them. + if ($this->pendingAttributes !== [] && !$lastItemHadBlankAfter) { + $listItem->setAttributes($this->pendingAttributes); + } + $this->pendingAttributes = []; + $this->pendingAttributeSourceLines = []; $bodyItem = $listItem; $bodyContentIndent = 0; $listItemSourceLine = $this->sourceLineFor($lineMap, $i); @@ -2742,7 +2749,8 @@ protected function tryParseList(Node $parent, array $lines, int $start, ?array $ if (!$itemState['inFence'] && !$itemState['inComment'] && $itemDivFences === []) { break; } - $itemLines[] = ''; + // Keep the spaces past the content column: a raw or code block shows them. + $itemLines[] = IndentationHelper::stripLeadingIndent($nextLine, $contentIndent); $itemLineMap[] = $this->sourceLineFor($lineMap, $i); $this->trackDefinitionLazyState('', $itemState, $itemDivFences, $itemQuoteDepth); if (!$itemState['paragraphOpen']) { @@ -3107,7 +3115,7 @@ protected function tryParseList(Node $parent, array $lines, int $start, ?array $ $firstLine = ltrim($itemLines[0], ' '); if ( $parseItemLinesAsBlocks || $this->isBlockElementStart($firstLine) - || preg_match('/^\[[^\]]+\]:(?:[ \t]+\S*)?[ \t]*$/', $firstLine) + || preg_match('/^\[[^\]]*\]:(?:[ \t]+\S*)?[ \t]*$/', $firstLine) // A marker-line `{...}` may be a block attribute for the lines below. || preg_match('/^\{.*\}[ \t]*$/', $firstLine) ) { @@ -4619,8 +4627,7 @@ protected function tryParseFootnoteDefinition(array $lines, int $start, ?Node $p return $end - $start; } - // The note body must neither take nor leave attributes of the blocks around it. - $pending = [$this->pendingAttributes, $this->pendingAttributeSourceLines]; + // The note consumes attributes before it (djot.js), and its body leaves none behind. $this->pendingAttributes = []; $this->pendingAttributeSourceLines = []; $footnote = new Footnote(StringUtil::normalizeLabel($matches[1])); @@ -4631,7 +4638,8 @@ protected function tryParseFootnoteDefinition(array $lines, int $start, ?Node $p ); $this->parseBlocks($footnote, $contentLines, 0, $map); } - [$this->pendingAttributes, $this->pendingAttributeSourceLines] = $pending; + $this->pendingAttributes = []; + $this->pendingAttributeSourceLines = []; $this->footnotes[$footnote->getLabel()] = $footnote; return $end - $start; diff --git a/tests/TestCase/ConformanceFollowupsTest.php b/tests/TestCase/ConformanceFollowupsTest.php new file mode 100644 index 0000000..db7fcc6 --- /dev/null +++ b/tests/TestCase/ConformanceFollowupsTest.php @@ -0,0 +1,88 @@ +assertSame($expected, (new DjotConverter())->convert($djot)); + $this->assertSame($expected, (new DjotConverter(warnings: true))->convert($djot)); + } + + /** + * @return array + */ + public static function casesProvider(): array + { + return [ + 'reference definition with an empty label in a quote' => [ + "> []: p\npara\n", + "
    \n
    \n

    para

    \n", + ], + 'reference definition with an empty label after quoted text' => [ + "> x\n>\n> []: \nr\n", + "
    \n

    x

    \n
    \n

    r

    \n", + ], + 'attributes in an empty item go to the next item' => [ + "- {.a}\n- b\n", + "
      \n
    • \n
    • \n
    • \nb\n
    • \n
    \n", + ], + 'attributes in an empty ordered item go to the next item' => [ + "1. {#x}\n2. b\n c\n", + "
      \n
    1. \n
    2. \n
    3. \nb\nc\n
    4. \n
    \n", + ], + 'attributes after a blank line in an item go to the next item' => [ + "- x\n\n {.a}\n- b\n", + "
      \n
    • \n

      x

      \n
    • \n
    • \n

      b

      \n
    • \n
    \n", + ], + 'blank line before the next item drops the attributes' => [ + "- {.a}\n\n- b\n", + "
      \n
    • \n
    • \n
    • \n

      b

      \n
    • \n
    \n", + ], + 'footnote definition consumes the attributes before it' => [ + "{.a}\n[^n]: x\n> q\n", + "
    \n

    q

    \n
    \n", + ], + 'footnote definition in an item consumes the attributes' => [ + "- {.a}\n [^n]: x\n- b\n", + "
      \n
    • \n
    • \n
    • \nb\n
    • \n
    \n", + ], + 'tab after an escaped space in a cell' => [ + "| a\\ \t|\n", + "\n\n\n\n
    a \t
    \n", + ], + 'tab and spaces after the text of a cell' => [ + "| a \t |\n", + "\n\n\n\n
    a \t
    \n", + ], + 'whitespace-only line in a raw block in a tight item' => [ + "- ``` =html\n x\n \n y\n ```\n", + "
      \n
    • \nx\n \ny\n
    • \n
    \n", + ], + 'whitespace-only line in a code block in a tight item' => [ + "- ```\n x\n \n ```\n", + "
      \n
    • \n
      x\n  \n
      \n
    • \n
    \n", + ], + 'whitespace-only line in a code block in a loose item' => [ + "- a\n\n ```\n x\n \n y\n ```\n", + "
      \n
    • \n

      a

      \n
      x\n   \ny\n
      \n
    • \n
    \n", + ], + 'whitespace-only line in a raw block in a loose item' => [ + "- a\n\n ``` =html\n x\n \n y\n ```\n", + "
      \n
    • \n

      a

      \nx\n \ny\n
    • \n
    \n", + ], + ]; + } +} From ad1bca2d93ea4685a1b1f9f993a93e7038d74c2a Mon Sep 17 00:00:00 2001 From: Mark Scherer Date: Wed, 11 Nov 2026 14:53:17 +0100 Subject: [PATCH 2/2] Drop the attribute leak and keep trimming trailing cell tabs Attributes left without a block in a list item or footnote no longer move to the next item: djot.js does this, but it is an upstream bug (jgm/djot.js issue 175). Trailing tabs in table cells are trimmed again, as proposed upstream in jgm/djot.js PR 176. --- src/Parser/Block/TableParser.php | 4 +-- src/Parser/BlockParser.php | 13 ++------ tests/TestCase/ConformanceFollowupsTest.php | 36 ++++----------------- 3 files changed, 10 insertions(+), 43 deletions(-) diff --git a/src/Parser/Block/TableParser.php b/src/Parser/Block/TableParser.php index 1087aa8..5671b7f 100644 --- a/src/Parser/Block/TableParser.php +++ b/src/Parser/Block/TableParser.php @@ -713,13 +713,11 @@ public function lineEndsWithPipeOutsideCodeSpan(string $line): bool /** * Trim a cell's content, keeping a space that a backslash escapes (`| C:\\ |`). * - * Like djot.js, only spaces are trimmed at the end; a trailing tab is cell text. - * * @param string $content */ public function trimCellContent(string $content): string { - $trimmed = rtrim($content, ' '); + $trimmed = rtrim($content, " \t"); $backslashes = strlen($trimmed) - strlen(rtrim($trimmed, '\\')); if ($backslashes % 2 === 1 && strlen($trimmed) < strlen($content)) { $trimmed .= $content[strlen($trimmed)]; diff --git a/src/Parser/BlockParser.php b/src/Parser/BlockParser.php index 07a4f97..70bcbac 100644 --- a/src/Parser/BlockParser.php +++ b/src/Parser/BlockParser.php @@ -2661,13 +2661,6 @@ protected function tryParseList(Node $parent, array $lines, int $start, ?array $ /** @var string|null $taskMarker */ $taskMarker = $itemInfo['taskMarker'] ?? null; $listItem = new ListItem($taskMarker); - // Attributes the previous item left without a target go to this item (djot.js); - // a blank line before it drops them. - if ($this->pendingAttributes !== [] && !$lastItemHadBlankAfter) { - $listItem->setAttributes($this->pendingAttributes); - } - $this->pendingAttributes = []; - $this->pendingAttributeSourceLines = []; $bodyItem = $listItem; $bodyContentIndent = 0; $listItemSourceLine = $this->sourceLineFor($lineMap, $i); @@ -4627,7 +4620,8 @@ protected function tryParseFootnoteDefinition(array $lines, int $start, ?Node $p return $end - $start; } - // The note consumes attributes before it (djot.js), and its body leaves none behind. + // The note body must neither take nor leave attributes of the blocks around it. + $pending = [$this->pendingAttributes, $this->pendingAttributeSourceLines]; $this->pendingAttributes = []; $this->pendingAttributeSourceLines = []; $footnote = new Footnote(StringUtil::normalizeLabel($matches[1])); @@ -4638,8 +4632,7 @@ protected function tryParseFootnoteDefinition(array $lines, int $start, ?Node $p ); $this->parseBlocks($footnote, $contentLines, 0, $map); } - $this->pendingAttributes = []; - $this->pendingAttributeSourceLines = []; + [$this->pendingAttributes, $this->pendingAttributeSourceLines] = $pending; $this->footnotes[$footnote->getLabel()] = $footnote; return $end - $start; diff --git a/tests/TestCase/ConformanceFollowupsTest.php b/tests/TestCase/ConformanceFollowupsTest.php index db7fcc6..dbb28c0 100644 --- a/tests/TestCase/ConformanceFollowupsTest.php +++ b/tests/TestCase/ConformanceFollowupsTest.php @@ -9,8 +9,8 @@ use PHPUnit\Framework\TestCase; /** - * Empty-label references in quotes, attributes left by an item or a note, - * trailing tabs in cells and whitespace-only lines in item code (matches djot.js). + * Empty-label references in quotes and whitespace-only lines in item code (as djot.js), + * and trailing tabs trimmed from cells (djot.js keeps them, see jgm/djot.js#176). */ class ConformanceFollowupsTest extends TestCase { @@ -35,37 +35,13 @@ public static function casesProvider(): array "> x\n>\n> []: \nr\n", "
    \n

    x

    \n
    \n

    r

    \n", ], - 'attributes in an empty item go to the next item' => [ - "- {.a}\n- b\n", - "
      \n
    • \n
    • \n
    • \nb\n
    • \n
    \n", - ], - 'attributes in an empty ordered item go to the next item' => [ - "1. {#x}\n2. b\n c\n", - "
      \n
    1. \n
    2. \n
    3. \nb\nc\n
    4. \n
    \n", - ], - 'attributes after a blank line in an item go to the next item' => [ - "- x\n\n {.a}\n- b\n", - "
      \n
    • \n

      x

      \n
    • \n
    • \n

      b

      \n
    • \n
    \n", - ], - 'blank line before the next item drops the attributes' => [ - "- {.a}\n\n- b\n", - "
      \n
    • \n
    • \n
    • \n

      b

      \n
    • \n
    \n", - ], - 'footnote definition consumes the attributes before it' => [ - "{.a}\n[^n]: x\n> q\n", - "
    \n

    q

    \n
    \n", - ], - 'footnote definition in an item consumes the attributes' => [ - "- {.a}\n [^n]: x\n- b\n", - "
      \n
    • \n
    • \n
    • \nb\n
    • \n
    \n", - ], - 'tab after an escaped space in a cell' => [ + 'tab after an escaped space in a cell is trimmed' => [ "| a\\ \t|\n", - "\n\n\n\n
    a \t
    \n", + "\n\n\n\n
    a 
    \n", ], - 'tab and spaces after the text of a cell' => [ + 'tab and spaces after the text of a cell are trimmed' => [ "| a \t |\n", - "\n\n\n\n
    a \t
    \n", + "\n\n\n\n
    a
    \n", ], 'whitespace-only line in a raw block in a tight item' => [ "- ``` =html\n x\n \n y\n ```\n",