From 41a99e9100a8e7b6f1f57a3f5d306e0905040f3c Mon Sep 17 00:00:00 2001 From: Margret Riegert Date: Thu, 19 Nov 2026 04:35:48 -0500 Subject: [PATCH 01/18] Normalize Markdown input line endings Treat CRLF as one line ending and a standalone CR as a separate line ending so mixed input preserves blank lines. Assisted-by: GPT-6.2 Sol --- spec/parser_regression_spec.cr | 7 +++++++ src/markd/parsers/block.cr | 1 + 2 files changed, 8 insertions(+) create mode 100644 spec/parser_regression_spec.cr diff --git a/spec/parser_regression_spec.cr b/spec/parser_regression_spec.cr new file mode 100644 index 0000000..c7aeedf --- /dev/null +++ b/spec/parser_regression_spec.cr @@ -0,0 +1,7 @@ +require "./spec_helper" + +describe "Markdown parser regressions" do + it "treats CR, CRLF, and LF as line endings" do + Markd.to_html("line1\rline2\r\nline3\n").should eq("

line1\nline2\nline3

\n") + end +end diff --git a/src/markd/parsers/block.cr b/src/markd/parsers/block.cr index 2f0d459..38b269e 100644 --- a/src/markd/parsers/block.cr +++ b/src/markd/parsers/block.cr @@ -67,6 +67,7 @@ module Markd::Parser end private def parse_blocks(source) + source = source.gsub(/\r\n?/, '\n') if source.byte_index('\r') lines_size = 0 source.each_line do |line| process_line(line) From 5015aaf812e05c7e1d6d830f37df33598718cd27 Mon Sep 17 00:00:00 2001 From: Margret Riegert Date: Thu, 19 Nov 2026 04:35:49 -0500 Subject: [PATCH 02/18] Use the last parsed line when closing blocks Use the parser's actual line counter for source positions, including input that ends in a newline. Assisted-by: GPT-6.2 Sol --- spec/parser_regression_spec.cr | 7 +++++++ src/markd/parsers/block.cr | 11 ++--------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/spec/parser_regression_spec.cr b/spec/parser_regression_spec.cr index c7aeedf..46e6a22 100644 --- a/spec/parser_regression_spec.cr +++ b/spec/parser_regression_spec.cr @@ -4,4 +4,11 @@ describe "Markdown parser regressions" do it "treats CR, CRLF, and LF as line endings" do Markd.to_html("line1\rline2\r\nline3\n").should eq("

line1\nline2\nline3

\n") end + + it "keeps the final block's source line with or without a trailing newline" do + options = Markd::Options.new(source_pos: true) + ["a", "a\n"].each do |input| + Markd.to_html(input, options).should eq(%(

a

\n)) + end + end end diff --git a/src/markd/parsers/block.cr b/src/markd/parsers/block.cr index 38b269e..b4eb2ad 100644 --- a/src/markd/parsers/block.cr +++ b/src/markd/parsers/block.cr @@ -68,17 +68,10 @@ module Markd::Parser private def parse_blocks(source) source = source.gsub(/\r\n?/, '\n') if source.byte_index('\r') - lines_size = 0 - source.each_line do |line| - process_line(line) - lines_size += 1 - end - - # ignore last blank line created by final newline - lines_size -= 1 if source.ends_with?('\n') + source.each_line { |line| process_line(line) } while (tip = tip?) - token(tip, lines_size) + token(tip, @current_line) end end From f9c53aa9584671fc20faf11176006148cff0bbcf Mon Sep 17 00:00:00 2001 From: Margret Riegert Date: Thu, 19 Nov 2026 04:35:49 -0500 Subject: [PATCH 03/18] Reset inline bracket state between blocks Keep an unmatched link or image opener from claiming text in a later paragraph. Assisted-by: GPT-6.2 Sol --- spec/parser_regression_spec.cr | 4 ++++ src/markd/parsers/inline.cr | 1 + 2 files changed, 5 insertions(+) diff --git a/spec/parser_regression_spec.cr b/spec/parser_regression_spec.cr index 46e6a22..97616e1 100644 --- a/spec/parser_regression_spec.cr +++ b/spec/parser_regression_spec.cr @@ -11,4 +11,8 @@ describe "Markdown parser regressions" do Markd.to_html(input, options).should eq(%(

a

\n)) end end + + it "keeps unmatched link openers from crossing paragraphs" do + Markd.to_html("[open\n\nclose](url)").should eq("

[open

\n

close](url)

\n") + end end diff --git a/src/markd/parsers/inline.cr b/src/markd/parsers/inline.cr index dddecaa..e86741a 100644 --- a/src/markd/parsers/inline.cr +++ b/src/markd/parsers/inline.cr @@ -18,6 +18,7 @@ module Markd::Parser def parse(node : Node) @pos = 0 + @brackets = nil @delimiters = nil @text = node.text.strip From 7f9707129f9579a6566764db86506cfb60998095 Mon Sep 17 00:00:00 2001 From: Margret Riegert Date: Thu, 19 Nov 2026 04:35:49 -0500 Subject: [PATCH 04/18] Leave malformed entity references as text Match the entity syntax before decoding so a stray ampersand cannot raise an exception or consume later inline markup. Assisted-by: GPT-6.2 Sol --- spec/parser_regression_spec.cr | 5 +++++ src/markd/parsers/inline.cr | 30 ++++-------------------------- 2 files changed, 9 insertions(+), 26 deletions(-) diff --git a/spec/parser_regression_spec.cr b/spec/parser_regression_spec.cr index 97616e1..bc4bf62 100644 --- a/spec/parser_regression_spec.cr +++ b/spec/parser_regression_spec.cr @@ -15,4 +15,9 @@ describe "Markdown parser regressions" do it "keeps unmatched link openers from crossing paragraphs" do Markd.to_html("[open\n\nclose](url)").should eq("

[open

\n

close](url)

\n") end + + it "treats malformed entity references as text without swallowing inlines" do + Markd.to_html("&;").should eq("

&;

\n") + Markd.to_html("x & *hi* &").should eq("

x & hi &

\n") + end end diff --git a/src/markd/parsers/inline.cr b/src/markd/parsers/inline.cr index e86741a..884d561 100644 --- a/src/markd/parsers/inline.cr +++ b/src/markd/parsers/inline.cr @@ -541,32 +541,10 @@ module Markd::Parser end private def entity(node : Node) - if char_at?(@pos) == '&' - if char_at?(@pos + 1) == '#' - text = match(Rule::NUMERIC_HTML_ENTITY) || return false - text = text.byte_slice(1, text.bytesize - 2) - else - pos = @pos + 1 - loop do - char = char_at?(pos) - pos += 1 - case char - when ';' - break - when Char::ZERO, nil - return false - end - end - text = @text.byte_slice((@pos + 1), (pos - 1) - (@pos + 1)) - @pos = pos - end - - decoded_text = HTML.decode_entity text - node.append_child(text(decoded_text)) - true - else - false - end + rule = char_at?(@pos + 1) == '#' ? Rule::NUMERIC_HTML_ENTITY : Rule::HTML_ENTITY + token = match(rule) || return false + node.append_child(text(HTML.decode_entity(token.byte_slice(1, token.bytesize - 2)))) + true end private def emoji(node : Node) From 12fab6f1f9c33cc257f9f09ef30a7a876d70a140 Mon Sep 17 00:00:00 2001 From: Margret Riegert Date: Thu, 19 Nov 2026 04:35:50 -0500 Subject: [PATCH 05/18] Decode supplementary Unicode entities Use U+10FFFF as the upper bound so valid supplementary code points render and invalid code points still become the replacement character. Assisted-by: GPT-6.2 Sol --- spec/parser_regression_spec.cr | 6 ++++++ src/markd/html_entities.cr | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/spec/parser_regression_spec.cr b/spec/parser_regression_spec.cr index bc4bf62..4589b18 100644 --- a/spec/parser_regression_spec.cr +++ b/spec/parser_regression_spec.cr @@ -20,4 +20,10 @@ describe "Markdown parser regressions" do Markd.to_html("&;").should eq("

&;

\n") Markd.to_html("x & *hi* &").should eq("

x & hi &

\n") end + + it "decodes numeric entities above the basic multilingual plane" do + Markd.to_html("😀").should eq("

😀

\n") + Markd.to_html("[x](😀)").should eq(%(

x

\n)) + Markd.to_html("�").should eq("

�

\n") + end end diff --git a/src/markd/html_entities.cr b/src/markd/html_entities.cr index b383ad1..a041a38 100644 --- a/src/markd/html_entities.cr +++ b/src/markd/html_entities.cr @@ -46,7 +46,7 @@ module Markd::HTMLEntities end def self.decode_codepoint(codepoint) - return "\uFFFD" if codepoint >= 0xD800 && codepoint <= 0xDFFF || codepoint > 0x10FFF + return "\uFFFD" if codepoint.in?(0xD800..0xDFFF) || codepoint > 0x10FFFF if (decoded = Markd::HTMLEntities::DECODE_MAPPINGS[codepoint]?) codepoint = decoded From 5ddff2d4b10d6681c11e76da06b17549b8f0db1e Mon Sep 17 00:00:00 2001 From: Margret Riegert Date: Thu, 19 Nov 2026 04:54:29 -0500 Subject: [PATCH 06/18] Preserve encoded path and query delimiters Keep escapes such as %2F intact so a link destination retains its meaning. Render parentheses as required by the existing double-encoding fixture and continue to decode Markdown entities once. Assisted-by: GPT-6.2 Sol --- spec/parser_regression_spec.cr | 8 ++++++++ src/markd/parsers/inline.cr | 26 +++++++++++++++----------- 2 files changed, 23 insertions(+), 11 deletions(-) diff --git a/spec/parser_regression_spec.cr b/spec/parser_regression_spec.cr index 4589b18..9ce6c3b 100644 --- a/spec/parser_regression_spec.cr +++ b/spec/parser_regression_spec.cr @@ -26,4 +26,12 @@ describe "Markdown parser regressions" do Markd.to_html("[x](😀)").should eq(%(

x

\n)) Markd.to_html("�").should eq("

�

\n") end + + it "preserves URI escapes in inline links, references, and autolinks" do + Markd.to_html("[x](a%2Fb)").should eq(%(

x

\n)) + Markd.to_html("[r]: /a%3Fb%26c\n\n[r]").should eq(%(

r

\n)) + Markd.to_html("").should eq( + %(

https://example.com/a%26b

\n) + ) + end end diff --git a/src/markd/parsers/inline.cr b/src/markd/parsers/inline.cr index 884d561..b8ca34a 100644 --- a/src/markd/parsers/inline.cr +++ b/src/markd/parsers/inline.cr @@ -1054,24 +1054,28 @@ module Markd::Parser text[1..-2].strip.downcase.gsub("\n", " ") end - private RESERVED_CHARS = ['&', '+', ',', '(', ')', '\'', '#', '*', '!', '#', '$', '/', ':', ';', '?', '@', '='] + private RESERVED_CHARS = "&+,()'#*!$/:;?@=" def normalize_uri(uri : String) + # Preserve the existing rendering of encoded parentheses. + uri = uri.gsub(/%2[89]/) { |escape| escape.ends_with?('8') ? '(' : ')' } + String.build(capacity: uri.bytesize) do |io| - URI.encode(decode_uri(uri), io) do |byte| - URI.unreserved?(byte) || RESERVED_CHARS.includes?(byte.chr) + uri.each_byte.with_index do |byte, index| + char = byte.unsafe_chr + if URI.unreserved?(byte) || RESERVED_CHARS.includes?(char) || + (char == '%' && uri.byte_at?(index + 1).try(&.unsafe_chr.hex?) && + uri.byte_at?(index + 2).try(&.unsafe_chr.hex?)) + io << char + else + io << '%' + io << '0' if byte < 16 + byte.to_s(io, 16, upcase: true) + end end end end - def decode_uri(text : String) - decoded = URI.decode(text) - if decoded.includes?('&') && decoded.includes?(';') - decoded = decoded.gsub(/^&(\w+);$/) { |chars| HTML.decode_entities(chars) } - end - decoded - end - class Bracket property node : Node property! previous : Bracket? From a5f2f5b865912d85ed5e8c10f107420ac2510d70 Mon Sep 17 00:00:00 2001 From: Margret Riegert Date: Thu, 19 Nov 2026 04:35:50 -0500 Subject: [PATCH 07/18] Resolve one-character reference labels A full reference link can use a label containing a single character. Assisted-by: GPT-6.2 Sol --- spec/parser_regression_spec.cr | 4 ++++ src/markd/parsers/inline.cr | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/spec/parser_regression_spec.cr b/spec/parser_regression_spec.cr index 9ce6c3b..6d637f4 100644 --- a/spec/parser_regression_spec.cr +++ b/spec/parser_regression_spec.cr @@ -34,4 +34,8 @@ describe "Markdown parser regressions" do %(

https://example.com/a%26b

\n) ) end + + it "resolves one-character labels" do + Markd.to_html("[r]: /url\n\n[x][r]").should eq(%(

x

\n)) + end end diff --git a/src/markd/parsers/inline.cr b/src/markd/parsers/inline.cr index b8ca34a..7e258fc 100644 --- a/src/markd/parsers/inline.cr +++ b/src/markd/parsers/inline.cr @@ -277,7 +277,7 @@ module Markd::Parser # Next, see if there's a link label before_label = @pos label_size = link_label - if label_size > 2 + if label_size > 1 ref_label = normalize_reference(@text.byte_slice(before_label, label_size + 1)) elsif !opener.bracket_after? # Empty or missing second label means to use the first label as the reference. From 54856d830a25376a4e87efdfd927579b0c0c6f78 Mon Sep 17 00:00:00 2001 From: Margret Riegert Date: Thu, 19 Nov 2026 04:35:50 -0500 Subject: [PATCH 08/18] Normalize whitespace in reference labels Collapse runs of spaces and tabs in references and accept tabs where reference parsing expects horizontal whitespace. Assisted-by: GPT-6.2 Sol --- spec/parser_regression_spec.cr | 5 +++++ src/markd/parsers/inline.cr | 8 ++++---- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/spec/parser_regression_spec.cr b/spec/parser_regression_spec.cr index 6d637f4..0c6477d 100644 --- a/spec/parser_regression_spec.cr +++ b/spec/parser_regression_spec.cr @@ -38,4 +38,9 @@ describe "Markdown parser regressions" do it "resolves one-character labels" do Markd.to_html("[r]: /url\n\n[x][r]").should eq(%(

x

\n)) end + + it "collapses whitespace in reference labels" do + Markd.to_html("[a b]: /url\n\n[a b]").should eq(%(

a b

\n)) + Markd.to_html("[a\tb]: /url\t\n\n[a b]").should eq(%(

a b

\n)) + end end diff --git a/src/markd/parsers/inline.cr b/src/markd/parsers/inline.cr index 7e258fc..603c8b0 100644 --- a/src/markd/parsers/inline.cr +++ b/src/markd/parsers/inline.cr @@ -848,7 +848,7 @@ module Markd::Parser end private def space_at_end_of_line? - while char_at?(@pos) == ' ' + while char_at?(@pos).in?(' ', '\t') @pos += 1 end @@ -863,13 +863,13 @@ module Markd::Parser true end - # Parse zero or more space characters, including at most one newline + # Parse spaces and tabs, including at most one newline. private def spnl seen_newline = false while (c = char_at?(@pos)) if !seen_newline && c == '\n' seen_newline = true - elsif c != ' ' + elsif !c.in?(' ', '\t') break end @@ -1051,7 +1051,7 @@ module Markd::Parser # Normalize reference label: collapse internal whitespace # to single space, remove leading/trailing whitespace, case fold. def normalize_reference(text : String) - text[1..-2].strip.downcase.gsub("\n", " ") + text[1..-2].downcase.gsub(/[ \t\r\n]+/, ' ').strip(' ') end private RESERVED_CHARS = "&+,()'#*!$/:;?@=" From ad383f1b785183628ea0da660b90f2b376526607 Mon Sep 17 00:00:00 2001 From: Margret Riegert Date: Thu, 19 Nov 2026 04:35:51 -0500 Subject: [PATCH 09/18] Use Unicode character classes at emphasis boundaries Check whitespace on the correct side of each delimiter and classify Unicode separator and symbol characters according to CommonMark. Assisted-by: GPT-6.2 Sol --- spec/parser_regression_spec.cr | 7 +++++++ src/markd/parsers/inline.cr | 15 +++++++-------- src/markd/rule.cr | 9 +++++---- 3 files changed, 19 insertions(+), 12 deletions(-) diff --git a/spec/parser_regression_spec.cr b/spec/parser_regression_spec.cr index 0c6477d..61d03d7 100644 --- a/spec/parser_regression_spec.cr +++ b/spec/parser_regression_spec.cr @@ -43,4 +43,11 @@ describe "Markdown parser regressions" do Markd.to_html("[a b]: /url\n\n[a b]").should eq(%(

a b

\n)) Markd.to_html("[a\tb]: /url\t\n\n[a b]").should eq(%(

a b

\n)) end + + it "uses Unicode spaces and symbols to determine emphasis boundaries" do + Markd.to_html("a*💡*").should eq("

a*💡*

\n") + Markd.to_html("a\u2002*💡*").should eq("

a\u2002💡

\n") + Markd.to_html("*a*\u00A0b").should eq("

a\u00A0b

\n") + Markd.to_html("*a\u2002* b").should eq("

*a\u2002* b

\n") + end end diff --git a/src/markd/parsers/inline.cr b/src/markd/parsers/inline.cr index 603c8b0..764853a 100644 --- a/src/markd/parsers/inline.cr +++ b/src/markd/parsers/inline.cr @@ -739,14 +739,13 @@ module Markd::Parser return if num_delims == 0 - char_before = start_pos == 0 ? '\n' : previous_unicode_char_at(start_pos) - char_after = unicode_char_at?(@pos) || '\n' - - # Match ASCII code 160 => \xA0 (See http://www.adamkoch.com/2009/07/25/white-space-and-character-160/) - after_is_whitespace = char_after.ascii_whitespace? || char_after == '\u00A0' - after_is_punctuation = !!char_after.to_s.match(Rule::PUNCTUATION) - before_is_whitespace = char_before.ascii_whitespace? || char_after == '\u00A0' - before_is_punctuation = !!char_before.to_s.match(Rule::PUNCTUATION) + before = (start_pos == 0 ? '\n' : previous_unicode_char_at(start_pos)).to_s + after = (unicode_char_at?(@pos) || '\n').to_s + + after_is_whitespace = after.matches?(Rule::UNICODE_WHITESPACE) + after_is_punctuation = after.matches?(Rule::PUNCTUATION) + before_is_whitespace = before.matches?(Rule::UNICODE_WHITESPACE) + before_is_punctuation = before.matches?(Rule::PUNCTUATION) left_flanking = !after_is_whitespace && (!after_is_punctuation || before_is_whitespace || before_is_punctuation) diff --git a/src/markd/rule.cr b/src/markd/rule.cr index 5f57db0..d82889b 100644 --- a/src/markd/rule.cr +++ b/src/markd/rule.cr @@ -90,10 +90,11 @@ module Markd MAILTO_AUTO_LINK = /^mailto:[A-Za-z0-9]+@#{DOMAIN_NAME}/ PROTOCOL_AUTO_LINK = /#{AUTOLINK_PROTOCOLS}#{DOMAIN_NAME}#{VALID_URL_PATH}[^\s?!.,:*_~]/ - WHITESPACE_CHAR = /^[ \t\n\x0b\x0c\x0d]/ - WHITESPACE = /[ \t\n\x0b\x0c\x0d]+/ - LINE_ENDING = /\n|\x0d|\x0d\n/ - PUNCTUATION = /[$+<=>^`|~\p{P}]/ # Regex.new("[!"#$%&'()*+,\-./:;<=>?@\[\]^_`{|}~\xA1\xA7\xAB\xB6\xB7\xBB\xBF\u037E\u0387\u055A-\u055F\u0589\u058A\u05BE\u05C0\u05C3\u05C6\u05F3\u05F4\u0609\u060A\u060C\u060D\u061B\u061E\u061F\u066A-\u066D\u06D4\u0700-\u070D\u07F7-\u07F9\u0830-\u083E\u085E\u0964\u0965\u0970\u0AF0\u0DF4\u0E4F\u0E5A\u0E5B\u0F04-\u0F12\u0F14\u0F3A-\u0F3D\u0F85\u0FD0-\u0FD4\u0FD9\u0FDA\u104A-\u104F\u10FB\u1360-\u1368\u1400\u166D\u166E\u169B\u169C\u16EB-\u16ED\u1735\u1736\u17D4-\u17D6\u17D8-\u17DA\u1800-\u180A\u1944\u1945\u1A1E\u1A1F\u1AA0-\u1AA6\u1AA8-\u1AAD\u1B5A-\u1B60\u1BFC-\u1BFF\u1C3B-\u1C3F\u1C7E\u1C7F\u1CC0-\u1CC7\u1CD3\u2010-\u2027\u2030-\u2043\u2045-\u2051\u2053-\u205E\u207D\u207E\u208D\u208E\u2308-\u230B\u2329\u232A\u2768-\u2775\u27C5\u27C6\u27E6-\u27EF\u2983-\u2998\u29D8-\u29DB\u29FC\u29FD\u2CF9-\u2CFC\u2CFE\u2CFF\u2D70\u2E00-\u2E2E\u2E30-\u2E42\u3001-\u3003\u3008-\u3011\u3014-\u301F\u3030\u303D\u30A0\u30FB\uA4FE\uA4FF\uA60D-\uA60F\uA673\uA67E\uA6F2-\uA6F7\uA874-\uA877\uA8CE\uA8CF\uA8F8-\uA8FA\uA8FC\uA92E\uA92F\uA95F\uA9C1-\uA9CD\uA9DE\uA9DF\uAA5C-\uAA5F\uAADE\uAADF\uAAF0\uAAF1\uABEB\uFD3E\uFD3F\uFE10-\uFE19\uFE30-\uFE52\uFE54-\uFE61\uFE63\uFE68\uFE6A\uFE6B\uFF01-\uFF03\uFF05-\uFF0A\uFF0C-\uFF0F\uFF1A\uFF1B\uFF1F\uFF20\uFF3B-\uFF3D\uFF3F\uFF5B\uFF5D\uFF5F-\uFF65]|\uD800[\uDD00-\uDD02\uDF9F\uDFD0]|\uD801\uDD6F|\uD802[\uDC57\uDD1F\uDD3F\uDE50-\uDE58\uDE7F\uDEF0-\uDEF6\uDF39-\uDF3F\uDF99-\uDF9C]|\uD804[\uDC47-\uDC4D\uDCBB\uDCBC\uDCBE-\uDCC1\uDD40-\uDD43\uDD74\uDD75\uDDC5-\uDDC9\uDDCD\uDDDB\uDDDD-\uDDDF\uDE38-\uDE3D\uDEA9]|\uD805[\uDCC6\uDDC1-\uDDD7\uDE41-\uDE43\uDF3C-\uDF3E]|\uD809[\uDC70-\uDC74]|\uD81A[\uDE6E\uDE6F\uDEF5\uDF37-\uDF3B\uDF44]|\uD82F\uDC9F|\uD836[\uDE87-\uDE8B]") + WHITESPACE_CHAR = /^[ \t\n\x0b\x0c\x0d]/ + WHITESPACE = /[ \t\n\x0b\x0c\x0d]+/ + LINE_ENDING = /\n|\x0d|\x0d\n/ + PUNCTUATION = /[\p{P}\p{S}]/ + UNICODE_WHITESPACE = /[\t\n\f\r\p{Zs}]/ UNSAFE_PROTOCOL = /^javascript:|vbscript:|file:|data:/i UNSAFE_DATA_PROTOCOL = /^data:image\/(?:png|gif|jpeg|webp)/i From d6f958dd49ecb7a41692518903a3318971a7d4ad Mon Sep 17 00:00:00 2001 From: Margret Riegert Date: Thu, 19 Nov 2026 04:35:51 -0500 Subject: [PATCH 10/18] Recognize Setext headings inside block quotes Allow the heading rule to consume an underline inside the container while keeping an unquoted thematic break outside the quote. Assisted-by: GPT-6.2 Sol --- spec/parser_regression_spec.cr | 4 ++++ src/markd/rules/heading.cr | 3 +-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/spec/parser_regression_spec.cr b/spec/parser_regression_spec.cr index 61d03d7..ee1f1b6 100644 --- a/spec/parser_regression_spec.cr +++ b/spec/parser_regression_spec.cr @@ -50,4 +50,8 @@ describe "Markdown parser regressions" do Markd.to_html("*a*\u00A0b").should eq("

a\u00A0b

\n") Markd.to_html("*a\u2002* b").should eq("

*a\u2002* b

\n") end + + it "recognizes Setext headings inside block quotes" do + Markd.to_html("> foo\n> ---").should eq("
\n

foo

\n
\n") + end end diff --git a/src/markd/rules/heading.cr b/src/markd/rules/heading.cr index 23c1db1..b1e52ab 100644 --- a/src/markd/rules/heading.cr +++ b/src/markd/rules/heading.cr @@ -22,8 +22,7 @@ module Markd::Rule MatchValue::Leaf elsif (match = match?(parser, SETEXT_HEADING_MARKER)) && - container.type.paragraph? && (parent = container.parent?) && - !parent.type.block_quote? + container.type.paragraph? && container.parent? # Setext Heading matched parser.close_unmatched_blocks From 552a8b05eee50e1db7730d4463485d2d33da9ef1 Mon Sep 17 00:00:00 2001 From: Margret Riegert Date: Thu, 19 Nov 2026 04:35:51 -0500 Subject: [PATCH 11/18] End GFM tables at ATX headings A pipe in a valid heading must not cause the heading to be consumed as another table row. Assisted-by: GPT-6.2 Sol --- spec/parser_regression_spec.cr | 6 ++++++ src/markd/rules/table.cr | 6 +++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/spec/parser_regression_spec.cr b/spec/parser_regression_spec.cr index ee1f1b6..27627ac 100644 --- a/spec/parser_regression_spec.cr +++ b/spec/parser_regression_spec.cr @@ -54,4 +54,10 @@ describe "Markdown parser regressions" do it "recognizes Setext headings inside block quotes" do Markd.to_html("> foo\n> ---").should eq("
\n

foo

\n
\n") end + + it "ends a table before an ATX heading that contains a pipe" do + options = Markd::Options.new(gfm: true) + html = Markd.to_html("A | B\n--- | ---\n# Heading | Text", options) + html.should eq("\n\n\n\n\n\n\n
AB
\n

Heading | Text

\n") + end end diff --git a/src/markd/rules/table.cr b/src/markd/rules/table.cr index 078cf92..00e6b5b 100644 --- a/src/markd/rules/table.cr +++ b/src/markd/rules/table.cr @@ -122,12 +122,16 @@ module Markd::Rule # Match only lines that look like a table separator # or start with a | or look like multiple cells separated by | private def match_continuation?(parser : Parser) + if !parser.indented && Heading::ATX_HEADING_MARKER.matches?(parser.line[parser.next_nonspace..]) + return false + end + !parser.indented && (parser.line[0]? == '|' || parser.line.match(TABLE_HEADING_SEPARATOR) || parser.line.match(TABLE_CELL_SEPARATOR)) || # Lines that are not empty and are not the start of a # block-level structure are ALSO continuations (see gfm-spec.txt:3397) - !(parser.line.strip.empty? || parser.line.matches?(/^(?:>|\#{1,6}|`{3}|\t{1}|\s{4}|(?:[*-+]\s)+|[0-9]+\.)+/)) + !(parser.line.strip.empty? || parser.line.matches?(/^(?:>|`{3}|\t{1}|\s{4}|(?:[*-+]\s)+|[0-9]+\.)+/)) end private def strip_pipe(text : String) : String From 942cbe2f161995c0e3cf888466c7f5d3295d3b59 Mon Sep 17 00:00:00 2001 From: Margret Riegert Date: Thu, 19 Nov 2026 04:35:52 -0500 Subject: [PATCH 12/18] Keep autolink scanning on UTF-8 character boundaries Advance offsets by bytes without leaving the cursor inside a UTF-8 character before a regex match. Assisted-by: GPT-6.2 Sol --- spec/parser_regression_spec.cr | 5 +++++ src/markd/parsers/inline.cr | 11 ++++++----- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/spec/parser_regression_spec.cr b/spec/parser_regression_spec.cr index 27627ac..4deb1c7 100644 --- a/spec/parser_regression_spec.cr +++ b/spec/parser_regression_spec.cr @@ -60,4 +60,9 @@ describe "Markdown parser regressions" do html = Markd.to_html("A | B\n--- | ---\n# Heading | Text", options) html.should eq("\n\n\n\n\n\n\n
AB
\n

Heading | Text

\n") end + + it "preserves UTF-8 characters while scanning autolinks" do + options = Markd::Options.new(autolink: true) + Markd.to_html("ä @foo", options).should eq("

ä @foo

\n") + end end diff --git a/src/markd/parsers/inline.cr b/src/markd/parsers/inline.cr index 764853a..69c88d9 100644 --- a/src/markd/parsers/inline.cr +++ b/src/markd/parsers/inline.cr @@ -111,8 +111,9 @@ module Markd::Parser end unless res - @pos += 1 - node.append_child(text(char)) + character = char.ascii? ? char : unicode_char_at?(@pos).not_nil! + @pos += character.bytesize + node.append_child(text(character)) end true @@ -914,7 +915,7 @@ module Markd::Parser # If we detect a special character, we need to break break if !main_char?(char) - @pos += 1 + @pos += char.ascii? ? 1 : unicode_char_at?(@pos).not_nil!.bytesize end if start_pos == @pos @@ -943,10 +944,10 @@ module Markd::Parser end m = autolink_cleanup(text.match(Rule::PROTOCOL_AUTO_LINK).to_s) - m.size + m.bytesize elsif text.starts_with?("www.") && text.matches?(Rule::WWW_AUTO_LINK) m = autolink_cleanup(text.match(Rule::WWW_AUTO_LINK).to_s) - m.size + m.bytesize elsif text.includes?("@") && text.matches?(Rule::EXTENDED_EMAIL_AUTO_LINK) # m = autolink_cleanup(text.match(Rule::EMAIL_AUTO_LINK).to_s) matched_text = text.match(Rule::EMAIL_AUTO_LINK).to_s From f05d3e882b067109fbf63842d7c016e80ea9d20c Mon Sep 17 00:00:00 2001 From: Margret Riegert Date: Thu, 19 Nov 2026 04:35:52 -0500 Subject: [PATCH 13/18] Resume inline scanning after the autolink Rewind over trimmed punctuation instead of copying the remaining paragraph into the current node. Keep scheme detection and URL matching on the same byte offsets. Assisted-by: GPT-6.2 Sol --- spec/parser_regression_spec.cr | 10 +++ src/markd/parsers/inline.cr | 153 +++++++++------------------------ src/markd/rule.cr | 2 +- 3 files changed, 51 insertions(+), 114 deletions(-) diff --git a/spec/parser_regression_spec.cr b/spec/parser_regression_spec.cr index 4deb1c7..e2c592a 100644 --- a/spec/parser_regression_spec.cr +++ b/spec/parser_regression_spec.cr @@ -65,4 +65,14 @@ describe "Markdown parser regressions" do options = Markd::Options.new(autolink: true) Markd.to_html("ä @foo", options).should eq("

ä @foo

\n") end + + it "emits punctuation and following text once after autolinks" do + options = Markd::Options.new(autolink: true) + Markd.to_html("www.example.com/...? tail", options).should eq( + %(

www.example.com/...? tail

\n) + ) + Markd.to_html("(http://www.example.com/path.) end", options).should eq( + %(

(http://www.example.com/path.) end

\n) + ) + end end diff --git a/src/markd/parsers/inline.cr b/src/markd/parsers/inline.cr index 69c88d9..c48b93f 100644 --- a/src/markd/parsers/inline.cr +++ b/src/markd/parsers/inline.cr @@ -471,58 +471,42 @@ module Markd::Parser private def auto_link(node : Node) if (matched_text = match(Rule::EMAIL_AUTO_LINK)) - node.append_child(link(matched_text, true)) - return true + node.append_child(link(matched_text, email: true)) elsif (matched_text = match(Rule::AUTO_LINK)) - node.append_child(link(matched_text, false)) - return true - elsif @options.autolink? - # These are all the extended autolinks from the - # autolink extension - - if (matched_text = match(Rule::WWW_AUTO_LINK)) - clean_text = autolink_cleanup(matched_text) - if clean_text.empty? - node.append_child(text(matched_text)) - else - _, post = @text.split(clean_text, 2) - node.append_child(link(clean_text, false, true)) - node.append_child(text(post)) if post.size > 0 && matched_text != clean_text - end - return true - elsif (matched_text = ( - match(Rule::PROTOCOL_AUTO_LINK) || - match(Rule::XMPP_AUTO_LINK) || - match(Rule::MAILTO_AUTO_LINK) - )) - clean_text = autolink_cleanup(matched_text) - if clean_text.empty? - node.append_child(text(matched_text)) - else - _, post = @text.split(clean_text, 2) - node.append_child(link(clean_text, false, false)) - node.append_child(text(post)) if post.size > 0 && matched_text != clean_text - end - return true - elsif (matched_text = match(Rule::EXTENDED_EMAIL_AUTO_LINK)) - # Emails that end in - or _ are declared not to be links by the spec: - # - # `.`, `-`, and `_` can occur on both sides of the `@`, but only `.` may occur at - # the end of the email address, in which case it will not be considered part of - # the address: + node.append_child(link(matched_text)) + else + return extended_autolink(node) + end - # a.b-c_d@a.b_ =>

a.b-c_d@a.b_

+ true + end - if "-_".includes?(matched_text[-1]) - node.append_child(text(matched_text)) - else - node.append_child(link(matched_text, true, false)) - end - return true + private def extended_autolink(node : Node) + return false if !@options.autolink? + + www = match(Rule::WWW_AUTO_LINK) + if (matched_text = www || match(Rule::PROTOCOL_AUTO_LINK) || + match(Rule::XMPP_AUTO_LINK) || + match(Rule::MAILTO_AUTO_LINK)) + clean_text = autolink_cleanup(matched_text) + if clean_text.empty? + node.append_child(text(matched_text)) + else + # Leave any trailing punctuation for the inline parser. + @pos -= matched_text.bytesize - clean_text.bytesize + node.append_child(link(clean_text, add_proto: !www.nil?)) end + elsif (matched_text = match(Rule::EXTENDED_EMAIL_AUTO_LINK)) + if "-_".includes?(matched_text[-1]) + node.append_child(text(matched_text)) + else + node.append_child(link(matched_text, email: true)) + end + else + return false end - false + true end private def html_tag(node : Node) @@ -887,82 +871,25 @@ module Markd::Parser end end - # This function advances @pos as far as possible until it finds a - # "special" character, such as '<', ']', or a special string (like a URL). - # - # Then it returns the chunk before it found that match, or in the case - # of special strings, the chunk matched. - private def match_main : String? start_pos = @pos - while (char = char_at?(@pos)) - # If we detected a special string (like a URL), and it's - # not the beggining of the string, we need to break right away. - # - # If we are at the beginning of the string, then we return - # the chunk matched - if @options.autolink? - advance = special_string?(@text, @pos) - if advance > 0 - if @pos > start_pos - break - else - @pos += advance - break - end - end + while (char = char_at?(@pos)) && main_char?(char) + if @pos > start_pos && char.in?('h', 'f') && @options.autolink? && + starts_extended_protocol?(@pos) + break end - - # If we detect a special character, we need to break - break if !main_char?(char) @pos += char.ascii? ? 1 : unicode_char_at?(@pos).not_nil!.bytesize end - if start_pos == @pos - nil - else - @text.byte_slice(start_pos, @pos - start_pos) - end + @text.byte_slice(start_pos, @pos - start_pos) if @pos > start_pos end - # Identify "special" strings by matching against - # regular expressions. It returns the number of characters - # that were matched. - - private def special_string?(full_text : String, pos : Int) : Int - text = full_text.byte_slice(pos) - # All such recognized autolinks can only come at the beginning of - # a line, after whitespace, or any of the delimiting characters `*`, `_`, `~`, - # and `(`. - if pos > 0 && !("*_~( \n\t".includes? char_at(pos - 1)) - 0 - elsif text.starts_with?("http://") || text.starts_with?("https://") || text.starts_with?("ftp://") - # This should not be an autolink: - # < ftp://example.com > - if full_text[...pos].includes?("<") && full_text[...pos].matches?(/<\s*$/) - return 0 - end + private def starts_extended_protocol?(pos : Int32) : Bool + return false unless "*_~( \n\t".includes?(char_at(pos - 1)) + return false unless Rule::AUTOLINK_PROTOCOLS.matches?(@text.byte_slice(pos)) - m = autolink_cleanup(text.match(Rule::PROTOCOL_AUTO_LINK).to_s) - m.bytesize - elsif text.starts_with?("www.") && text.matches?(Rule::WWW_AUTO_LINK) - m = autolink_cleanup(text.match(Rule::WWW_AUTO_LINK).to_s) - m.bytesize - elsif text.includes?("@") && text.matches?(Rule::EXTENDED_EMAIL_AUTO_LINK) - # m = autolink_cleanup(text.match(Rule::EMAIL_AUTO_LINK).to_s) - matched_text = text.match(Rule::EMAIL_AUTO_LINK).to_s - - # `.`, `-`, and `_` can occur on both sides of the `@`, but only `.` may occur at - # the end of the email address, in which case it will not be considered part of - # the address: - - if "-_".includes? char_at(pos + matched_text.size + 1) - return 0 - end - matched_text.size - else - 0 - end + # Do not link the URL in text such as < ftp://example.com >. + !@text.byte_slice(0, pos).matches?(/<\s*$/) end # These cleanups are defined in the spec diff --git a/src/markd/rule.cr b/src/markd/rule.cr index d82889b..e8ddd73 100644 --- a/src/markd/rule.cr +++ b/src/markd/rule.cr @@ -88,7 +88,7 @@ module Markd WWW_AUTO_LINK = /^www\.#{DOMAIN_NAME}#{VALID_URL_PATH}/ XMPP_AUTO_LINK = /^xmpp:[A-Za-z0-9]+@#{DOMAIN_NAME}#{VALID_URL_PATH}/ MAILTO_AUTO_LINK = /^mailto:[A-Za-z0-9]+@#{DOMAIN_NAME}/ - PROTOCOL_AUTO_LINK = /#{AUTOLINK_PROTOCOLS}#{DOMAIN_NAME}#{VALID_URL_PATH}[^\s?!.,:*_~]/ + PROTOCOL_AUTO_LINK = /#{AUTOLINK_PROTOCOLS}#{DOMAIN_NAME}#{VALID_URL_PATH}/ WHITESPACE_CHAR = /^[ \t\n\x0b\x0c\x0d]/ WHITESPACE = /[ \t\n\x0b\x0c\x0d]+/ From abf30eb872431b1f5ccd1cfe030e797a26b62733 Mon Sep 17 00:00:00 2001 From: Margret Riegert Date: Thu, 19 Nov 2026 04:35:52 -0500 Subject: [PATCH 14/18] Link dotted and single-label hosts in scheme URLs Match the cmark-gfm fixture for a single-label hostname in a URL with an explicit scheme. Assisted-by: GPT-6.2 Sol --- spec/parser_regression_spec.cr | 7 +++++++ src/markd/rule.cr | 6 +++--- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/spec/parser_regression_spec.cr b/spec/parser_regression_spec.cr index e2c592a..5e9ca49 100644 --- a/spec/parser_regression_spec.cr +++ b/spec/parser_regression_spec.cr @@ -75,4 +75,11 @@ describe "Markdown parser regressions" do %(

(http://www.example.com/path.) end

\n) ) end + + it "links a scheme URL with a single-label host" do + options = Markd::Options.new(autolink: true) + Markd.to_html("**Autolink and http://inlines**", options).should eq( + %(

Autolink and http://inlines

\n) + ) + end end diff --git a/src/markd/rule.cr b/src/markd/rule.cr index e8ddd73..5838c64 100644 --- a/src/markd/rule.cr +++ b/src/markd/rule.cr @@ -66,8 +66,8 @@ module Markd # A valid domain name is: # # segments of alphanumeric characters, underscores (_) and hyphens (-) - # separated by periods (.). There must be at least one period, and no - # underscores may be present in the last two segments of the domain. + # separated by periods (.). No underscores may be present in the last + # two segments. A URL with a scheme can have a single-label hostname. # # Alphanumeric characters in this context include emojis. LAST_DOMAIN_SEGMENT = /(?:[a-zA-Z0-9\-\p{Emoji_Presentation}\-]+)/ @@ -77,7 +77,7 @@ module Markd # For example: www.xxx._yyy.zzz is never linked because of the # _ in the last segment. DOMAIN_NAME = /(?:#{OTHER_DOMAIN_SEGMENTS}\.)*#{OTHER_DOMAIN_SEGMENTS}/ - VALID_DOMAIN_NAME = /^(?:#{OTHER_DOMAIN_SEGMENTS}\.)*(?:#{LAST_DOMAIN_SEGMENT}\.)+#{LAST_DOMAIN_SEGMENT}$/ + VALID_DOMAIN_NAME = /^(?:(?:#{OTHER_DOMAIN_SEGMENTS}\.)*#{LAST_DOMAIN_SEGMENT}\.)?#{LAST_DOMAIN_SEGMENT}$/ VALID_URL_PATH = /(?:\/[^\s<]*)?/ AUTOLINK_PROTOCOLS = /^(?:http|https|ftp):\/\// From e1106034deddd24cf0a25bb5a9dbb63736f034fe Mon Sep 17 00:00:00 2001 From: Margret Riegert Date: Thu, 19 Nov 2026 04:35:52 -0500 Subject: [PATCH 15/18] Respect boundaries around embedded mailto prefixes Leave a prefix embedded in a word as literal text while allowing the following address, including one-character local parts, to autolink. Assisted-by: GPT-6.2 Sol --- spec/parser_regression_spec.cr | 5 +++++ src/markd/parsers/inline.cr | 2 +- src/markd/rule.cr | 2 +- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/spec/parser_regression_spec.cr b/spec/parser_regression_spec.cr index 5e9ca49..f1eb58a 100644 --- a/spec/parser_regression_spec.cr +++ b/spec/parser_regression_spec.cr @@ -82,4 +82,9 @@ describe "Markdown parser regressions" do %(

Autolink and http://inlines

\n) ) end + + it "links a one-character address after an embedded mailto prefix" do + options = Markd::Options.new(autolink: true) + Markd.to_html("xmailto:a@b.com", options).should eq(%(

xmailto:a@b.com

\n)) + end end diff --git a/src/markd/parsers/inline.cr b/src/markd/parsers/inline.cr index c48b93f..d8d0d79 100644 --- a/src/markd/parsers/inline.cr +++ b/src/markd/parsers/inline.cr @@ -487,7 +487,7 @@ module Markd::Parser www = match(Rule::WWW_AUTO_LINK) if (matched_text = www || match(Rule::PROTOCOL_AUTO_LINK) || match(Rule::XMPP_AUTO_LINK) || - match(Rule::MAILTO_AUTO_LINK)) + (match(Rule::MAILTO_AUTO_LINK) if @pos == 0 || !char_at(@pos - 1).ascii_alphanumeric?)) clean_text = autolink_cleanup(matched_text) if clean_text.empty? node.append_child(text(matched_text)) diff --git a/src/markd/rule.cr b/src/markd/rule.cr index 5838c64..0df2ab8 100644 --- a/src/markd/rule.cr +++ b/src/markd/rule.cr @@ -83,7 +83,7 @@ module Markd AUTOLINK_PROTOCOLS = /^(?:http|https|ftp):\/\// EMAIL_AUTO_LINK = /^<([a-zA-Z0-9.!#$%&'*+\/=?^_`{|}~-]+@[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(?:\.[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)*)>/ - EXTENDED_EMAIL_AUTO_LINK = /^([a-zA-Z0-9][a-zA-Z0-9.!#$%&'*+\/=?^_`{|}~-]+@[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(?:\.[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)+)[-_]*/ + EXTENDED_EMAIL_AUTO_LINK = /^([a-zA-Z0-9][a-zA-Z0-9.!#$%&'*+\/=?^_`{|}~-]*@[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(?:\.[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)+)[-_]*/ AUTO_LINK = /^<[A-Za-z][A-Za-z0-9.+-]{1,31}:[^<>\x00-\x20]*>/i WWW_AUTO_LINK = /^www\.#{DOMAIN_NAME}#{VALID_URL_PATH}/ XMPP_AUTO_LINK = /^xmpp:[A-Za-z0-9]+@#{DOMAIN_NAME}#{VALID_URL_PATH}/ From e19f869fd0b81d92f58e298fb574b36323105278 Mon Sep 17 00:00:00 2001 From: Margret Riegert Date: Thu, 19 Nov 2026 04:35:53 -0500 Subject: [PATCH 16/18] Keep all tags on fixture examples A pending example can still carry extension tags such as autolink and footnotes. Parse the full tag list. Assisted-by: GPT-6.2 Sol --- spec/parser_regression_spec.cr | 6 ++++++ spec/spec_helper.cr | 8 ++++---- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/spec/parser_regression_spec.cr b/spec/parser_regression_spec.cr index f1eb58a..958eae1 100644 --- a/spec/parser_regression_spec.cr +++ b/spec/parser_regression_spec.cr @@ -87,4 +87,10 @@ describe "Markdown parser regressions" do options = Markd::Options.new(autolink: true) Markd.to_html("xmailto:a@b.com", options).should eq(%(

xmailto:a@b.com

\n)) end + + it "retains autolink alongside the pending fixture tag" do + fixtures = extract_spec_tests(File.join(__DIR__, "fixtures/gfm-regression.txt")) + fixtures["Regression tests"].values.find!(&.["markdown"].includes?("[^widely-cited]")) + .["test_tags"].split.should eq(%w[footnotes autolink pending]) + end end diff --git a/spec/spec_helper.cr b/spec/spec_helper.cr index 8cd47f4..9c1542e 100644 --- a/spec/spec_helper.cr +++ b/spec/spec_helper.cr @@ -54,7 +54,7 @@ def assert_example(file, section, index, example, smart, gfm = false) ) options.smart = true if smart - if example["test_tags"].ends_with?("pending") + if tags.includes?("pending") pending "- #{index}\n#{show_space(markdown)}", file, line do output = Markd.to_html(markdown, options) output.should eq(html), file: file, line: line @@ -92,9 +92,9 @@ def extract_spec_tests(file) examples[current_section] = {} of Int32 => Hash(String, String) example_count = 0 else - if !test_start && !result_start && line =~ /^`{32} example([a-z ])*$/ + if !test_start && !result_start && (example = line.match(/^`{32} example((?: [a-z]+)*)$/)) test_start = true - test_tags = line[line.rindex!(' ') + 1..-1] + test_tags = example[1].strip elsif test_start && !result_start && line =~ /^\.$/ test_start = false result_start = true @@ -106,7 +106,7 @@ def extract_spec_tests(file) "line" => line_number.to_s, "markdown" => "", "html" => "", - "test_tags" => (test_tags == "example" ? "" : test_tags), + "test_tags" => test_tags, } of String => String examples[current_section][example_count]["markdown"] += line + "\n" From 07c967e22e208d33bf3082db5fcf197de6ef41dd Mon Sep 17 00:00:00 2001 From: Margret Riegert Date: Thu, 19 Nov 2026 04:35:53 -0500 Subject: [PATCH 17/18] Preserve line endings in parser fixtures Restore the original CR CR LF input for the Windows line-ending regression and pass those bytes unchanged to the parser. Assisted-by: GPT-6.2 Sol --- spec/fixtures/gfm-regression.txt | 3 +-- spec/spec_helper.cr | 8 ++++---- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/spec/fixtures/gfm-regression.txt b/spec/fixtures/gfm-regression.txt index b611a0c..d92ebf0 100644 --- a/spec/fixtures/gfm-regression.txt +++ b/spec/fixtures/gfm-regression.txt @@ -4,8 +4,7 @@ Issue #113: EOL character weirdness on Windows (Important: first line ends with CR + CR + LF) ```````````````````````````````` example -line1 - +line1 line2 .

line1

diff --git a/spec/spec_helper.cr b/spec/spec_helper.cr index 9c1542e..f8e5b70 100644 --- a/spec/spec_helper.cr +++ b/spec/spec_helper.cr @@ -41,7 +41,7 @@ def assert_section(file, section, examples, smart, gfm = false) end def assert_example(file, section, index, example, smart, gfm = false) - markdown = example["markdown"].gsub("→", "\t").chomp + markdown = example["markdown"].gsub("→", "\t") html = example["html"].gsub("→", "\t") line = example["line"].to_i tags = example["test_tags"].split(" ") @@ -82,9 +82,9 @@ def extract_spec_tests(file) line_number = 0 test_tags = "" - while (line = input.read_line) + while (raw_line = input.read_line(chomp: false)) line_number += 1 - line = line.gsub(/\r\n?/, "\n") + line = raw_line.chomp break if line.includes?("") if !test_start && !result_start && (match = line.match(/^\#{1,6}\s+(.*)$/)) @@ -109,7 +109,7 @@ def extract_spec_tests(file) "test_tags" => test_tags, } of String => String - examples[current_section][example_count]["markdown"] += line + "\n" + examples[current_section][example_count]["markdown"] += raw_line elsif !test_start && result_start examples[current_section][example_count]["html"] += line + "\n" end From 3db59c322ab67513ae785ee2d67ceef89832004a Mon Sep 17 00:00:00 2001 From: Margret Riegert Date: Thu, 19 Nov 2026 04:37:29 -0500 Subject: [PATCH 18/18] Keep extended autolinks out of explicit links Give explicit links precedence over extended autolinks in their labels. Keep autolinks in ordinary bracketed text and leave closing brackets for the inline parser. An angle-bracket autolink still disables enclosing link openers. Assisted-by: GPT-6.2 Sol --- spec/parser_regression_spec.cr | 22 +++++++++++++++++++ src/markd/parsers/inline.cr | 40 +++++++++++++++++++++++++--------- 2 files changed, 52 insertions(+), 10 deletions(-) diff --git a/spec/parser_regression_spec.cr b/spec/parser_regression_spec.cr index 958eae1..0b9bbe6 100644 --- a/spec/parser_regression_spec.cr +++ b/spec/parser_regression_spec.cr @@ -93,4 +93,26 @@ describe "Markdown parser regressions" do fixtures["Regression tests"].values.find!(&.["markdown"].includes?("[^widely-cited]")) .["test_tags"].split.should eq(%w[footnotes autolink pending]) end + + it "keeps explicit links and autolinks from nesting" do + options = Markd::Options.new(autolink: true) + Markd.to_html("[www.example.com](/p)", options).should eq(%(

www.example.com

\n)) + Markd.to_html("[label a@b.com](/p)", options).should eq(%(

label a@b.com

\n)) + Markd.to_html("[see https://example.com/a](/p)", options).should eq( + %(

see https://example.com/a

\n) + ) + Markd.to_html("[](u)").should eq( + %(

[https://example.com](u)

\n) + ) + end + + it "autolinks URLs inside literal brackets" do + options = Markd::Options.new(autolink: true) + Markd.to_html("[see www.example.com", options).should eq( + %(

[see www.example.com

\n) + ) + Markd.to_html("[see https://example.com/a]", options).should eq( + %(

[see https://example.com/a]

\n) + ) + end end diff --git a/src/markd/parsers/inline.cr b/src/markd/parsers/inline.cr index d8d0d79..f3008d5 100644 --- a/src/markd/parsers/inline.cr +++ b/src/markd/parsers/inline.cr @@ -13,12 +13,14 @@ module Markd::Parser def initialize(@options : Options) @text = "" @pos = 0 + @extended_links = Set(Node).new @refmap = {} of String => Hash(String, String) | String end def parse(node : Node) @pos = 0 @brackets = nil + @extended_links.clear @delimiters = nil @text = node.text.strip @@ -310,7 +312,12 @@ module Markd::Parser while tmp next_node = tmp.next? tmp.unlink - child.append_child(tmp) + # Extended autolinks have lower precedence than an explicit link. + if !is_image && @extended_links.includes?(tmp) + child.append_child(tmp.first_child) + else + child.append_child(tmp) + end tmp = next_node end @@ -319,13 +326,7 @@ module Markd::Parser remove_bracket opener.node.unlink - unless is_image - opener = @brackets - while opener - opener.active = false unless opener.image? - opener = opener.previous? - end - end + deactivate_link_openers unless is_image else remove_bracket @pos = start_pos @@ -469,6 +470,14 @@ module Markd::Parser end end + private def deactivate_link_openers + opener = @brackets + while opener + opener.active = false unless opener.image? + opener = opener.previous? + end + end + private def auto_link(node : Node) if (matched_text = match(Rule::EMAIL_AUTO_LINK)) node.append_child(link(matched_text, email: true)) @@ -478,6 +487,7 @@ module Markd::Parser return extended_autolink(node) end + deactivate_link_openers true end @@ -488,19 +498,24 @@ module Markd::Parser if (matched_text = www || match(Rule::PROTOCOL_AUTO_LINK) || match(Rule::XMPP_AUTO_LINK) || (match(Rule::MAILTO_AUTO_LINK) if @pos == 0 || !char_at(@pos - 1).ascii_alphanumeric?)) + if @brackets && (closing = matched_text.byte_index(']')) + # Leave the bracket for the inline link parser. + @pos -= matched_text.bytesize - closing + matched_text = matched_text.byte_slice(0, closing) + end clean_text = autolink_cleanup(matched_text) if clean_text.empty? node.append_child(text(matched_text)) else # Leave any trailing punctuation for the inline parser. @pos -= matched_text.bytesize - clean_text.bytesize - node.append_child(link(clean_text, add_proto: !www.nil?)) + append_extended_link(node, link(clean_text, add_proto: !www.nil?)) end elsif (matched_text = match(Rule::EXTENDED_EMAIL_AUTO_LINK)) if "-_".includes?(matched_text[-1]) node.append_child(text(matched_text)) else - node.append_child(link(matched_text, email: true)) + append_extended_link(node, link(matched_text, email: true)) end else return false @@ -509,6 +524,11 @@ module Markd::Parser true end + private def append_extended_link(node : Node, child : Node) + @extended_links.add(child) if @brackets + node.append_child(child) + end + private def html_tag(node : Node) if (text = match(Rule::HTML_TAG)) child = Node.new(Node::Type::HTMLInline)