diff --git a/spec/fixtures/gfm-regression.txt b/spec/fixtures/gfm-regression.txt index b611a0c..d92ebf0 100644 --- a/spec/fixtures/gfm-regression.txt +++ b/spec/fixtures/gfm-regression.txt @@ -4,8 +4,7 @@ Issue #113: EOL character weirdness on Windows (Important: first line ends with CR + CR + LF) ```````````````````````````````` example -line1 - +line1 line2 .
line1
diff --git a/spec/parser_regression_spec.cr b/spec/parser_regression_spec.cr new file mode 100644 index 0000000..0b9bbe6 --- /dev/null +++ b/spec/parser_regression_spec.cr @@ -0,0 +1,118 @@ +require "./spec_helper" + +describe "Markdown parser regressions" do + it "treats CR, CRLF, and LF as line endings" do + Markd.to_html("line1\rline2\r\nline3\n").should eq("line1\nline2\nline3
\n") + end + + it "keeps the final block's source line with or without a trailing newline" do + options = Markd::Options.new(source_pos: true) + ["a", "a\n"].each do |input| + Markd.to_html(input, options).should eq(%(a
\n)) + end + end + + it "keeps unmatched link openers from crossing paragraphs" do + Markd.to_html("[open\n\nclose](url)").should eq("[open
\nclose](url)
\n") + end + + it "treats malformed entity references as text without swallowing inlines" do + Markd.to_html("&;").should eq("&;
\n") + Markd.to_html("x & *hi* &").should eq("x & hi &
\n") + end + + it "decodes numeric entities above the basic multilingual plane" do + Markd.to_html("😀").should eq("๐
\n") + Markd.to_html("[x](😀)").should eq(%(\n)) + Markd.to_html("").should eq("๏ฟฝ
\n") + end + + it "preserves URI escapes in inline links, references, and autolinks" do + Markd.to_html("[x](a%2Fb)").should eq(%(\n)) + Markd.to_html("[r]: /a%3Fb%26c\n\n[r]").should eq(%(\n)) + Markd.to_html("a*๐ก*
\n") + Markd.to_html("a\u2002*๐ก*").should eq("a\u2002๐ก
\n") + Markd.to_html("*a*\u00A0b").should eq("a\u00A0b
\n") + Markd.to_html("*a\u2002* b").should eq("*a\u2002* b
\n") + end + + it "recognizes Setext headings inside block quotes" do + Markd.to_html("> foo\n> ---").should eq("\n\n") + end + + it "ends a table before an ATX heading that contains a pipe" do + options = Markd::Options.new(gfm: true) + html = Markd.to_html("A | B\n--- | ---\n# Heading | Text", options) + html.should eq("foo
\n
| A | \nB | \n
|---|
รค @foo
\n") + end + + it "emits punctuation and following text once after autolinks" do + options = Markd::Options.new(autolink: true) + Markd.to_html("www.example.com/...? tail", options).should eq( + %(www.example.com/...? tail
\n) + ) + Markd.to_html("(http://www.example.com/path.) end", options).should eq( + %((http://www.example.com/path.) end
\n) + ) + end + + it "links a scheme URL with a single-label host" do + options = Markd::Options.new(autolink: true) + Markd.to_html("**Autolink and http://inlines**", options).should eq( + %(Autolink and http://inlines
\n) + ) + end + + it "links a one-character address after an embedded mailto prefix" do + options = Markd::Options.new(autolink: true) + Markd.to_html("xmailto:a@b.com", options).should eq(%(xmailto:a@b.com
\n)) + end + + it "retains autolink alongside the pending fixture tag" do + fixtures = extract_spec_tests(File.join(__DIR__, "fixtures/gfm-regression.txt")) + fixtures["Regression tests"].values.find!(&.["markdown"].includes?("[^widely-cited]")) + .["test_tags"].split.should eq(%w[footnotes autolink pending]) + end + + it "keeps explicit links and autolinks from nesting" do + options = Markd::Options.new(autolink: true) + Markd.to_html("[www.example.com](/p)", options).should eq(%(\n)) + Markd.to_html("[label a@b.com](/p)", options).should eq(%(\n)) + Markd.to_html("[see https://example.com/a](/p)", options).should eq( + %(\n) + ) + Markd.to_html("[[https://example.com](u)
\n) + ) + end + + it "autolinks URLs inside literal brackets" do + options = Markd::Options.new(autolink: true) + Markd.to_html("[see www.example.com", options).should eq( + %([see www.example.com
\n) + ) + Markd.to_html("[see https://example.com/a]", options).should eq( + %([see https://example.com/a]
\n) + ) + end +end diff --git a/spec/spec_helper.cr b/spec/spec_helper.cr index 8cd47f4..f8e5b70 100644 --- a/spec/spec_helper.cr +++ b/spec/spec_helper.cr @@ -41,7 +41,7 @@ def assert_section(file, section, examples, smart, gfm = false) end def assert_example(file, section, index, example, smart, gfm = false) - markdown = example["markdown"].gsub("โ", "\t").chomp + markdown = example["markdown"].gsub("โ", "\t") html = example["html"].gsub("โ", "\t") line = example["line"].to_i tags = example["test_tags"].split(" ") @@ -54,7 +54,7 @@ def assert_example(file, section, index, example, smart, gfm = false) ) options.smart = true if smart - if example["test_tags"].ends_with?("pending") + if tags.includes?("pending") pending "- #{index}\n#{show_space(markdown)}", file, line do output = Markd.to_html(markdown, options) output.should eq(html), file: file, line: line @@ -82,9 +82,9 @@ def extract_spec_tests(file) line_number = 0 test_tags = "" - while (line = input.read_line) + while (raw_line = input.read_line(chomp: false)) line_number += 1 - line = line.gsub(/\r\n?/, "\n") + line = raw_line.chomp break if line.includes?("") if !test_start && !result_start && (match = line.match(/^\#{1,6}\s+(.*)$/)) @@ -92,9 +92,9 @@ def extract_spec_tests(file) examples[current_section] = {} of Int32 => Hash(String, String) example_count = 0 else - if !test_start && !result_start && line =~ /^`{32} example([a-z ])*$/ + if !test_start && !result_start && (example = line.match(/^`{32} example((?: [a-z]+)*)$/)) test_start = true - test_tags = line[line.rindex!(' ') + 1..-1] + test_tags = example[1].strip elsif test_start && !result_start && line =~ /^\.$/ test_start = false result_start = true @@ -106,10 +106,10 @@ def extract_spec_tests(file) "line" => line_number.to_s, "markdown" => "", "html" => "", - "test_tags" => (test_tags == "example" ? "" : test_tags), + "test_tags" => test_tags, } of String => String - examples[current_section][example_count]["markdown"] += line + "\n" + examples[current_section][example_count]["markdown"] += raw_line elsif !test_start && result_start examples[current_section][example_count]["html"] += line + "\n" end diff --git a/src/markd/html_entities.cr b/src/markd/html_entities.cr index b383ad1..a041a38 100644 --- a/src/markd/html_entities.cr +++ b/src/markd/html_entities.cr @@ -46,7 +46,7 @@ module Markd::HTMLEntities end def self.decode_codepoint(codepoint) - return "\uFFFD" if codepoint >= 0xD800 && codepoint <= 0xDFFF || codepoint > 0x10FFF + return "\uFFFD" if codepoint.in?(0xD800..0xDFFF) || codepoint > 0x10FFFF if (decoded = Markd::HTMLEntities::DECODE_MAPPINGS[codepoint]?) codepoint = decoded diff --git a/src/markd/parsers/block.cr b/src/markd/parsers/block.cr index 2f0d459..b4eb2ad 100644 --- a/src/markd/parsers/block.cr +++ b/src/markd/parsers/block.cr @@ -67,17 +67,11 @@ module Markd::Parser end private def parse_blocks(source) - lines_size = 0 - source.each_line do |line| - process_line(line) - lines_size += 1 - end - - # ignore last blank line created by final newline - lines_size -= 1 if source.ends_with?('\n') + source = source.gsub(/\r\n?/, '\n') if source.byte_index('\r') + source.each_line { |line| process_line(line) } while (tip = tip?) - token(tip, lines_size) + token(tip, @current_line) end end diff --git a/src/markd/parsers/inline.cr b/src/markd/parsers/inline.cr index dddecaa..f3008d5 100644 --- a/src/markd/parsers/inline.cr +++ b/src/markd/parsers/inline.cr @@ -13,11 +13,14 @@ module Markd::Parser def initialize(@options : Options) @text = "" @pos = 0 + @extended_links = Set(Node).new @refmap = {} of String => Hash(String, String) | String end def parse(node : Node) @pos = 0 + @brackets = nil + @extended_links.clear @delimiters = nil @text = node.text.strip @@ -110,8 +113,9 @@ module Markd::Parser end unless res - @pos += 1 - node.append_child(text(char)) + character = char.ascii? ? char : unicode_char_at?(@pos).not_nil! + @pos += character.bytesize + node.append_child(text(character)) end true @@ -276,7 +280,7 @@ module Markd::Parser # Next, see if there's a link label before_label = @pos label_size = link_label - if label_size > 2 + if label_size > 1 ref_label = normalize_reference(@text.byte_slice(before_label, label_size + 1)) elsif !opener.bracket_after? # Empty or missing second label means to use the first label as the reference. @@ -308,7 +312,12 @@ module Markd::Parser while tmp next_node = tmp.next? tmp.unlink - child.append_child(tmp) + # Extended autolinks have lower precedence than an explicit link. + if !is_image && @extended_links.includes?(tmp) + child.append_child(tmp.first_child) + else + child.append_child(tmp) + end tmp = next_node end @@ -317,13 +326,7 @@ module Markd::Parser remove_bracket opener.node.unlink - unless is_image - opener = @brackets - while opener - opener.active = false unless opener.image? - opener = opener.previous? - end - end + deactivate_link_openers unless is_image else remove_bracket @pos = start_pos @@ -467,60 +470,63 @@ module Markd::Parser end end + private def deactivate_link_openers + opener = @brackets + while opener + opener.active = false unless opener.image? + opener = opener.previous? + end + end + private def auto_link(node : Node) if (matched_text = match(Rule::EMAIL_AUTO_LINK)) - node.append_child(link(matched_text, true)) - return true + node.append_child(link(matched_text, email: true)) elsif (matched_text = match(Rule::AUTO_LINK)) - node.append_child(link(matched_text, false)) - return true - elsif @options.autolink? - # These are all the extended autolinks from the - # autolink extension - - if (matched_text = match(Rule::WWW_AUTO_LINK)) - clean_text = autolink_cleanup(matched_text) - if clean_text.empty? - node.append_child(text(matched_text)) - else - _, post = @text.split(clean_text, 2) - node.append_child(link(clean_text, false, true)) - node.append_child(text(post)) if post.size > 0 && matched_text != clean_text - end - return true - elsif (matched_text = ( - match(Rule::PROTOCOL_AUTO_LINK) || - match(Rule::XMPP_AUTO_LINK) || - match(Rule::MAILTO_AUTO_LINK) - )) - clean_text = autolink_cleanup(matched_text) - if clean_text.empty? - node.append_child(text(matched_text)) - else - _, post = @text.split(clean_text, 2) - node.append_child(link(clean_text, false, false)) - node.append_child(text(post)) if post.size > 0 && matched_text != clean_text - end - return true - elsif (matched_text = match(Rule::EXTENDED_EMAIL_AUTO_LINK)) - # Emails that end in - or _ are declared not to be links by the spec: - # - # `.`, `-`, and `_` can occur on both sides of the `@`, but only `.` may occur at - # the end of the email address, in which case it will not be considered part of - # the address: + node.append_child(link(matched_text)) + else + return extended_autolink(node) + end - # a.b-c_d@a.b_ =>a.b-c_d@a.b_
+ deactivate_link_openers + true + end - if "-_".includes?(matched_text[-1]) - node.append_child(text(matched_text)) - else - node.append_child(link(matched_text, true, false)) - end - return true + private def extended_autolink(node : Node) + return false if !@options.autolink? + + www = match(Rule::WWW_AUTO_LINK) + if (matched_text = www || match(Rule::PROTOCOL_AUTO_LINK) || + match(Rule::XMPP_AUTO_LINK) || + (match(Rule::MAILTO_AUTO_LINK) if @pos == 0 || !char_at(@pos - 1).ascii_alphanumeric?)) + if @brackets && (closing = matched_text.byte_index(']')) + # Leave the bracket for the inline link parser. + @pos -= matched_text.bytesize - closing + matched_text = matched_text.byte_slice(0, closing) + end + clean_text = autolink_cleanup(matched_text) + if clean_text.empty? + node.append_child(text(matched_text)) + else + # Leave any trailing punctuation for the inline parser. + @pos -= matched_text.bytesize - clean_text.bytesize + append_extended_link(node, link(clean_text, add_proto: !www.nil?)) end + elsif (matched_text = match(Rule::EXTENDED_EMAIL_AUTO_LINK)) + if "-_".includes?(matched_text[-1]) + node.append_child(text(matched_text)) + else + append_extended_link(node, link(matched_text, email: true)) + end + else + return false end - false + true + end + + private def append_extended_link(node : Node, child : Node) + @extended_links.add(child) if @brackets + node.append_child(child) end private def html_tag(node : Node) @@ -540,32 +546,10 @@ module Markd::Parser end private def entity(node : Node) - if char_at?(@pos) == '&' - if char_at?(@pos + 1) == '#' - text = match(Rule::NUMERIC_HTML_ENTITY) || return false - text = text.byte_slice(1, text.bytesize - 2) - else - pos = @pos + 1 - loop do - char = char_at?(pos) - pos += 1 - case char - when ';' - break - when Char::ZERO, nil - return false - end - end - text = @text.byte_slice((@pos + 1), (pos - 1) - (@pos + 1)) - @pos = pos - end - - decoded_text = HTML.decode_entity text - node.append_child(text(decoded_text)) - true - else - false - end + rule = char_at?(@pos + 1) == '#' ? Rule::NUMERIC_HTML_ENTITY : Rule::HTML_ENTITY + token = match(rule) || return false + node.append_child(text(HTML.decode_entity(token.byte_slice(1, token.bytesize - 2)))) + true end private def emoji(node : Node) @@ -760,14 +744,13 @@ module Markd::Parser return if num_delims == 0 - char_before = start_pos == 0 ? '\n' : previous_unicode_char_at(start_pos) - char_after = unicode_char_at?(@pos) || '\n' + before = (start_pos == 0 ? '\n' : previous_unicode_char_at(start_pos)).to_s + after = (unicode_char_at?(@pos) || '\n').to_s - # Match ASCII code 160 => \xA0 (See http://www.adamkoch.com/2009/07/25/white-space-and-character-160/) - after_is_whitespace = char_after.ascii_whitespace? || char_after == '\u00A0' - after_is_punctuation = !!char_after.to_s.match(Rule::PUNCTUATION) - before_is_whitespace = char_before.ascii_whitespace? || char_after == '\u00A0' - before_is_punctuation = !!char_before.to_s.match(Rule::PUNCTUATION) + after_is_whitespace = after.matches?(Rule::UNICODE_WHITESPACE) + after_is_punctuation = after.matches?(Rule::PUNCTUATION) + before_is_whitespace = before.matches?(Rule::UNICODE_WHITESPACE) + before_is_punctuation = before.matches?(Rule::PUNCTUATION) left_flanking = !after_is_whitespace && (!after_is_punctuation || before_is_whitespace || before_is_punctuation) @@ -869,7 +852,7 @@ module Markd::Parser end private def space_at_end_of_line? - while char_at?(@pos) == ' ' + while char_at?(@pos).in?(' ', '\t') @pos += 1 end @@ -884,13 +867,13 @@ module Markd::Parser true end - # Parse zero or more space characters, including at most one newline + # Parse spaces and tabs, including at most one newline. private def spnl seen_newline = false while (c = char_at?(@pos)) if !seen_newline && c == '\n' seen_newline = true - elsif c != ' ' + elsif !c.in?(' ', '\t') break end @@ -908,82 +891,25 @@ module Markd::Parser end end - # This function advances @pos as far as possible until it finds a - # "special" character, such as '<', ']', or a special string (like a URL). - # - # Then it returns the chunk before it found that match, or in the case - # of special strings, the chunk matched. - private def match_main : String? start_pos = @pos - while (char = char_at?(@pos)) - # If we detected a special string (like a URL), and it's - # not the beggining of the string, we need to break right away. - # - # If we are at the beginning of the string, then we return - # the chunk matched - if @options.autolink? - advance = special_string?(@text, @pos) - if advance > 0 - if @pos > start_pos - break - else - @pos += advance - break - end - end + while (char = char_at?(@pos)) && main_char?(char) + if @pos > start_pos && char.in?('h', 'f') && @options.autolink? && + starts_extended_protocol?(@pos) + break end - - # If we detect a special character, we need to break - break if !main_char?(char) - @pos += 1 + @pos += char.ascii? ? 1 : unicode_char_at?(@pos).not_nil!.bytesize end - if start_pos == @pos - nil - else - @text.byte_slice(start_pos, @pos - start_pos) - end + @text.byte_slice(start_pos, @pos - start_pos) if @pos > start_pos end - # Identify "special" strings by matching against - # regular expressions. It returns the number of characters - # that were matched. + private def starts_extended_protocol?(pos : Int32) : Bool + return false unless "*_~( \n\t".includes?(char_at(pos - 1)) + return false unless Rule::AUTOLINK_PROTOCOLS.matches?(@text.byte_slice(pos)) - private def special_string?(full_text : String, pos : Int) : Int - text = full_text.byte_slice(pos) - # All such recognized autolinks can only come at the beginning of - # a line, after whitespace, or any of the delimiting characters `*`, `_`, `~`, - # and `(`. - if pos > 0 && !("*_~( \n\t".includes? char_at(pos - 1)) - 0 - elsif text.starts_with?("http://") || text.starts_with?("https://") || text.starts_with?("ftp://") - # This should not be an autolink: - # < ftp://example.com > - if full_text[...pos].includes?("<") && full_text[...pos].matches?(/<\s*$/) - return 0 - end - - m = autolink_cleanup(text.match(Rule::PROTOCOL_AUTO_LINK).to_s) - m.size - elsif text.starts_with?("www.") && text.matches?(Rule::WWW_AUTO_LINK) - m = autolink_cleanup(text.match(Rule::WWW_AUTO_LINK).to_s) - m.size - elsif text.includes?("@") && text.matches?(Rule::EXTENDED_EMAIL_AUTO_LINK) - # m = autolink_cleanup(text.match(Rule::EMAIL_AUTO_LINK).to_s) - matched_text = text.match(Rule::EMAIL_AUTO_LINK).to_s - - # `.`, `-`, and `_` can occur on both sides of the `@`, but only `.` may occur at - # the end of the email address, in which case it will not be considered part of - # the address: - - if "-_".includes? char_at(pos + matched_text.size + 1) - return 0 - end - matched_text.size - else - 0 - end + # Do not link the URL in text such as < ftp://example.com >. + !@text.byte_slice(0, pos).matches?(/<\s*$/) end # These cleanups are defined in the spec @@ -1072,27 +998,31 @@ module Markd::Parser # Normalize reference label: collapse internal whitespace # to single space, remove leading/trailing whitespace, case fold. def normalize_reference(text : String) - text[1..-2].strip.downcase.gsub("\n", " ") + text[1..-2].downcase.gsub(/[ \t\r\n]+/, ' ').strip(' ') end - private RESERVED_CHARS = ['&', '+', ',', '(', ')', '\'', '#', '*', '!', '#', '$', '/', ':', ';', '?', '@', '='] + private RESERVED_CHARS = "&+,()'#*!$/:;?@=" def normalize_uri(uri : String) + # Preserve the existing rendering of encoded parentheses. + uri = uri.gsub(/%2[89]/) { |escape| escape.ends_with?('8') ? '(' : ')' } + String.build(capacity: uri.bytesize) do |io| - URI.encode(decode_uri(uri), io) do |byte| - URI.unreserved?(byte) || RESERVED_CHARS.includes?(byte.chr) + uri.each_byte.with_index do |byte, index| + char = byte.unsafe_chr + if URI.unreserved?(byte) || RESERVED_CHARS.includes?(char) || + (char == '%' && uri.byte_at?(index + 1).try(&.unsafe_chr.hex?) && + uri.byte_at?(index + 2).try(&.unsafe_chr.hex?)) + io << char + else + io << '%' + io << '0' if byte < 16 + byte.to_s(io, 16, upcase: true) + end end end end - def decode_uri(text : String) - decoded = URI.decode(text) - if decoded.includes?('&') && decoded.includes?(';') - decoded = decoded.gsub(/^&(\w+);$/) { |chars| HTML.decode_entities(chars) } - end - decoded - end - class Bracket property node : Node property! previous : Bracket? diff --git a/src/markd/rule.cr b/src/markd/rule.cr index 5f57db0..0df2ab8 100644 --- a/src/markd/rule.cr +++ b/src/markd/rule.cr @@ -66,8 +66,8 @@ module Markd # A valid domain name is: # # segments of alphanumeric characters, underscores (_) and hyphens (-) - # separated by periods (.). There must be at least one period, and no - # underscores may be present in the last two segments of the domain. + # separated by periods (.). No underscores may be present in the last + # two segments. A URL with a scheme can have a single-label hostname. # # Alphanumeric characters in this context include emojis. LAST_DOMAIN_SEGMENT = /(?:[a-zA-Z0-9\-\p{Emoji_Presentation}\-]+)/ @@ -77,23 +77,24 @@ module Markd # For example: www.xxx._yyy.zzz is never linked because of the # _ in the last segment. DOMAIN_NAME = /(?:#{OTHER_DOMAIN_SEGMENTS}\.)*#{OTHER_DOMAIN_SEGMENTS}/ - VALID_DOMAIN_NAME = /^(?:#{OTHER_DOMAIN_SEGMENTS}\.)*(?:#{LAST_DOMAIN_SEGMENT}\.)+#{LAST_DOMAIN_SEGMENT}$/ + VALID_DOMAIN_NAME = /^(?:(?:#{OTHER_DOMAIN_SEGMENTS}\.)*#{LAST_DOMAIN_SEGMENT}\.)?#{LAST_DOMAIN_SEGMENT}$/ VALID_URL_PATH = /(?:\/[^\s<]*)?/ AUTOLINK_PROTOCOLS = /^(?:http|https|ftp):\/\// EMAIL_AUTO_LINK = /^<([a-zA-Z0-9.!#$%&'*+\/=?^_`{|}~-]+@[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(?:\.[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)*)>/ - EXTENDED_EMAIL_AUTO_LINK = /^([a-zA-Z0-9][a-zA-Z0-9.!#$%&'*+\/=?^_`{|}~-]+@[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(?:\.[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)+)[-_]*/ + EXTENDED_EMAIL_AUTO_LINK = /^([a-zA-Z0-9][a-zA-Z0-9.!#$%&'*+\/=?^_`{|}~-]*@[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(?:\.[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)+)[-_]*/ AUTO_LINK = /^<[A-Za-z][A-Za-z0-9.+-]{1,31}:[^<>\x00-\x20]*>/i WWW_AUTO_LINK = /^www\.#{DOMAIN_NAME}#{VALID_URL_PATH}/ XMPP_AUTO_LINK = /^xmpp:[A-Za-z0-9]+@#{DOMAIN_NAME}#{VALID_URL_PATH}/ MAILTO_AUTO_LINK = /^mailto:[A-Za-z0-9]+@#{DOMAIN_NAME}/ - PROTOCOL_AUTO_LINK = /#{AUTOLINK_PROTOCOLS}#{DOMAIN_NAME}#{VALID_URL_PATH}[^\s?!.,:*_~]/ + PROTOCOL_AUTO_LINK = /#{AUTOLINK_PROTOCOLS}#{DOMAIN_NAME}#{VALID_URL_PATH}/ - WHITESPACE_CHAR = /^[ \t\n\x0b\x0c\x0d]/ - WHITESPACE = /[ \t\n\x0b\x0c\x0d]+/ - LINE_ENDING = /\n|\x0d|\x0d\n/ - PUNCTUATION = /[$+<=>^`|~\p{P}]/ # Regex.new("[!"#$%&'()*+,\-./:;<=>?@\[\]^_`{|}~\xA1\xA7\xAB\xB6\xB7\xBB\xBF\u037E\u0387\u055A-\u055F\u0589\u058A\u05BE\u05C0\u05C3\u05C6\u05F3\u05F4\u0609\u060A\u060C\u060D\u061B\u061E\u061F\u066A-\u066D\u06D4\u0700-\u070D\u07F7-\u07F9\u0830-\u083E\u085E\u0964\u0965\u0970\u0AF0\u0DF4\u0E4F\u0E5A\u0E5B\u0F04-\u0F12\u0F14\u0F3A-\u0F3D\u0F85\u0FD0-\u0FD4\u0FD9\u0FDA\u104A-\u104F\u10FB\u1360-\u1368\u1400\u166D\u166E\u169B\u169C\u16EB-\u16ED\u1735\u1736\u17D4-\u17D6\u17D8-\u17DA\u1800-\u180A\u1944\u1945\u1A1E\u1A1F\u1AA0-\u1AA6\u1AA8-\u1AAD\u1B5A-\u1B60\u1BFC-\u1BFF\u1C3B-\u1C3F\u1C7E\u1C7F\u1CC0-\u1CC7\u1CD3\u2010-\u2027\u2030-\u2043\u2045-\u2051\u2053-\u205E\u207D\u207E\u208D\u208E\u2308-\u230B\u2329\u232A\u2768-\u2775\u27C5\u27C6\u27E6-\u27EF\u2983-\u2998\u29D8-\u29DB\u29FC\u29FD\u2CF9-\u2CFC\u2CFE\u2CFF\u2D70\u2E00-\u2E2E\u2E30-\u2E42\u3001-\u3003\u3008-\u3011\u3014-\u301F\u3030\u303D\u30A0\u30FB\uA4FE\uA4FF\uA60D-\uA60F\uA673\uA67E\uA6F2-\uA6F7\uA874-\uA877\uA8CE\uA8CF\uA8F8-\uA8FA\uA8FC\uA92E\uA92F\uA95F\uA9C1-\uA9CD\uA9DE\uA9DF\uAA5C-\uAA5F\uAADE\uAADF\uAAF0\uAAF1\uABEB\uFD3E\uFD3F\uFE10-\uFE19\uFE30-\uFE52\uFE54-\uFE61\uFE63\uFE68\uFE6A\uFE6B\uFF01-\uFF03\uFF05-\uFF0A\uFF0C-\uFF0F\uFF1A\uFF1B\uFF1F\uFF20\uFF3B-\uFF3D\uFF3F\uFF5B\uFF5D\uFF5F-\uFF65]|\uD800[\uDD00-\uDD02\uDF9F\uDFD0]|\uD801\uDD6F|\uD802[\uDC57\uDD1F\uDD3F\uDE50-\uDE58\uDE7F\uDEF0-\uDEF6\uDF39-\uDF3F\uDF99-\uDF9C]|\uD804[\uDC47-\uDC4D\uDCBB\uDCBC\uDCBE-\uDCC1\uDD40-\uDD43\uDD74\uDD75\uDDC5-\uDDC9\uDDCD\uDDDB\uDDDD-\uDDDF\uDE38-\uDE3D\uDEA9]|\uD805[\uDCC6\uDDC1-\uDDD7\uDE41-\uDE43\uDF3C-\uDF3E]|\uD809[\uDC70-\uDC74]|\uD81A[\uDE6E\uDE6F\uDEF5\uDF37-\uDF3B\uDF44]|\uD82F\uDC9F|\uD836[\uDE87-\uDE8B]") + WHITESPACE_CHAR = /^[ \t\n\x0b\x0c\x0d]/ + WHITESPACE = /[ \t\n\x0b\x0c\x0d]+/ + LINE_ENDING = /\n|\x0d|\x0d\n/ + PUNCTUATION = /[\p{P}\p{S}]/ + UNICODE_WHITESPACE = /[\t\n\f\r\p{Zs}]/ UNSAFE_PROTOCOL = /^javascript:|vbscript:|file:|data:/i UNSAFE_DATA_PROTOCOL = /^data:image\/(?:png|gif|jpeg|webp)/i diff --git a/src/markd/rules/heading.cr b/src/markd/rules/heading.cr index 23c1db1..b1e52ab 100644 --- a/src/markd/rules/heading.cr +++ b/src/markd/rules/heading.cr @@ -22,8 +22,7 @@ module Markd::Rule MatchValue::Leaf elsif (match = match?(parser, SETEXT_HEADING_MARKER)) && - container.type.paragraph? && (parent = container.parent?) && - !parent.type.block_quote? + container.type.paragraph? && container.parent? # Setext Heading matched parser.close_unmatched_blocks diff --git a/src/markd/rules/table.cr b/src/markd/rules/table.cr index 078cf92..00e6b5b 100644 --- a/src/markd/rules/table.cr +++ b/src/markd/rules/table.cr @@ -122,12 +122,16 @@ module Markd::Rule # Match only lines that look like a table separator # or start with a | or look like multiple cells separated by | private def match_continuation?(parser : Parser) + if !parser.indented && Heading::ATX_HEADING_MARKER.matches?(parser.line[parser.next_nonspace..]) + return false + end + !parser.indented && (parser.line[0]? == '|' || parser.line.match(TABLE_HEADING_SEPARATOR) || parser.line.match(TABLE_CELL_SEPARATOR)) || # Lines that are not empty and are not the start of a # block-level structure are ALSO continuations (see gfm-spec.txt:3397) - !(parser.line.strip.empty? || parser.line.matches?(/^(?:>|\#{1,6}|`{3}|\t{1}|\s{4}|(?:[*-+]\s)+|[0-9]+\.)+/)) + !(parser.line.strip.empty? || parser.line.matches?(/^(?:>|`{3}|\t{1}|\s{4}|(?:[*-+]\s)+|[0-9]+\.)+/)) end private def strip_pipe(text : String) : String