Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
18 commits
Select commit Hold shift + click to select a range
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 1 addition & 2 deletions spec/fixtures/gfm-regression.txt
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,7 @@ Issue #113: EOL character weirdness on Windows
(Important: first line ends with CR + CR + LF)

```````````````````````````````` example
line1

line1
line2
.
<p>line1</p>
Expand Down
118 changes: 118 additions & 0 deletions spec/parser_regression_spec.cr
Original file line number Diff line number Diff line change
@@ -0,0 +1,118 @@
require "./spec_helper"

describe "Markdown parser regressions" do
it "treats CR, CRLF, and LF as line endings" do
Markd.to_html("line1\rline2\r\nline3\n").should eq("<p>line1\nline2\nline3</p>\n")
end

it "keeps the final block's source line with or without a trailing newline" do
options = Markd::Options.new(source_pos: true)
["a", "a\n"].each do |input|
Markd.to_html(input, options).should eq(%(<p data-source-pos="1:1-1:1">a</p>\n))
end
end

it "keeps unmatched link openers from crossing paragraphs" do
Markd.to_html("[open\n\nclose](url)").should eq("<p>[open</p>\n<p>close](url)</p>\n")
end

it "treats malformed entity references as text without swallowing inlines" do
Markd.to_html("&;").should eq("<p>&amp;;</p>\n")
Markd.to_html("x & *hi* &amp;").should eq("<p>x &amp; <em>hi</em> &amp;</p>\n")
end

it "decodes numeric entities above the basic multilingual plane" do
Markd.to_html("&#x1F600;").should eq("<p>😀</p>\n")
Markd.to_html("[x](&#x1F600;)").should eq(%(<p><a href="%F0%9F%98%80">x</a></p>\n))
Markd.to_html("&#x110000;").should eq("<p>�</p>\n")
end

it "preserves URI escapes in inline links, references, and autolinks" do
Markd.to_html("[x](a%2Fb)").should eq(%(<p><a href="a%2Fb">x</a></p>\n))
Markd.to_html("[r]: /a%3Fb%26c\n\n[r]").should eq(%(<p><a href="/a%3Fb%26c">r</a></p>\n))
Markd.to_html("<https://example.com/a%26b>").should eq(
%(<p><a href="https://example.com/a%26b">https://example.com/a%26b</a></p>\n)
)
end

it "resolves one-character labels" do
Markd.to_html("[r]: /url\n\n[x][r]").should eq(%(<p><a href="/url">x</a></p>\n))
end

it "collapses whitespace in reference labels" do
Markd.to_html("[a b]: /url\n\n[a b]").should eq(%(<p><a href="/url">a b</a></p>\n))
Markd.to_html("[a\tb]: /url\t\n\n[a b]").should eq(%(<p><a href="/url">a b</a></p>\n))
end

it "uses Unicode spaces and symbols to determine emphasis boundaries" do
Markd.to_html("a*💡*").should eq("<p>a*💡*</p>\n")
Markd.to_html("a\u2002*💡*").should eq("<p>a\u2002<em>💡</em></p>\n")
Markd.to_html("*a*\u00A0b").should eq("<p><em>a</em>\u00A0b</p>\n")
Markd.to_html("*a\u2002* b").should eq("<p>*a\u2002* b</p>\n")
end

it "recognizes Setext headings inside block quotes" do
Markd.to_html("> foo\n> ---").should eq("<blockquote>\n<h2>foo</h2>\n</blockquote>\n")
end

it "ends a table before an ATX heading that contains a pipe" do
options = Markd::Options.new(gfm: true)
html = Markd.to_html("A | B\n--- | ---\n# Heading | Text", options)
html.should eq("<table>\n<thead>\n<tr>\n<th>A</th>\n<th>B</th>\n</tr>\n</thead>\n</table>\n<h1>Heading | Text</h1>\n")
end

it "preserves UTF-8 characters while scanning autolinks" do
options = Markd::Options.new(autolink: true)
Markd.to_html("ä @foo", options).should eq("<p>ä @foo</p>\n")
end

it "emits punctuation and following text once after autolinks" do
options = Markd::Options.new(autolink: true)
Markd.to_html("www.example.com/...? tail", options).should eq(
%(<p><a href="http://www.example.com/">www.example.com/</a>...? tail</p>\n)
)
Markd.to_html("(http://www.example.com/path.) end", options).should eq(
%(<p>(<a href="http://www.example.com/path">http://www.example.com/path</a>.) end</p>\n)
)
end

it "links a scheme URL with a single-label host" do
options = Markd::Options.new(autolink: true)
Markd.to_html("**Autolink and http://inlines**", options).should eq(
%(<p><strong>Autolink and <a href="http://inlines">http://inlines</a></strong></p>\n)
)
end

it "links a one-character address after an embedded mailto prefix" do
options = Markd::Options.new(autolink: true)
Markd.to_html("xmailto:a@b.com", options).should eq(%(<p>xmailto:<a href="mailto:a@b.com">a@b.com</a></p>\n))
end

it "retains autolink alongside the pending fixture tag" do
fixtures = extract_spec_tests(File.join(__DIR__, "fixtures/gfm-regression.txt"))
fixtures["Regression tests"].values.find!(&.["markdown"].includes?("[^widely-cited]"))
.["test_tags"].split.should eq(%w[footnotes autolink pending])
end

it "keeps explicit links and autolinks from nesting" do
options = Markd::Options.new(autolink: true)
Markd.to_html("[www.example.com](/p)", options).should eq(%(<p><a href="/p">www.example.com</a></p>\n))
Markd.to_html("[label a@b.com](/p)", options).should eq(%(<p><a href="/p">label a@b.com</a></p>\n))
Markd.to_html("[see https://example.com/a](/p)", options).should eq(
%(<p><a href="/p">see https://example.com/a</a></p>\n)
)
Markd.to_html("[<https://example.com>](u)").should eq(
%(<p>[<a href="https://example.com">https://example.com</a>](u)</p>\n)
)
end

it "autolinks URLs inside literal brackets" do
options = Markd::Options.new(autolink: true)
Markd.to_html("[see www.example.com", options).should eq(
%(<p>[see <a href="http://www.example.com">www.example.com</a></p>\n)
)
Markd.to_html("[see https://example.com/a]", options).should eq(
%(<p>[see <a href="https://example.com/a">https://example.com/a</a>]</p>\n)
)
end
end
16 changes: 8 additions & 8 deletions spec/spec_helper.cr
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ def assert_section(file, section, examples, smart, gfm = false)
end

def assert_example(file, section, index, example, smart, gfm = false)
markdown = example["markdown"].gsub("→", "\t").chomp
markdown = example["markdown"].gsub("→", "\t")
html = example["html"].gsub("→", "\t")
line = example["line"].to_i
tags = example["test_tags"].split(" ")
Expand All @@ -54,7 +54,7 @@ def assert_example(file, section, index, example, smart, gfm = false)
)
options.smart = true if smart

if example["test_tags"].ends_with?("pending")
if tags.includes?("pending")
pending "- #{index}\n#{show_space(markdown)}", file, line do
output = Markd.to_html(markdown, options)
output.should eq(html), file: file, line: line
Expand Down Expand Up @@ -82,19 +82,19 @@ def extract_spec_tests(file)
line_number = 0
test_tags = ""

while (line = input.read_line)
while (raw_line = input.read_line(chomp: false))
line_number += 1
line = line.gsub(/\r\n?/, "\n")
line = raw_line.chomp
break if line.includes?("<!-- END TESTS -->")

if !test_start && !result_start && (match = line.match(/^\#{1,6}\s+(.*)$/))
current_section = match[1]
examples[current_section] = {} of Int32 => Hash(String, String)
example_count = 0
else
if !test_start && !result_start && line =~ /^`{32} example([a-z ])*$/
if !test_start && !result_start && (example = line.match(/^`{32} example((?: [a-z]+)*)$/))
test_start = true
test_tags = line[line.rindex!(' ') + 1..-1]
test_tags = example[1].strip
elsif test_start && !result_start && line =~ /^\.$/
test_start = false
result_start = true
Expand All @@ -106,10 +106,10 @@ def extract_spec_tests(file)
"line" => line_number.to_s,
"markdown" => "",
"html" => "",
"test_tags" => (test_tags == "example" ? "" : test_tags),
"test_tags" => test_tags,
} of String => String

examples[current_section][example_count]["markdown"] += line + "\n"
examples[current_section][example_count]["markdown"] += raw_line
elsif !test_start && result_start
examples[current_section][example_count]["html"] += line + "\n"
end
Expand Down
2 changes: 1 addition & 1 deletion src/markd/html_entities.cr
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ module Markd::HTMLEntities
end

def self.decode_codepoint(codepoint)
return "\uFFFD" if codepoint >= 0xD800 && codepoint <= 0xDFFF || codepoint > 0x10FFF
return "\uFFFD" if codepoint.in?(0xD800..0xDFFF) || codepoint > 0x10FFFF

if (decoded = Markd::HTMLEntities::DECODE_MAPPINGS[codepoint]?)
codepoint = decoded
Expand Down
12 changes: 3 additions & 9 deletions src/markd/parsers/block.cr
Original file line number Diff line number Diff line change
Expand Up @@ -67,17 +67,11 @@ module Markd::Parser
end

private def parse_blocks(source)
lines_size = 0
source.each_line do |line|
process_line(line)
lines_size += 1
end

# ignore last blank line created by final newline
lines_size -= 1 if source.ends_with?('\n')
source = source.gsub(/\r\n?/, '\n') if source.byte_index('\r')
source.each_line { |line| process_line(line) }

while (tip = tip?)
token(tip, lines_size)
token(tip, @current_line)
end
end

Expand Down
Loading
Loading