diff --git a/spec/markd/lexer/blockquote_spec.cr b/spec/markd/lexer/blockquote_spec.cr deleted file mode 100644 index 49c2e6545bed..000000000000 --- a/spec/markd/lexer/blockquote_spec.cr +++ /dev/null @@ -1,56 +0,0 @@ -require "../../spec_helper" - -describe Markd::Lexer do - assert_lexer_render "> # Foo\n> bar\n> baz", [{ - "type" => :blockquote_start - }, - { - "type" => :heading, - "level" => 1, - "text" => "Foo" - }, - { - "type" => :paragraph, - "text" => "bar\nbaz" - }, - { - "type" => :blockquote_end - }] - - assert_lexer_render "> # Foo\n>bar\n> baz", [{ - "type" => :blockquote_start - }, - { - "type" => :heading, - "level" => 1, - "text" => "Foo" - }, - { - "type" => :paragraph, - "text" => "bar\nbaz" - }, - { - "type" => :blockquote_end - }] - - assert_lexer_render "> # Foo\n> bar\n > baz", [{ - "type" => :blockquote_start - }, - { - "type" => :heading, - "level" => 1, - "text" => "Foo" - }, - { - "type" => :paragraph, - "text" => "bar\nbaz" - }, - { - "type" => :blockquote_end - }] - - assert_lexer_render " > # Foo\n > bar\n > baz", [{ - "type" => :code, - "text" => "> # Foo\n> bar\n> baz" - }] -end \ No newline at end of file diff --git a/spec/markd/lexer/code_spec.cr b/spec/markd/lexer/code_spec.cr deleted file mode 100644 index 42318bcd5a15..000000000000 --- a/spec/markd/lexer/code_spec.cr +++ /dev/null @@ -1,42 +0,0 @@ -require "../../spec_helper" - -describe Markd::Lexer do - assert_lexer_render " echo hello world", [{ - "type" => :code, - "text" => "echo hello world" - }] - - assert_lexer_render " echo hello world\n pwd", [{ - "type" => :code, - "text" => "echo hello world\npwd" - }] - - assert_lexer_render "Hello World\n\n echo hello world", [{ - "type" => :paragraph, - "text" => "Hello World" - }, - { - "type" => :code, - "text" => "echo hello world" - }] - - assert_lexer_render " echo hello world", [ - { - "type" => :code, - "text" => "echo hello world" - }] - - assert_lexer_render " echo hello world\n\nHello Earch\n\n echo hello crystal", [ - { - "type" => :code, - "text" => "echo hello world" - }, - { - "type" => :paragraph, - "text" => "Hello Earch" - }, - { - "type" => :code, - "text" => "echo hello crystal" - }] -end diff --git a/spec/markd/lexer/common_lexer/blockquote_spec.cr b/spec/markd/lexer/common_lexer/blockquote_spec.cr new file mode 100644 index 000000000000..3dbe03bd6cf9 --- /dev/null +++ b/spec/markd/lexer/common_lexer/blockquote_spec.cr @@ -0,0 +1,56 @@ +require "../../../spec_helper" + +describe Markd::Lexer do + assert_common_lexer_render "> # Foo\n> bar\n> baz", [{ + "type" => "blockquote_start" + }, + { + "type" => "heading", + "level" => 1, + "text" => "Foo" + }, + { + "type" => "paragraph", + "text" => "bar\nbaz" + }, + { + "type" => "blockquote_end" + }] + + assert_common_lexer_render "> # Foo\n>bar\n> baz", [{ + "type" => "blockquote_start" + }, + { + "type" => "heading", + "level" => 1, + "text" => "Foo" + }, + { + "type" => "paragraph", + "text" => "bar\nbaz" + }, + { + "type" => "blockquote_end" + }] + + assert_common_lexer_render "> # Foo\n> bar\n > baz", [{ + "type" => "blockquote_start" + }, + { + "type" => "heading", + "level" => 1, + "text" => "Foo" + }, + { + "type" => "paragraph", + "text" => "bar\nbaz" + }, + { + "type" => "blockquote_end" + }] + + assert_common_lexer_render " > # Foo\n > bar\n > baz", [{ + "type" => "code", + "text" => "> # Foo\n> bar\n> baz" + }] +end \ No newline at end of file diff --git a/spec/markd/lexer/common_lexer/code_spec.cr b/spec/markd/lexer/common_lexer/code_spec.cr new file mode 100644 index 000000000000..6519d0e85f6d --- /dev/null +++ b/spec/markd/lexer/common_lexer/code_spec.cr @@ -0,0 +1,42 @@ +require "../../../spec_helper" + +describe Markd::Lexer do + assert_common_lexer_render " echo hello world", [{ + "type" => "code", + "text" => "echo hello world" + }] + + assert_common_lexer_render " echo hello world\n pwd", [{ + "type" => "code", + "text" => "echo hello world\npwd" + }] + + assert_common_lexer_render "Hello World\n\n echo hello world", [{ + "type" => "paragraph", + "text" => "Hello World" + }, + { + "type" => "code", + "text" => "echo hello world" + }] + + assert_common_lexer_render " echo hello world", [ + { + "type" => "code", + "text" => "echo hello world" + }] + + assert_common_lexer_render " echo hello world\n\nHello Earch\n\n echo hello crystal", [ + { + "type" => "code", + "text" => "echo hello world" + }, + { + "type" => "paragraph", + "text" => "Hello Earch" + }, + { + "type" => "code", + "text" => "echo hello crystal" + }] +end diff --git a/spec/markd/lexer/common_lexer/fences_spec.cr b/spec/markd/lexer/common_lexer/fences_spec.cr new file mode 100644 index 000000000000..70d01f98acdc --- /dev/null +++ b/spec/markd/lexer/common_lexer/fences_spec.cr @@ -0,0 +1,50 @@ +require "../../../spec_helper" + +describe Markd::Lexer do + assert_common_lexer_render "```\necho hello world\n```", [{ + "type" => "code", + "text" => "echo hello world" + }] + + assert_common_lexer_render "```bash\necho hello world\npwd```", [{ + "type" => "code", + "lang" => "bash", + "text" => "echo hello world\npwd", + }] + + assert_common_lexer_render "`````\nhi ther `` ok ```\n`````", [{ + "type" => "code", + "text" => "hi ther `` ok ```" + }] + + assert_common_lexer_render "```\n<\n >\n```", [{ + "type" => "code", + "text" => "<\n >" + }] + + assert_common_lexer_render "~~~\n<\n >\n~~~", [{ + "type" => "code", + "text" => "<\n >" + }] + + assert_common_lexer_render "```\naaa\n~~~\n```", [{ + "type" => "code", + "text" => "aaa\n~~~" + }] + + assert_common_lexer_render "~~~\naaa\n```\n~~~", [{ + "type" => "code", + "text" => "aaa\n```" + }] + + # TODO: fix it + # assert_common_lexer_render "````\naaa\n```\n``````", [{ + # "type" => "code", + # "text" => "aaa\n```" + # }] + + # assert_common_lexer_render "~~~~\naaa\n~~~\n~~~~~", [{ + # "type" => "code", + # "text" => "aaa\n~~~" + # }] +end diff --git a/spec/markd/lexer/common_lexer/heading_spec.cr b/spec/markd/lexer/common_lexer/heading_spec.cr new file mode 100644 index 000000000000..bf40776b1532 --- /dev/null +++ b/spec/markd/lexer/common_lexer/heading_spec.cr @@ -0,0 +1,57 @@ +require "../../../spec_helper" + +describe Markd::Lexer do + assert_common_lexer_render "# Heading 1", [{ + "type" => "heading", + "level" => 1, + "text" => "Heading 1", + }] + + assert_common_lexer_render "## Heading 2", [{ + "type" => "heading", + "level" => 2, + "text" => "Heading 2", + }] + + assert_common_lexer_render "###### Heading 6", [{ + "type" => "heading", + "level" => 6, + "text" => "Heading 6", + }] + + assert_common_lexer_render "## Heading 2\n### Heading 3", [{ + "type" => "heading", + "level" => 2, + "text" => "Heading 2", + }, + { + "type" => "heading", + "level" => 3, + "text" => "Heading 3", + }] + + assert_common_lexer_render "####### Heading 7", [{ + "type" => "paragraph", + "text" => "####### Heading 7", + }] + + assert_common_lexer_render "#Heading 1", [{ + "type" => "paragraph", + "text" => "#Heading 1", + }] + + assert_common_lexer_render "#Heading 1", [{ + "type" => "paragraph", + "text" => "#Heading 1", + }] + + assert_common_lexer_render "#Heading 1", [{ + "type" => "paragraph", + "text" => "#Heading 1", + }] + + assert_common_lexer_render "#Heading 1\n#Heading 2", [{ + "type" => "paragraph", + "text" => "#Heading 1\n#Heading 2", + }] +end diff --git a/spec/markd/lexer/common_lexer/hr_spec.cr b/spec/markd/lexer/common_lexer/hr_spec.cr new file mode 100644 index 000000000000..137eb74254bb --- /dev/null +++ b/spec/markd/lexer/common_lexer/hr_spec.cr @@ -0,0 +1,75 @@ +require "../../../spec_helper" + +describe Markd::Lexer do + assert_common_lexer_render "---\n___", [{ + "type" => "hr" + }, + { + "type" => "hr" + }] + + assert_common_lexer_render "+++", [{ + "type" => "paragraph", + "text" => "+++" + }] + + assert_common_lexer_render "===", [{ + "type" => "paragraph", + "text" => "===" + }] + + assert_common_lexer_render "--\n**\n__", [{ + "type" => "paragraph", + "text" => "--\n**\n__" + }] + + assert_common_lexer_render " ***\n ***\n ***", [{ + "type" => "hr" + }, + { + "type" => "hr" + }, + { + "type" => "hr" + }] + + assert_common_lexer_render " ***", [{ + "type" => "code", + "text" => "***" + }] + + assert_common_lexer_render "Foo\n ***", [{ + "type" => "paragraph", + "text" => "Foo\n ***" + }] + + assert_common_lexer_render "_____________________________________\n - - -\n ** * ** * ** * **\n- - - -\n- - - - ", [{ + "type" => "hr" + }, + { + "type" => "hr" + }, + { + "type" => "hr" + }, + { + "type" => "hr" + }, + { + "type" => "hr" + }] + + assert_common_lexer_render "_ _ _ _ a\n\na------\n\n---a---", [{ + "type" => "paragraph", + "text" => "_ _ _ _ a" + }, + { + "type" => "paragraph", + "text" => "a------" + }, + { + "type" => "paragraph", + "text" => "---a---" + }] + +end diff --git a/spec/markd/lexer/common_lexer/lheading_spec.cr b/spec/markd/lexer/common_lexer/lheading_spec.cr new file mode 100644 index 000000000000..2029781d632a --- /dev/null +++ b/spec/markd/lexer/common_lexer/lheading_spec.cr @@ -0,0 +1,68 @@ +require "../../../spec_helper" + +describe Markd::Lexer do + assert_common_lexer_render "Heading 1\n=========\n", [{ + "type" => "heading", + "level" => 1, + "text" => "Heading 1", + }] + + assert_common_lexer_render "Heading 2\n---------\n", [{ + "type" => "heading", + "level" => 2, + "text" => "Heading 2", + }] + + assert_common_lexer_render " Heading 2\n---", [{ + "type" => "heading", + "level" => 2, + "text" => "Heading 2", + }] + + assert_common_lexer_render " Heading 2\n---", [{ + "type" => "heading", + "level" => 2, + "text" => "Heading 2", + }] + + assert_common_lexer_render " Heading 2\n ---", [{ + "type" => "heading", + "level" => 2, + "text" => "Heading 2", + }] + + assert_common_lexer_render "Heading 2\n --- ", [{ + "type" => "heading", + "level" => 2, + "text" => "Heading 2", + }] + + assert_common_lexer_render "Foo \n-----", [{ + "type" => "heading", + "level" => 2, + "text" => "Foo", + }] + + assert_common_lexer_render "Foo\\\n-----", [{ + "type" => "heading", + "level" => 2, + "text" => "Foo\\", + }] + + + # TODO: fix it + # assert_common_lexer_render "Heading 2\n ----", [{ + # "type" => "paragraph", + # "text" => "Heading 2\n----", + # }] + + assert_common_lexer_render "Foo\n= =\n\nFoo\n--- -", [{ + "type" => "paragraph", + "text" => "Foo\n= =", + }, + { + "type" => "paragraph", + "text" => "Foo\n--- -", + }] + +end diff --git a/spec/markd/lexer/common_lexer/list_spec.cr b/spec/markd/lexer/common_lexer/list_spec.cr new file mode 100644 index 000000000000..539bf90a24c6 --- /dev/null +++ b/spec/markd/lexer/common_lexer/list_spec.cr @@ -0,0 +1,96 @@ +require "../../../spec_helper" + +describe Markd::Lexer do + assert_common_lexer_render "- one\n\n two", [{ + "type" => "list_start", + "style" => "unordered" + }, + { + "type" => "list_item_start" + }, + { + "type" => "text", + "text" => "one" + }, + { + "type" => "space", + }, + { + "type" => "text", + "text" => "two", + }, + { + "type" => "list_item_end" + }, + { + "type" => "list_end" + }] + + assert_common_lexer_render "- one\n\n two", [{ + "type" => "list_start", + "style" => "unordered" + }, + { + "type" => "list_item_start" + }, + { + "type" => "text", + "text" => "one" + }, + { + "type" => "space", + }, + { + "type" => "text", + "text" => "two", + }, + { + "type" => "list_item_end" + }, + { + "type" => "list_end" + }] + + assert_common_lexer_render "123456789. ok", [{ + "type" => "list_start", + "style" => "ordered" + }, + { + "type" => "list_item_start" + }, + { + "type" => "text", + "text" => "ok" + }, + { + "type" => "list_item_end" + }, + { + "type" => "list_end" + }] + + assert_common_lexer_render "-one\n\n2.two", [{ + "type" => "paragraph", + "text" => "-one" + }, + { + "type" => "paragraph", + "text" => "2.two" + }] + + assert_common_lexer_render "1. foo\n\n ```\n bar\n ```\n\n baz\n\n > bam", [ + {"type" => "list_start", "style" => "ordered"}, + {"type" => "list_item_start"}, + {"type" => "text", "text" => "foo"}, + {"type" => "space"}, + {"type" => "code", "text" => "bar"}, + {"type" => "text", "text" => "baz"}, + {"type" => "space"}, + {"type" => "blockquote_start"}, + {"type" => "paragraph", "text" => "bam"}, + {"type" => "blockquote_end"}, + {"type" => "list_item_end"}, + {"type" => "list_end"} + ] + +end \ No newline at end of file diff --git a/spec/markd/lexer/common_lexer/paragraph_spec.cr b/spec/markd/lexer/common_lexer/paragraph_spec.cr new file mode 100644 index 000000000000..f34e56cdd709 --- /dev/null +++ b/spec/markd/lexer/common_lexer/paragraph_spec.cr @@ -0,0 +1,60 @@ +require "../../../spec_helper" + +describe Markd::Lexer do + assert_common_lexer_render "", [] of Markd::Lexer::Token + + assert_common_lexer_render "Hello", [{ + "type" => "paragraph", + "text" => "Hello", + }] + + assert_common_lexer_render "\n\nHello", [ + { + "type" => "space", + }, + { + "type" => "paragraph", + "text" => "Hello", + }, + ] + + assert_common_lexer_render "Hello\nWorld", [{ + "type" => "paragraph", + "text" => "Hello\nWorld", + }] + + assert_common_lexer_render "Hello\n\nWorld", [{ + "type" => "paragraph", + "text" => "Hello", + }, + { + "type" => "paragraph", + "text" => "World", + }] + + assert_common_lexer_render "Hello\n\n\n\n\nWorld", [{ + "type" => "paragraph", + "text" => "Hello", + }, + { + "type" => "paragraph", + "text" => "World", + }] + + assert_common_lexer_render "Hello\n \nWorld", [{ + "type" => "paragraph", + "text" => "Hello", + }, + { + "type" => "paragraph", + "text" => "World", + }] + assert_common_lexer_render "Hello\nWorld\n\nGood\nBye", [{ + "type" => "paragraph", + "text" => "Hello\nWorld", + }, + { + "type" => "paragraph", + "text" => "Good\nBye", + }] +end diff --git a/spec/markd/lexer/fences_spec.cr b/spec/markd/lexer/fences_spec.cr deleted file mode 100644 index e9dfc3f20ad6..000000000000 --- a/spec/markd/lexer/fences_spec.cr +++ /dev/null @@ -1,50 +0,0 @@ -require "../../spec_helper" - -describe Markd::Lexer do - assert_lexer_render "```\necho hello world\n```", [{ - "type" => :code, - "text" => "echo hello world" - }] - - assert_lexer_render "```bash\necho hello world\npwd```", [{ - "type" => :code, - "lang" => "bash", - "text" => "echo hello world\npwd", - }] - - assert_lexer_render "`````\nhi ther `` ok ```\n`````", [{ - "type" => :code, - "text" => "hi ther `` ok ```" - }] - - assert_lexer_render "```\n<\n >\n```", [{ - "type" => :code, - "text" => "<\n >" - }] - - assert_lexer_render "~~~\n<\n >\n~~~", [{ - "type" => :code, - "text" => "<\n >" - }] - - assert_lexer_render "```\naaa\n~~~\n```", [{ - "type" => :code, - "text" => "aaa\n~~~" - }] - - assert_lexer_render "~~~\naaa\n```\n~~~", [{ - "type" => :code, - "text" => "aaa\n```" - }] - - # TODO: fix it - # assert_lexer_render "````\naaa\n```\n``````", [{ - # "type" => :code, - # "text" => "aaa\n```" - # }] - - # assert_lexer_render "~~~~\naaa\n~~~\n~~~~~", [{ - # "type" => :code, - # "text" => "aaa\n~~~" - # }] -end diff --git a/spec/markd/lexer/heading_spec.cr b/spec/markd/lexer/heading_spec.cr deleted file mode 100644 index 862406f06460..000000000000 --- a/spec/markd/lexer/heading_spec.cr +++ /dev/null @@ -1,58 +0,0 @@ -require "../../spec_helper" -require "markdown" - -describe Markd::Lexer do - assert_lexer_render "# Heading 1", [{ - "type" => :heading, - "level" => 1, - "text" => "Heading 1", - }] - - assert_lexer_render "## Heading 2", [{ - "type" => :heading, - "level" => 2, - "text" => "Heading 2", - }] - - assert_lexer_render "###### Heading 6", [{ - "type" => :heading, - "level" => 6, - "text" => "Heading 6", - }] - - assert_lexer_render "## Heading 2\n### Heading 3", [{ - "type" => :heading, - "level" => 2, - "text" => "Heading 2", - }, - { - "type" => :heading, - "level" => 3, - "text" => "Heading 3", - }] - - assert_lexer_render "####### Heading 7", [{ - "type" => :paragraph, - "text" => "####### Heading 7", - }] - - assert_lexer_render "#Heading 1", [{ - "type" => :paragraph, - "text" => "#Heading 1", - }] - - assert_lexer_render "#Heading 1", [{ - "type" => :paragraph, - "text" => "#Heading 1", - }] - - assert_lexer_render "#Heading 1", [{ - "type" => :paragraph, - "text" => "#Heading 1", - }] - - assert_lexer_render "#Heading 1\n#Heading 2", [{ - "type" => :paragraph, - "text" => "#Heading 1\n#Heading 2", - }] -end diff --git a/spec/markd/lexer/hr_spec.cr b/spec/markd/lexer/hr_spec.cr deleted file mode 100644 index 820cf450e21c..000000000000 --- a/spec/markd/lexer/hr_spec.cr +++ /dev/null @@ -1,75 +0,0 @@ -require "../../spec_helper" - -describe Markd::Lexer do - assert_lexer_render "---\n___", [{ - "type" => :hr - }, - { - "type" => :hr - }] - - assert_lexer_render "+++", [{ - "type" => :paragraph, - "text" => "+++" - }] - - assert_lexer_render "===", [{ - "type" => :paragraph, - "text" => "===" - }] - - assert_lexer_render "--\n**\n__", [{ - "type" => :paragraph, - "text" => "--\n**\n__" - }] - - assert_lexer_render " ***\n ***\n ***", [{ - "type" => :hr - }, - { - "type" => :hr - }, - { - "type" => :hr - }] - - assert_lexer_render " ***", [{ - "type" => :code, - "text" => "***" - }] - - assert_lexer_render "Foo\n ***", [{ - "type" => :paragraph, - "text" => "Foo\n ***" - }] - - assert_lexer_render "_____________________________________\n - - -\n ** * ** * ** * **\n- - - -\n- - - - ", [{ - "type" => :hr - }, - { - "type" => :hr - }, - { - "type" => :hr - }, - { - "type" => :hr - }, - { - "type" => :hr - }] - - assert_lexer_render "_ _ _ _ a\n\na------\n\n---a---", [{ - "type" => :paragraph, - "text" => "_ _ _ _ a" - }, - { - "type" => :paragraph, - "text" => "a------" - }, - { - "type" => :paragraph, - "text" => "---a---" - }] - -end diff --git a/spec/markd/lexer/lheading_spec.cr b/spec/markd/lexer/lheading_spec.cr deleted file mode 100644 index 81aff87651bd..000000000000 --- a/spec/markd/lexer/lheading_spec.cr +++ /dev/null @@ -1,69 +0,0 @@ -require "../../spec_helper" -require "markdown" - -describe Markd::Lexer do - assert_lexer_render "Heading 1\n=========\n", [{ - "type" => :heading, - "level" => 1, - "text" => "Heading 1", - }] - - assert_lexer_render "Heading 2\n---------\n", [{ - "type" => :heading, - "level" => 2, - "text" => "Heading 2", - }] - - assert_lexer_render " Heading 2\n---", [{ - "type" => :heading, - "level" => 2, - "text" => "Heading 2", - }] - - assert_lexer_render " Heading 2\n---", [{ - "type" => :heading, - "level" => 2, - "text" => "Heading 2", - }] - - assert_lexer_render " Heading 2\n ---", [{ - "type" => :heading, - "level" => 2, - "text" => "Heading 2", - }] - - assert_lexer_render "Heading 2\n --- ", [{ - "type" => :heading, - "level" => 2, - "text" => "Heading 2", - }] - - assert_lexer_render "Foo \n-----", [{ - "type" => :heading, - "level" => 2, - "text" => "Foo", - }] - - assert_lexer_render "Foo\\\n-----", [{ - "type" => :heading, - "level" => 2, - "text" => "Foo\\", - }] - - - # TODO: fix it - # assert_lexer_render "Heading 2\n ----", [{ - # "type" => :paragraph, - # "text" => "Heading 2\n----", - # }] - - assert_lexer_render "Foo\n= =\n\nFoo\n--- -", [{ - "type" => :paragraph, - "text" => "Foo\n= =", - }, - { - "type" => :paragraph, - "text" => "Foo\n--- -", - }] - -end diff --git a/spec/markd/lexer/paragraph_spec.cr b/spec/markd/lexer/paragraph_spec.cr deleted file mode 100644 index 37c928a8e4ff..000000000000 --- a/spec/markd/lexer/paragraph_spec.cr +++ /dev/null @@ -1,60 +0,0 @@ -require "../../spec_helper" - -describe Markd::Lexer do - assert_lexer_render "", [] of Markd::Lexer::Token - - assert_lexer_render "Hello", [{ - "type" => :paragraph, - "text" => "Hello", - }] - - assert_lexer_render "\n\nHello", [ - { - "type" => :space, - }, - { - "type" => :paragraph, - "text" => "Hello", - }, - ] - - assert_lexer_render "Hello\nWorld", [{ - "type" => :paragraph, - "text" => "Hello\nWorld", - }] - - assert_lexer_render "Hello\n\nWorld", [{ - "type" => :paragraph, - "text" => "Hello", - }, - { - "type" => :paragraph, - "text" => "World", - }] - - assert_lexer_render "Hello\n\n\n\n\nWorld", [{ - "type" => :paragraph, - "text" => "Hello", - }, - { - "type" => :paragraph, - "text" => "World", - }] - - assert_lexer_render "Hello\n \nWorld", [{ - "type" => :paragraph, - "text" => "Hello", - }, - { - "type" => :paragraph, - "text" => "World", - }] - assert_lexer_render "Hello\nWorld\n\nGood\nBye", [{ - "type" => :paragraph, - "text" => "Hello\nWorld", - }, - { - "type" => :paragraph, - "text" => "Good\nBye", - }] -end diff --git a/spec/spec_helper.cr b/spec/spec_helper.cr index fd5c89d99600..f92eec955517 100644 --- a/spec/spec_helper.cr +++ b/spec/spec_helper.cr @@ -1,8 +1,17 @@ require "spec" require "../src/markd" -def assert_lexer_render(input, output, file = __FILE__, line = __LINE__) +def assert_common_lexer_render(input, output, file = __FILE__, line = __LINE__) it "renders #{input.inspect}", file, line do - Markd::Lexer.new(input).lex.should eq(output), file, line + context = Markd::Lexer::Context.new(input) + lexer = Markd::CommonLexer.new.call(context) + + document = Markd::Lexer::Document.new + context.document.each do |token| + token.delete("source") + document.push(token) + end + + document.should eq(output), file, line end end \ No newline at end of file diff --git a/src/markd.cr b/src/markd.cr index fc3c916d1e34..1ce87caabb01 100644 --- a/src/markd.cr +++ b/src/markd.cr @@ -1,5 +1,11 @@ -require "./markd/*" +require "./markd/lexer" +require "./markd/renderer" +require "./markd/parser" +require "./markd/version" module Markd - # TODO Put your code here + def self.to_html(src, renderer = HTMLRenderer.new) + parser = Parser.new(src) + parser.parser(renderer) + end end diff --git a/src/markd/lexer.cr b/src/markd/lexer.cr index 3c014feb6727..63a80fe68392 100644 --- a/src/markd/lexer.cr +++ b/src/markd/lexer.cr @@ -1,188 +1,36 @@ -require "html" - module Markd - class Lexer - alias Token = Hash(String, Symbol | String | Int32) - - module Rule - # BULLET = /(?:[*+-]|\d+\.)/ + module Lexer + alias Token = Hash(String, String | Int32 | Bool) + alias Document = Array(Token) - # LIST_HR = /\\n+(?=\\1?(?:[-*_] *){3,}(?:\\n+|$))/ - # LIST_DEF = /\\n+(?=#{BULLET})/ + property next : Lexer | Nil - NEWLINE = /^\n+/ - CODE = /^( {4}[^\n]+\n*)+/ - FENCES = /^ *(`{3,}|~{3,})[ \.]*(\S+)? *\n([\s\S]*?)\s*\1 *(?:\n+|$)/ - HEADING = /^ *(\#{1,6}) +([^\n]+?) *#* *(?:\n+|$)/ - LHEADING = /^([^\n]+)\n *(=|-){2,} *(?:\n+|$)/ - HR = /^( *[-*_]){3,} *(?:\n+|$)/ + abstract def call(context : Context) - INLINE_LINK = /^ *\[([^\]]+)\]: *<?([^\s>]+)>?(?: +["(]([^\n]+)[")])? *(?:\n+|$)/ - BLOCKQUOTE = /^( *>[^\n]+(\n(?!#{INLINE_LINK})[^\n]+)*\n*)+/ - # LIST = /^( *)(#{BULLET}) [\s\S]+?(?:#{LIST_HR}|#{LIST_DEF}|\n{2,}(?! )(?!\1#{BULLET} )\n*|\s*$)/ - # ITEM = /^( *)(#{BULLET}) [^\n]*(?:\n(?!\1#{BULLET} )[^\n]*)*/m - # HTML = /^ *(?:comment *(?:\n|\s*$)|closed *(?:\n{2,}|\s*$)|closing *(?:\n{2,}|\s*$))/ - - PARAGRAPH = /^((?:[^\n]+\n?(?!hr|heading|lheading|blockquote|tag|def))+)\n*/ - TEXT = /^[^\n]+/ - end - - struct Block - property newline, code, fences, heading, lheading, hr, blockquote, paragraph, text - - def initialize - @newline = Rule::NEWLINE - @code = Rule::CODE - @fences = Rule::FENCES - @heading = Rule::HEADING - @lheading = Rule::LHEADING - @hr = Rule::HR - @blockquote = Rule::BLOCKQUOTE - # @list = Rule::LIST - # @html = Rule::HTML - # @inline_link = Rule::INLINE_LINK - @paragraph = Rule::PARAGRAPH - @text = Rule::TEXT + def call_next(context : Context) + if next_handler = @next + next_handler.call(context) end end - @src : String - @rules = Block.new - @tokens = [] of Token - - def initialize(src) - @src = src.gsub(/\r\n|\r/, "\n") - .gsub(/\t/, " ") - .gsub(/\x{00a0}/, " ") - .gsub(/\x{2424}/, "\n") - end - - def lex - token(@src, top: true) - end - - def token(src, top = false, bq = false) - src = src.gsub(/^ +$/m, "") - - while src - break if src.empty? - - # newline - if match = @rules.newline.match(src) - src = delete_match_text(src, match) - if match[0].size > 1 - @tokens.push({ - "type" => :space, - }) - end - end - - # indented code - if match = @rules.code.match(src) - src = delete_match_text(src, match) - text = match[0].gsub(/^ {4}/m, "") - @tokens.push({ - "type" => :code, - "text" => text_escape(text.strip), - }) - next - end - - # fences code - if match = @rules.fences.match(src) - src = delete_match_text(src, match) - token = { - "type" => :code, - "text" => text_escape(match[3]), - } - token["lang"] = match[2].downcase if match[2]? - @tokens.push(token) - next - end - - # ATX heading - if match = @rules.heading.match(src) - src = delete_match_text(src, match) - @tokens.push({ - "type" => :heading, - "level" => match[1].size, - "text" => match[2], - }) - next - end - - # setext heading - if match = @rules.lheading.match(src) - src = delete_match_text(src, match) - @tokens.push({ - "type" => :heading, - "level" => match[2] == "=" ? 1 : 2, - "text" => match[1].strip, - }) - next - end - - # thematic break(hr) - if match = @rules.hr.match(src) - src = delete_match_text(src, match) - @tokens.push({ - "type" => :hr, - }) - next - end - - # blockquote - if match = @rules.blockquote.match(src) - src = delete_match_text(src, match) - - @tokens.push({ - "type" => :blockquote_start, - }) - - text = match[0].gsub(/^ *> ?/m, "") - token(text, top, bq: true) - - @tokens.push({ - "type" => :blockquote_end, - }) - - next - end - - # top-level paragraph - if top && (match = src.match(@rules.paragraph)) - src = delete_match_text(src, match) - @tokens.push({ - "type" => :paragraph, - "text" => match[1].strip, - }) - next - end - - # text - if match = src.match(@rules.text) - # Top-level should never reach here. - src = delete_match_text(src, match) - @tokens.push({ - "type" => :text, - "text" => match[0], - }) - next - end - end - - @tokens + def text_clean(src) + src.gsub(/\r\n|\r/, "\n") + .gsub(/\t/, " ") + .gsub(/\x{00a0}/, " ") + .gsub(/\x{2424}/, "\n") end - protected def text_escape(src) + def text_escape(src) src.gsub("&", "&") .gsub("<", "<") .gsub(">", ">") .gsub("\"", """) end - protected def delete_match_text(src, match, index = 0) + def delete_match(src, match, index = 0) src[match[index].size..-1] end end end + +require "./lexers/*" diff --git a/src/markd/lexers/common.cr b/src/markd/lexers/common.cr new file mode 100644 index 000000000000..23c54096720d --- /dev/null +++ b/src/markd/lexers/common.cr @@ -0,0 +1,286 @@ +module Markd + class CommonLexer + include Lexer + + struct Block + property newline, code, fences, heading, lheading, hr, blockquote, list, item, html, + paragraph, text + + HTML_COMMENT = /<!---->|<!--(?:-?[^>-])(?:-?[^-])*-->/ + HTML_TAG = /(?!(?:a|em|strong|small|s|cite|q|dfn|abbr|data|time|code|var|samp|kbd|sub|sup|i|b|u|mark|ruby|rt|rp|bdi|bdo|span|br|wbr|ins|del|img)\b)\w+(?!:\/|[^\w\s@]*@)\b/ + HTML_CLOSED_TAG = /<(#{HTML_TAG})[\s\S]+?<\/\1>/ + HTML_CLOSING_TAG = /<#{HTML_TAG}(?:"[^"]*"|'[^']*'|[^'">])*?>/ + + NEWLINE = /^\n+/ + CODE = /^( {4}[^\n]+\n*)+/ + FENCES = /^ *(`{3,}|~{3,})[ \.]*(\S+)? *\n([\s\S]*?)\s*\1 *(?:\n+|$)/ + HEADING = /^ *(\#{1,6}) +([^\n]+?) *#* *(?:\n+|$)/ + LHEADING = /^([^\n]+)\n *(=|-){2,} *(?:\n+|$)/ + HR = /^( *[-*_]){3,} *(?:\n+|$)/ + INLINE_LINK = /^ *\[([^\]]+)\]: *<?([^\s>]+)>?(?: +["(]([^\n]+)[")])? *(?:\n+|$)/ + BLOCKQUOTE = /^( *>[^\n]+(\n(?!#{INLINE_LINK})[^\n]+)*\n*)+/ + PARAGRAPH = /^((?:[^\n]+\n?(?!#{HR}|#{HEADING}|#{LHEADING}|#{BLOCKQUOTE}|<#{HTML_TAG}|#{INLINE_LINK}))+)\n*/ + TEXT = /^[^\n]+/ + + BULLET = /(?:[*+-]|\d+\.)/ + LIST_HR = /\\n+(?=\\1?(?:[-*_] *){3,}(?:\\n+|$))/ + LIST_DEF = /\\n+(?=#{BULLET})/ + LIST = /^( *)(#{BULLET}) [\s\S]+?(?:#{LIST_HR}|#{LIST_DEF}|\n{2,}(?! )(?!\1#{BULLET} )\n*|\s*$)/ + ITEM = /^(\s*(?:#{BULLET})) ([^\n]*)(?:\n(?!\1#{BULLET} )[^\n]*)*/m + + HTML = /^ *(?:#{HTML_COMMENT} *(?:\n|\s*$)|#{HTML_CLOSED_TAG} *(?:\n{2,}|\s*$)|#{HTML_CLOSING_TAG} *(?:\n{2,}|\s*$))/m + + def initialize + @newline = NEWLINE + @code = CODE + @fences = FENCES + @heading = HEADING + @lheading = LHEADING + @hr = HR + @blockquote = BLOCKQUOTE + @list = LIST + @item = ITEM + @html = HTML + @paragraph = PARAGRAPH + @text = TEXT + end + end + + @src : String + @document = Document.new + @rules = Block.new + + def initialize(@src = "") + end + + def call(context : Context) + @src = text_clean(context.source) + token(@src, top: true) + + context.document = @document + call_next(context) + end + + def token(src : String, top = false, bq = false) + src = src.gsub(/^ +$/m, "") + + while src + break if src.empty? + + # newline + if match = @rules.newline.match(src) + src = newline(src, match) + end + + # indented code + if match = @rules.code.match(src) + src = code(src, match) + next + end + + # fences code + if match = @rules.fences.match(src) + src = fences(src, match) + next + end + + # ATX heading + if match = @rules.heading.match(src) + src = heading(src, match) + next + end + + # setext heading + if match = @rules.lheading.match(src) + src = lheading(src, match) + next + end + + # thematic break(hr) + if match = @rules.hr.match(src) + src = hr(src, match) + next + end + + # blockquote + if match = @rules.blockquote.match(src) + src = blockquote(src, match) + next + end + + # list + if match = @rules.list.match(src) + src = list(src, match) + next + end + + # html + # if match = @rules.html.match(src) + # src = html(src, match) + # next + # end + + # top-level paragraph + if top && (match = src.match(@rules.paragraph)) + src = paragraph(src, match) + next + end + + # text + if match = @rules.text.match(src) + # Top-level should never reach here. + src = delete_match(src, match) + @document.push({ + "type" => "text", + "source" => match[0], + "text" => match[0], + }) + next + end + + break + end + + @document + end + + def newline(src : String, match : Regex::MatchData) + if match[0].size > 1 + @document.push({ + "type" => "space", + "source" => match[0], + }) + end + + delete_match(src, match) + end + + def code(src : String, match : Regex::MatchData) + text = match[0].gsub(/^ {4}/m, "") + @document.push({ + "type" => "code", + "source" => match[0], + "text" => text_escape(text.strip), + }) + + delete_match(src, match) + end + + def fences(src : String, match : Regex::MatchData) + token = { + "type" => "code", + "source" => match[0], + "text" => text_escape(match[3]), + } + token["lang"] = match[2].downcase if match[2]? + @document.push(token) + + delete_match(src, match) + end + + def heading(src : String, match : Regex::MatchData) + @document.push({ + "type" => "heading", + "source" => match[0], + "level" => match[1].size, + "text" => match[2], + }) + + delete_match(src, match) + end + + def lheading(src : String, match : Regex::MatchData) + @document.push({ + "type" => "heading", + "source" => match[0], + "level" => match[2] == "=" ? 1 : 2, + "text" => match[1].strip, + }) + + delete_match(src, match) + end + + def hr(src : String, match : Regex::MatchData) + @document.push({ + "type" => "hr", + "source" => match[0], + }) + + delete_match(src, match) + end + + def blockquote(src : String, match : Regex::MatchData) + @document.push({ + "type" => "blockquote_start", + }) + text = match[0].gsub(/^ *> ?/m, "") + token(text, true, bq: true) + @document.push({ + "type" => "blockquote_end", + }) + + delete_match(src, match) + end + + def list(src : String, match : Regex::MatchData) + bullet = match[2] + + @document.push({ + "type" => "list_start", + "style" => (bullet.size > 1) ? "ordered" : "unordered" + }) + + # to_next = false + items = match[0].scan(@rules.item) + items.each do |item| + full_space = item[0].size + bullet_space = item[1].size + + text = item[0].gsub(/^( *)(?:#{Block::BULLET}) +/, "") + if text.includes?("\n ") + full_space -= text.size + text = text.gsub(/^ {1,#{full_space}}/m, "") + end + + # loose = + + @document.push({ + "type" => "list_item_start", + }) + + token(text, false) + + @document.push({ + "type" => "list_item_end", + }) + end + + @document.push({ + "type" => "list_end", + }) + + delete_match(src, match) + end + + def html(src : String, match : Regex::MatchData) + @document.push({ + "type" => "html", + # "pre" => ["pre", "script", "style"].includes?(match[1]), + "source" => match[0], + "text" => text_escape(match[0]), + }) + + delete_match(src, match) + end + + def paragraph(src : String, match : Regex::MatchData) + @document.push({ + "type" => "paragraph", + "source" => match[0], + "text" => match[1].strip, + }) + + delete_match(src, match) + end + end +end diff --git a/src/markd/lexers/context.cr b/src/markd/lexers/context.cr new file mode 100644 index 000000000000..ea016311a77f --- /dev/null +++ b/src/markd/lexers/context.cr @@ -0,0 +1,13 @@ +module Markd + module Lexer + class Context + getter source : String + + property document : Document + + # :nodoc: + def initialize(@source : String, @document = Document.new) + end + end + end +end diff --git a/src/markd/lexers/inline.cr b/src/markd/lexers/inline.cr new file mode 100644 index 000000000000..07de38e1c696 --- /dev/null +++ b/src/markd/lexers/inline.cr @@ -0,0 +1,94 @@ +module Markd + class InlineLexer + include Lexer + + struct Block + property strong, text + + STRONG = /^__([\s\S]+?)__(?!_)|^\*\*([\s\S]+?)\*\*(?!\*)/ + TEXT = /^[^\n]+/ + + def initialize + @strong = STRONG + @text = TEXT + end + end + + @rules = Block.new + @document = Document.new + @tokens = Document.new + + def initialize(@src = "") + end + + def call(context : Context) + @document = context.document + + @document.each_with_index do |token, i| + @tokens = Document.new + + case token["type"] + when "paragraph" + paragraph(token, i) + end + end + + context.document = @document + call_next(context) + end + + def paragraph(token : Token, index : Int32) + @document[index] = { + "type" => "paragaph_start", + } + + token(token["text"], top: true).each_with_index do | new_token, shift| + @document.insert(index + shift + 1, new_token) + end + + @document.insert(index + @tokens.size + 1, { + "type" => "paragaph_end", + }) + end + + def lex(token : Token, index : Int32) + token(token.text, top: true) + end + + def token(src, top = false) + src = src.to_s.gsub(/^ +$/m, "") + + while src + break if src.empty? + + # strong + if match = @rules.strong.match(src) + + src = delete_match(src, match) + @tokens.push({ + "type" => "strong", + "source" => match[0], + "text" => match[2], + }) + next + end + + # text + if match = @rules.text.match(src) + # Top-level should never reach here. + src = delete_match(src, match) + @tokens.push({ + "type" => "text", + "source" => match[0], + "text" => match[0], + }) + next + end + + break + end + + @tokens + end + end +end diff --git a/src/markd/parser.cr b/src/markd/parser.cr new file mode 100644 index 000000000000..efbb2f999e5d --- /dev/null +++ b/src/markd/parser.cr @@ -0,0 +1,45 @@ +module Markd + class Parser + getter context + + def initialize(source : String, lexers = [] of Lexer) + @context = Lexer::Context.new(source) + lexers = default_lexers if lexers.empty? + lexer = build_lexer(lexers) + lexer.call(@context) + end + + # def parse(renderer = HTMLRenderer.new) + # out = "" + # while next_token + # out += tok + # end + # end + + # def next_token + # @tokens.pop + # end + + # def peek_token + # @tokens[@tokens.size - 1] || nil; + # end + + # def tok + # case @token["type"] + # when :space + # next + # end + # end + + def build_lexer(lexers) + raise ArgumentError.new "You must specify at least one Markd Lexer." if lexers.empty? + + 0.upto(lexers.size - 2) { |i| lexers[i].next = lexers[i + 1] } + lexers.first + end + + private def default_lexers + [CommonLexer.new.as(Lexer), InlineLexer.new.as(Lexer)] + end + end +end \ No newline at end of file diff --git a/src/markd/renderer.cr b/src/markd/renderer.cr new file mode 100644 index 000000000000..4a46d48b1278 --- /dev/null +++ b/src/markd/renderer.cr @@ -0,0 +1,7 @@ +module Markd + abstract class Renderer + abstract def render(token) + end +end + +require "./renderers/*" \ No newline at end of file diff --git a/src/markd/renderers/html_renderer.cr b/src/markd/renderers/html_renderer.cr new file mode 100644 index 000000000000..4c8acc46f7f7 --- /dev/null +++ b/src/markd/renderers/html_renderer.cr @@ -0,0 +1,6 @@ +module Markd + class HTMLRenderer < Renderer + def render(token) + end + end +end \ No newline at end of file