parse: fix parsing of non-empty blank lines

Closes GH-473. Closes GH-474. Reviewed-by: Victor Felder <[email protected]> Reviewed-by: Titus Wormer <[email protected]>
remarkjs · Mar 27, 2020 · b4c993e · b4c993e
1 parent 121688c
commit b4c993e
Show file tree

Hide file tree

Showing 7 changed files with 94 additions and 56 deletions.
diff --git a/packages/remark-parse/lib/parser.js b/packages/remark-parse/lib/parser.js
@@ -97,7 +97,7 @@ proto.interruptBlockquote = [
 
 // Handlers.
 proto.blockTokenizers = {
-  newline: require('./tokenize/newline'),
+  blankLine: require('./tokenize/blank-line'),
   indentedCode: require('./tokenize/code-indented'),
   fencedCode: require('./tokenize/code-fenced'),
   blockquote: require('./tokenize/blockquote'),

diff --git a/packages/remark-parse/lib/tokenize/blank-line.js b/packages/remark-parse/lib/tokenize/blank-line.js
@@ -0,0 +1,38 @@
+'use strict'
+
+// A line containing no characters, or a line containing only spaces (U+0020) or tabs (U+0009), is called a blank line.
+// See https://spec.commonmark.org/0.29/#blank-line
+var reBlankLine = /^[\u0020\t]*(\n|$)/
+
+// NOTE: Though blank lines play a special role in lists to determine whether the list is tight or loose  (https://spec.commonmark.org/0.29/#blank-lines),
+// it's done by the list tokenizer and this blank-line tokenizer does not have to be responsible for that.
+// Therefore, configs such as `blankLine.notInList` do not have to be set here.
+module.exports = blankLine
+
+function blankLine(eat, value, silent) {
+  var match
+  var subvalue = ''
+  var index = 0
+  var length = value.length
+
+  while (index < length) {
+    match = reBlankLine.exec(value.slice(index))
+    if (match == null) {
+      break
+    }
+
+    index += match[0].length
+    subvalue += match[0]
+  }
+
+  if (subvalue === '') {
+    return
+  }
+
+  /* istanbul ignore if - never used (yet) */
+  if (silent) {
+    return true
+  }
+
+  eat(subvalue)
+}
diff --git a/packages/remark-parse/lib/tokenize/newline.js b/packages/remark-parse/lib/tokenize/newline.js
diff --git a/packages/remark-parse/lib/tokenize/paragraph.js b/packages/remark-parse/lib/tokenize/paragraph.js
@@ -96,12 +96,6 @@ function paragraph(eat, value, silent) {
 
   subvalue = value.slice(0, index)
 
-  if (trim(subvalue) === '') {
-    eat(subvalue)
-
-    return null
-  }
-
   /* istanbul ignore if - never used (yet) */
   if (silent) {
     return true

diff --git a/packages/remark-parse/readme.md b/packages/remark-parse/readme.md
@@ -252,7 +252,7 @@ Precedence of default block methods is as follows:
 
 <!--methods-block start-->
 
-*   `newline`
+*   `blankLine`
 *   `indentedCode`
 *   `fencedCode`
 *   `blockquote`

diff --git a/test/fixtures/input/isolated-hard-break.text b/test/fixtures/input/isolated-hard-break.text
@@ -0,0 +1,2 @@
+
+The previous line, which contains 2 spaces, should be considered to be empty.
diff --git a/test/fixtures/tree/isolated-hard-break.json b/test/fixtures/tree/isolated-hard-break.json
@@ -0,0 +1,52 @@
+{
+  "type": "root",
+  "children": [
+    {
+      "type": "paragraph",
+      "children": [
+        {
+          "type": "text",
+          "value": "The previous line, which contains 2 spaces, should be considered to be empty.",
+          "position": {
+            "start": {
+              "line": 2,
+              "column": 1,
+              "offset": 3
+            },
+            "end": {
+              "line": 2,
+              "column": 78,
+              "offset": 80
+            },
+            "indent": []
+          }
+        }
+      ],
+      "position": {
+        "start": {
+          "line": 2,
+          "column": 1,
+          "offset": 3
+        },
+        "end": {
+          "line": 2,
+          "column": 78,
+          "offset": 80
+        },
+        "indent": []
+      }
+    }
+  ],
+  "position": {
+    "start": {
+      "line": 1,
+      "column": 1,
+      "offset": 0
+    },
+    "end": {
+      "line": 3,
+      "column": 1,
+      "offset": 81
+    }
+  }
+}
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@

		The previous line, which contains 2 spaces, should be considered to be empty.