🐎 Speed up token rendering a bit

By using a smaller function with lower memory footprint and that v8 can
optimize.

There’s still room for improvements in that area.

lib/mixins/canvas-drawer.js

@@ -410,55 +410,18 @@ export default class CanvasDrawer extends Mixin {
    * @return {Array<Array>} An array of tokens by line
    * @access private
    */
-  tokenLinesForScreenRows (startRow, endRow) {
+  eachTokenForScreenRows (startRow, endRow, callback) {
     const editor = this.getTextEditor()
-    let tokenLines = []
-    if (typeof editor.tokenizedLinesForScreenRows === 'function') {
-      const tokenizedLines = editor.tokenizedLinesForScreenRows(startRow, endRow)
-      for (let tokenizedLine of tokenizedLines) {
-        if (tokenizedLine) {
-          const invisibleRegExp = this.getInvisibleRegExpForLine(tokenizedLine)
-          tokenLines.push(tokenizedLine.tokens.map((token) => {
-            return {
-              value: token.value.replace(invisibleRegExp, ' '),
-              scopes: token.scopes.slice()
-            }
-          }))
-        } else {
-          return {
-            value: '',
-            scopes: []
-          }
-        }
-      }
-    } else {
-      const displayLayer = editor.displayLayer
-      const invisibleRegExp = this.getInvisibleRegExp()
-      const screenLines = displayLayer.getScreenLines(startRow, endRow)
-      for (let {lineText, tagCodes} of screenLines) {
-        let tokens = []
-        let scopes = []
-        let textIndex = 0
-        // console.log(lineText, invisibleRegExp, lineText.replace(invisibleRegExp, ' '))
-        for (let tagCode of tagCodes) {
-          if (displayLayer.isOpenTagCode(tagCode)) {
-            scopes.push(displayLayer.tagForCode(tagCode))
-          } else if (displayLayer.isCloseTagCode(tagCode)) {
-            scopes.pop()
-          } else {
-            let value = lineText.substr(textIndex, tagCode)
-            if (invisibleRegExp) {
-              value = value.replace(invisibleRegExp, ' ')
-            }
-            tokens.push({ value: value, scopes: scopes.slice() })
-            textIndex += tagCode
-          }
-        }
-
-        tokenLines.push(tokens)
-      }
+    const invisibleRegExp = this.getInvisibleRegExp()
+
+    for (let row = startRow; row < endRow; row++) {
+      editor.tokensForScreenRow(row).forEach(token => {
+        callback(row, {
+          text: token.text.replace(invisibleRegExp, ' '),
+          scopes: token.scopes
+        })
+      })
     }
-    return tokenLines
   }
 
   /**
@@ -484,33 +447,29 @@ export default class CanvasDrawer extends Mixin {
     const context = this.tokensLayer.context
     const {width: canvasWidth} = this.tokensLayer.getSize()
 
-    if (typeof this.tokenLinesForScreenRows !== 'function') {
-      console.error(`tokenLinesForScreenRows should be a function but it was ${typeof this.tokenLinesForScreenRows}`, this.tokenLinesForScreenRows)
-
-      return
-    }
-
-    const screenRowsTokens = this.tokenLinesForScreenRows(firstRow, lastRow)
-
-    let y = offsetRow * lineHeight
-    for (let i = 0; i < screenRowsTokens.length; i++) {
-      let tokens = screenRowsTokens[i]
-      let x = 0
-      context.clearRect(x, y, canvasWidth, lineHeight)
-      for (let j = 0; j < tokens.length; j++) {
-        let token = tokens[j]
-        if (/^\s+$/.test(token.value)) {
-          x += token.value.length * charWidth
-        } else {
-          const color = displayCodeHighlights ? this.getTokenColor(token) : this.getDefaultColor()
-          x = this.drawToken(context, token.value, color, x, y, charWidth, charHeight)
-        }
-        if (x > canvasWidth) { break }
+    let lastLine, x
+    let y = (offsetRow * lineHeight) - lineHeight
+    this.eachTokenForScreenRows(firstRow, lastRow, (line, token) => {
+      if (lastLine !== line) {
+        x = 0
+        y += lineHeight
+        lastLine = line
+        context.clearRect(x, y, canvasWidth, lineHeight)
       }
+      if (x > canvasWidth) { return }
 
-      y += lineHeight
-    }
+      if (/^\s+$/.test(token.text)) {
+        x += token.text.length * charWidth
+      } else {
+        const color = displayCodeHighlights
+          ? this.getTokenColor(token)
+          : this.getDefaultColor()
 
+        x = this.drawToken(
+          context, token.text, color, x, y, charWidth, charHeight
+        )
+      }
+    })
     context.fill()
   }
 
@@ -534,29 +493,6 @@ export default class CanvasDrawer extends Mixin {
     }).map(_.escapeRegExp).join('|'), 'g')
   }
 
-  /**
-   * Returns the regexp to replace invisibles substitution characters
-   * in editor lines.
-   *
-   * @param  {Object} line the tokenized line
-   * @return {RegExp} the regular expression to match invisible characters
-   * @deprecated Is used only to support Atom version before display layer API
-   * @access private
-   */
-  getInvisibleRegExpForLine (line) {
-    if ((line != null) && (line.invisibles != null)) {
-      const invisibles = []
-      if (line.invisibles.cr != null) { invisibles.push(line.invisibles.cr) }
-      if (line.invisibles.eol != null) { invisibles.push(line.invisibles.eol) }
-      if (line.invisibles.space != null) { invisibles.push(line.invisibles.space) }
-      if (line.invisibles.tab != null) { invisibles.push(line.invisibles.tab) }
-
-      return RegExp(invisibles.filter((s) => {
-        return typeof s === 'string'
-      }).map(_.escapeRegExp).join('|'), 'g')
-    }
-  }
-
   /**
    * Draws a single token on the given context.
    *