From 28d423e1f58d954713423910760ed2fd9be845cc Mon Sep 17 00:00:00 2001
From: CalebJohn <calebjohn4@gmail.com>
Date: Thu, 30 Jul 2020 20:07:34 -0600
Subject: [PATCH 1/3] Modify the codemirror linter plugin to fix katex

This stops the katex mode from trigger on a single $
This also allows the user to escape $ characters
---
 .../NoteBody/CodeMirror/utils/multiplex.js    | 132 ++++++++++++++++++
 .../CodeMirror/utils/useJoplinMode.ts         |  11 +-
 2 files changed, 141 insertions(+), 2 deletions(-)
 create mode 100644 ElectronClient/gui/NoteEditor/NoteBody/CodeMirror/utils/multiplex.js

diff --git a/ElectronClient/gui/NoteEditor/NoteBody/CodeMirror/utils/multiplex.js b/ElectronClient/gui/NoteEditor/NoteBody/CodeMirror/utils/multiplex.js
new file mode 100644
index 00000000000..62b55a9911a
--- /dev/null
+++ b/ElectronClient/gui/NoteEditor/NoteBody/CodeMirror/utils/multiplex.js
@@ -0,0 +1,132 @@
+// CodeMirror, copyright (c) by Marijn Haverbeke and others
+// Distributed under an MIT license: https://codemirror.net/LICENSE
+// Edited for the Joplin project
+// Main edits are within the token function there is an additional parameter to for the user
+// That allows them to specify a regex that will match a correctly formatted block
+// Additionally there is a check that disables blocks that begin with backslash
+
+function useMultiplexer(CodeMirror) {
+	CodeMirror.multiplexingMode = function(outer /* , others */) {
+		// Others should be {open, close, mode [, delimStyle] [, innerStyle]} objects
+		const others = Array.prototype.slice.call(arguments, 1);
+
+		function indexOf(string, pattern, from, returnEnd) {
+			if (typeof pattern == 'string') {
+				const found = string.indexOf(pattern, from);
+				return returnEnd && found > -1 ? found + pattern.length : found;
+			}
+			const m = pattern.exec(from ? string.slice(from) : string);
+			return m ? m.index + from + (returnEnd ? m[1].length : 0) : -1;
+		}
+
+		return {
+			startState: function() {
+				return {
+					outer: CodeMirror.startState(outer),
+					innerActive: null,
+					inner: null,
+				};
+			},
+
+			copyState: function(state) {
+				return {
+					outer: CodeMirror.copyState(outer, state.outer),
+					innerActive: state.innerActive,
+					inner: state.innerActive && CodeMirror.copyState(state.innerActive.mode, state.inner),
+				};
+			},
+
+			token: function(stream, state) {
+				if (!state.innerActive) {
+					let cutOff = Infinity;
+					const oldContent = stream.string;
+					for (let i = 0; i < others.length; ++i) {
+						const other = others[i];
+						// other.openCheck was added for Joplin
+						const found = indexOf(oldContent, other.openCheck ? other.openCheck : other.open, stream.pos);
+						// This first if clause was added to disable blocks that begin with backslash
+						if (found === stream.pos && stream.peek() === '\\') {
+							stream.eat('\\');
+							stream.match(other.open);
+						} else if (found == stream.pos) {
+							if (!other.parseDelimiters) stream.match(other.open);
+							state.innerActive = other;
+
+							// Get the outer indent, making sure to handle CodeMirror.Pass
+							let outerIndent = 0;
+							if (outer.indent) {
+								const possibleOuterIndent = outer.indent(state.outer, '', '');
+								if (possibleOuterIndent !== CodeMirror.Pass) outerIndent = possibleOuterIndent;
+							}
+
+							state.inner = CodeMirror.startState(other.mode, outerIndent);
+							return other.delimStyle && (`${other.delimStyle} ${other.delimStyle}-open`);
+						} else if (found != -1 && found < cutOff) {
+							cutOff = found;
+						}
+					}
+					if (cutOff != Infinity) stream.string = oldContent.slice(0, cutOff);
+					const outerToken = outer.token(stream, state.outer);
+					if (cutOff != Infinity) stream.string = oldContent;
+					return outerToken;
+				} else {
+					const curInner = state.innerActive, oldContent = stream.string;
+					if (!curInner.close && stream.sol()) {
+						state.innerActive = state.inner = null;
+						return this.token(stream, state);
+					}
+					const found = curInner.close ? indexOf(oldContent, curInner.close, stream.pos, curInner.parseDelimiters) : -1;
+					if (found == stream.pos && !curInner.parseDelimiters) {
+						stream.match(curInner.close);
+						state.innerActive = state.inner = null;
+						return curInner.delimStyle && (`${curInner.delimStyle} ${curInner.delimStyle}-close`);
+					}
+					if (found > -1) stream.string = oldContent.slice(0, found);
+					let innerToken = curInner.mode.token(stream, state.inner);
+					if (found > -1) stream.string = oldContent;
+
+					if (found == stream.pos && curInner.parseDelimiters) { state.innerActive = state.inner = null; }
+
+					if (curInner.innerStyle) {
+						if (innerToken) innerToken = `${innerToken} ${curInner.innerStyle}`;
+						else innerToken = curInner.innerStyle;
+					}
+
+					return innerToken;
+				}
+			},
+
+			indent: function(state, textAfter, line) {
+				const mode = state.innerActive ? state.innerActive.mode : outer;
+				if (!mode.indent) return CodeMirror.Pass;
+				return mode.indent(state.innerActive ? state.inner : state.outer, textAfter, line);
+			},
+
+			blankLine: function(state) {
+				const mode = state.innerActive ? state.innerActive.mode : outer;
+				if (mode.blankLine) {
+					mode.blankLine(state.innerActive ? state.inner : state.outer);
+				}
+				if (!state.innerActive) {
+					for (let i = 0; i < others.length; ++i) {
+						const other = others[i];
+						if (other.open === '\n') {
+							state.innerActive = other;
+							state.inner = CodeMirror.startState(other.mode, mode.indent ? mode.indent(state.outer, '', '') : 0);
+						}
+					}
+				} else if (state.innerActive.close === '\n') {
+					state.innerActive = state.inner = null;
+				}
+			},
+
+			electricChars: outer.electricChars,
+
+			innerMode: function(state) {
+				return state.inner ? { state: state.inner, mode: state.innerActive.mode } : { state: state.outer, mode: outer };
+			},
+		};
+	};
+}
+
+module.exports = { useMultiplexer };
diff --git a/ElectronClient/gui/NoteEditor/NoteBody/CodeMirror/utils/useJoplinMode.ts b/ElectronClient/gui/NoteEditor/NoteBody/CodeMirror/utils/useJoplinMode.ts
index 546f10c8c10..3c083a1863f 100644
--- a/ElectronClient/gui/NoteEditor/NoteBody/CodeMirror/utils/useJoplinMode.ts
+++ b/ElectronClient/gui/NoteEditor/NoteBody/CodeMirror/utils/useJoplinMode.ts
@@ -1,13 +1,20 @@
-import 'codemirror/addon/mode/multiplex';
 import 'codemirror/mode/stex/stex';
 
+// We use require here because multiplex is a javascript file
+// multiplex is copied from codemirror/addon/mode/multiplex and is orginally js
+const { useMultiplexer } = require('./multiplex');
+
 // Joplin markdown is a the same as markdown mode, but it has configured defaults
 // and support for katex math blocks
 export default function useJoplinMode(CodeMirror: any) {
+	useMultiplexer(CodeMirror);
+
 	CodeMirror.defineMode('joplin-markdown', (config: any) => {
 		const stex = CodeMirror.getMode(config, { name: 'stex', inMathMode: true });
 		const blocks = [{ open: '$$', close: '$$', mode: stex, delimStyle: 'katex-marker' },
-			{ open: '$', close: '$', mode: stex, delimStyle: 'katex-marker' }];
+			// This regex states that an inline katex block must have a closing deliminator to be valid
+			// it also has some stipulations about the surrounding characters
+			{ openCheck: /\\?\$\S[\s\S]*?[^\\\s]\$(?:\s|$)/, open: '$', close: '$', mode: stex, delimStyle: 'katex-marker' }];
 
 		const markdownOptions = {
 			name: 'markdown',

From 36cf0df0199cd9555ae735a1d65a35a2714bddb9 Mon Sep 17 00:00:00 2001
From: CalebJohn <calebjohn4@gmail.com>
Date: Sat, 1 Aug 2020 23:48:30 -0600
Subject: [PATCH 2/3] Rewrite the joplin mode to manually handle parsing

This gives us more control over katex parsing and the ability to upgrade
in the future
---
 .../NoteBody/CodeMirror/utils/multiplex.js    | 132 ------------------
 .../CodeMirror/utils/useJoplinMode.ts         | 109 +++++++++++++--
 ElectronClient/package.json                   |   2 +-
 3 files changed, 99 insertions(+), 144 deletions(-)
 delete mode 100644 ElectronClient/gui/NoteEditor/NoteBody/CodeMirror/utils/multiplex.js

diff --git a/ElectronClient/gui/NoteEditor/NoteBody/CodeMirror/utils/multiplex.js b/ElectronClient/gui/NoteEditor/NoteBody/CodeMirror/utils/multiplex.js
deleted file mode 100644
index 62b55a9911a..00000000000
--- a/ElectronClient/gui/NoteEditor/NoteBody/CodeMirror/utils/multiplex.js
+++ /dev/null
@@ -1,132 +0,0 @@
-// CodeMirror, copyright (c) by Marijn Haverbeke and others
-// Distributed under an MIT license: https://codemirror.net/LICENSE
-// Edited for the Joplin project
-// Main edits are within the token function there is an additional parameter to for the user
-// That allows them to specify a regex that will match a correctly formatted block
-// Additionally there is a check that disables blocks that begin with backslash
-
-function useMultiplexer(CodeMirror) {
-	CodeMirror.multiplexingMode = function(outer /* , others */) {
-		// Others should be {open, close, mode [, delimStyle] [, innerStyle]} objects
-		const others = Array.prototype.slice.call(arguments, 1);
-
-		function indexOf(string, pattern, from, returnEnd) {
-			if (typeof pattern == 'string') {
-				const found = string.indexOf(pattern, from);
-				return returnEnd && found > -1 ? found + pattern.length : found;
-			}
-			const m = pattern.exec(from ? string.slice(from) : string);
-			return m ? m.index + from + (returnEnd ? m[1].length : 0) : -1;
-		}
-
-		return {
-			startState: function() {
-				return {
-					outer: CodeMirror.startState(outer),
-					innerActive: null,
-					inner: null,
-				};
-			},
-
-			copyState: function(state) {
-				return {
-					outer: CodeMirror.copyState(outer, state.outer),
-					innerActive: state.innerActive,
-					inner: state.innerActive && CodeMirror.copyState(state.innerActive.mode, state.inner),
-				};
-			},
-
-			token: function(stream, state) {
-				if (!state.innerActive) {
-					let cutOff = Infinity;
-					const oldContent = stream.string;
-					for (let i = 0; i < others.length; ++i) {
-						const other = others[i];
-						// other.openCheck was added for Joplin
-						const found = indexOf(oldContent, other.openCheck ? other.openCheck : other.open, stream.pos);
-						// This first if clause was added to disable blocks that begin with backslash
-						if (found === stream.pos && stream.peek() === '\\') {
-							stream.eat('\\');
-							stream.match(other.open);
-						} else if (found == stream.pos) {
-							if (!other.parseDelimiters) stream.match(other.open);
-							state.innerActive = other;
-
-							// Get the outer indent, making sure to handle CodeMirror.Pass
-							let outerIndent = 0;
-							if (outer.indent) {
-								const possibleOuterIndent = outer.indent(state.outer, '', '');
-								if (possibleOuterIndent !== CodeMirror.Pass) outerIndent = possibleOuterIndent;
-							}
-
-							state.inner = CodeMirror.startState(other.mode, outerIndent);
-							return other.delimStyle && (`${other.delimStyle} ${other.delimStyle}-open`);
-						} else if (found != -1 && found < cutOff) {
-							cutOff = found;
-						}
-					}
-					if (cutOff != Infinity) stream.string = oldContent.slice(0, cutOff);
-					const outerToken = outer.token(stream, state.outer);
-					if (cutOff != Infinity) stream.string = oldContent;
-					return outerToken;
-				} else {
-					const curInner = state.innerActive, oldContent = stream.string;
-					if (!curInner.close && stream.sol()) {
-						state.innerActive = state.inner = null;
-						return this.token(stream, state);
-					}
-					const found = curInner.close ? indexOf(oldContent, curInner.close, stream.pos, curInner.parseDelimiters) : -1;
-					if (found == stream.pos && !curInner.parseDelimiters) {
-						stream.match(curInner.close);
-						state.innerActive = state.inner = null;
-						return curInner.delimStyle && (`${curInner.delimStyle} ${curInner.delimStyle}-close`);
-					}
-					if (found > -1) stream.string = oldContent.slice(0, found);
-					let innerToken = curInner.mode.token(stream, state.inner);
-					if (found > -1) stream.string = oldContent;
-
-					if (found == stream.pos && curInner.parseDelimiters) { state.innerActive = state.inner = null; }
-
-					if (curInner.innerStyle) {
-						if (innerToken) innerToken = `${innerToken} ${curInner.innerStyle}`;
-						else innerToken = curInner.innerStyle;
-					}
-
-					return innerToken;
-				}
-			},
-
-			indent: function(state, textAfter, line) {
-				const mode = state.innerActive ? state.innerActive.mode : outer;
-				if (!mode.indent) return CodeMirror.Pass;
-				return mode.indent(state.innerActive ? state.inner : state.outer, textAfter, line);
-			},
-
-			blankLine: function(state) {
-				const mode = state.innerActive ? state.innerActive.mode : outer;
-				if (mode.blankLine) {
-					mode.blankLine(state.innerActive ? state.inner : state.outer);
-				}
-				if (!state.innerActive) {
-					for (let i = 0; i < others.length; ++i) {
-						const other = others[i];
-						if (other.open === '\n') {
-							state.innerActive = other;
-							state.inner = CodeMirror.startState(other.mode, mode.indent ? mode.indent(state.outer, '', '') : 0);
-						}
-					}
-				} else if (state.innerActive.close === '\n') {
-					state.innerActive = state.inner = null;
-				}
-			},
-
-			electricChars: outer.electricChars,
-
-			innerMode: function(state) {
-				return state.inner ? { state: state.inner, mode: state.innerActive.mode } : { state: state.outer, mode: outer };
-			},
-		};
-	};
-}
-
-module.exports = { useMultiplexer };
diff --git a/ElectronClient/gui/NoteEditor/NoteBody/CodeMirror/utils/useJoplinMode.ts b/ElectronClient/gui/NoteEditor/NoteBody/CodeMirror/utils/useJoplinMode.ts
index 3c083a1863f..ea895005a4f 100644
--- a/ElectronClient/gui/NoteEditor/NoteBody/CodeMirror/utils/useJoplinMode.ts
+++ b/ElectronClient/gui/NoteEditor/NoteBody/CodeMirror/utils/useJoplinMode.ts
@@ -1,22 +1,18 @@
+import 'codemirror/addon/mode/multiplex';
 import 'codemirror/mode/stex/stex';
 
-// We use require here because multiplex is a javascript file
-// multiplex is copied from codemirror/addon/mode/multiplex and is orginally js
-const { useMultiplexer } = require('./multiplex');
-
 // Joplin markdown is a the same as markdown mode, but it has configured defaults
 // and support for katex math blocks
 export default function useJoplinMode(CodeMirror: any) {
-	useMultiplexer(CodeMirror);
 
 	CodeMirror.defineMode('joplin-markdown', (config: any) => {
 		const stex = CodeMirror.getMode(config, { name: 'stex', inMathMode: true });
-		const blocks = [{ open: '$$', close: '$$', mode: stex, delimStyle: 'katex-marker' },
-			// This regex states that an inline katex block must have a closing deliminator to be valid
-			// it also has some stipulations about the surrounding characters
-			{ openCheck: /\\?\$\S[\s\S]*?[^\\\s]\$(?:\s|$)/, open: '$', close: '$', mode: stex, delimStyle: 'katex-marker' }];
+		// const blocks = [{ open: '$$', close: '$$', mode: stex, delimStyle: 'katex-marker' },
+		//	// This regex states that an inline katex block must have a closing deliminator to be valid
+		//	// it also has some stipulations about the surrounding characters
+		//	{ open: /^\s\$(?=[^\s\$].*?[^\\\s\$]\$\s)/, close: /\$(?=\s)/, mode: stex, delimStyle: 'katex-marker' }];
 
-		const markdownOptions = {
+		const markdownConfig = {
 			name: 'markdown',
 			taskLists: true,
 			strikethrough: true,
@@ -26,7 +22,98 @@ export default function useJoplinMode(CodeMirror: any) {
 			},
 		};
 
-		return CodeMirror.multiplexingMode(CodeMirror.getMode(config, markdownOptions), ...blocks);
+		const markdownMode = CodeMirror.getMode(config, markdownConfig);
+
+		const inlineKatexOpenRE = /(?<!\S)\$(?=[^\s$].*?[^\\\s$]\$(?!\S))/;
+		const inlineKatexCloseRE = /(?<![\\\s$])\$(?!\S)/;
+		const blockKatexRE = /(?<!\\)\$\$/;
+
+		// Find token will search for a valid katex start or end token
+		// If found then it will return the index, otherwise -1
+		function findToken(stream: any, token: RegExp) {
+			const match = token.exec(stream.string.slice(stream.pos));
+
+			return match ? match.index + stream.pos : -1;
+		}
+
+		return {
+			startState: function(): {outer: any, openCharacter: string, inner: any} {
+				return {
+					outer: CodeMirror.startState(markdownMode),
+					openCharacter: '',
+					inner: CodeMirror.startState(stex),
+				};
+			},
+
+			copyState: function(state: any) {
+				return {
+					outer: CodeMirror.copyState(markdownMode, state.outer),
+					openCharacter: state.openCharacter,
+					inner: CodeMirror.copyState(stex, state.inner),
+				};
+			},
+
+			token: function(stream: any, state: any) {
+				if (!state.openCharacter) {
+					let nextTokenPos = stream.string.length;
+					const blockPos = findToken(stream, blockKatexRE);
+					if (blockPos !== -1) nextTokenPos = blockPos;
+					const inlinePos = findToken(stream, inlineKatexOpenRE);
+					if (inlinePos !== -1 && inlinePos < nextTokenPos) nextTokenPos = inlinePos;
+
+					if (nextTokenPos === stream.pos) {
+						if (blockPos === stream.pos) state.openCharacter = '$$';
+						if (inlinePos === stream.pos) state.openCharacter = '$';
+						stream.match(state.openCharacter);
+						return 'katex-marker-open';
+					}
+					// If we found a token in this stream but haven;t reached it yet, then we will
+					// pass all the characters up to our token to markdown mode
+					const oldString = stream.string;
+					stream.string = oldString.slice(0, nextTokenPos);
+					const token = markdownMode.token(stream, state.outer);
+					stream.string = oldString;
+					return token;
+				} else {
+					let nextTokenPos = stream.string.length;
+					if (state.openCharacter === '$$') { nextTokenPos = findToken(stream, blockKatexRE); } else if (state.openCharacter === '$') { nextTokenPos = findToken(stream, inlineKatexCloseRE); }
+
+					if (nextTokenPos < 0) nextTokenPos = stream.string.length;
 
+					if (nextTokenPos === stream.pos) {
+						stream.match(state.openCharacter);
+						state.openCharacter = '';
+						return 'katex-marker-close';
+					}
+					// If we found a token in this stream but haven;t reached it yet, then we will
+					// pass all the characters up to our token to markdown mode
+					const oldString = stream.string;
+					stream.string = oldString.slice(0, nextTokenPos);
+					const token = stex.token(stream, state.inner);
+					stream.string = oldString;
+					return token;
+				}
+			},
+
+			indent: function(state: any, textAfter: string, line: any) {
+				const mode = state.openCharacter ? stex : markdownMode;
+				if (!mode.indent) return CodeMirror.Pass;
+				return mode.indent(state.openCharacter ? state.inner : state.outer, textAfter, line);
+			},
+
+			blankLine: function(state: any) {
+				const mode = state.openCharacter ? stex : markdownMode;
+				if (mode.blankLine) {
+					mode.blankLine(state.openCharacter ? state.inner : state.outer);
+				}
+			},
+
+			electricChars: markdownMode.electricChars,
+
+			innerMode: function(state: any) {
+				return state.openCharacter ? { state: state.inner, mode: stex } : { state: state.outer, mode: markdownMode };
+			},
+
+		};
 	});
 }
diff --git a/ElectronClient/package.json b/ElectronClient/package.json
index 1bf235b76e9..5697723a72c 100644
--- a/ElectronClient/package.json
+++ b/ElectronClient/package.json
@@ -104,7 +104,7 @@
     "base64-stream": "^1.0.0",
     "chokidar": "^3.0.0",
     "clean-html": "^1.5.0",
-    "codemirror": "^5.54.0",
+    "codemirror": "^5.56.0",
     "color": "^3.1.2",
     "compare-versions": "^3.2.1",
     "countable": "^3.0.1",

From a73139172500dcd496c5bf48a4f33d3b6d506c75 Mon Sep 17 00:00:00 2001
From: CalebJohn <calebjohn4@gmail.com>
Date: Sun, 2 Aug 2020 00:10:53 -0600
Subject: [PATCH 3/3] clean up joplin mode katex parsing

---
 .../CodeMirror/utils/useJoplinMode.ts         | 83 ++++++++++---------
 1 file changed, 42 insertions(+), 41 deletions(-)

diff --git a/ElectronClient/gui/NoteEditor/NoteBody/CodeMirror/utils/useJoplinMode.ts b/ElectronClient/gui/NoteEditor/NoteBody/CodeMirror/utils/useJoplinMode.ts
index ea895005a4f..7c1b3e2c6f8 100644
--- a/ElectronClient/gui/NoteEditor/NoteBody/CodeMirror/utils/useJoplinMode.ts
+++ b/ElectronClient/gui/NoteEditor/NoteBody/CodeMirror/utils/useJoplinMode.ts
@@ -6,12 +6,6 @@ import 'codemirror/mode/stex/stex';
 export default function useJoplinMode(CodeMirror: any) {
 
 	CodeMirror.defineMode('joplin-markdown', (config: any) => {
-		const stex = CodeMirror.getMode(config, { name: 'stex', inMathMode: true });
-		// const blocks = [{ open: '$$', close: '$$', mode: stex, delimStyle: 'katex-marker' },
-		//	// This regex states that an inline katex block must have a closing deliminator to be valid
-		//	// it also has some stipulations about the surrounding characters
-		//	{ open: /^\s\$(?=[^\s\$].*?[^\\\s\$]\$\s)/, close: /\$(?=\s)/, mode: stex, delimStyle: 'katex-marker' }];
-
 		const markdownConfig = {
 			name: 'markdown',
 			taskLists: true,
@@ -23,6 +17,7 @@ export default function useJoplinMode(CodeMirror: any) {
 		};
 
 		const markdownMode = CodeMirror.getMode(config, markdownConfig);
+		const stex = CodeMirror.getMode(config, { name: 'stex', inMathMode: true });
 
 		const inlineKatexOpenRE = /(?<!\S)\$(?=[^\s$].*?[^\\\s$]\$(?!\S))/;
 		const inlineKatexCloseRE = /(?<![\\\s$])\$(?!\S)/;
@@ -54,45 +49,51 @@ export default function useJoplinMode(CodeMirror: any) {
 			},
 
 			token: function(stream: any, state: any) {
-				if (!state.openCharacter) {
-					let nextTokenPos = stream.string.length;
-					const blockPos = findToken(stream, blockKatexRE);
-					if (blockPos !== -1) nextTokenPos = blockPos;
+				let currentMode = markdownMode;
+				let currentState = state.outer;
+				let tokenLabel = 'katex-marker-open';
+				let nextTokenPos = stream.string.length;
+				let closing = false;
+
+				const blockPos = findToken(stream, blockKatexRE);
+
+				if (state.openCharacter) {
+					currentMode = stex;
+					currentState = state.inner;
+					tokenLabel = 'katex-marker-close';
+					closing = true;
+
+					const inlinePos = findToken(stream, inlineKatexCloseRE);
+
+					if (state.openCharacter === '$$' && blockPos !== -1) nextTokenPos = blockPos;
+					if (state.openCharacter === '$' && inlinePos !== -1) nextTokenPos = inlinePos;
+				} else {
 					const inlinePos = findToken(stream, inlineKatexOpenRE);
+
+					if (blockPos !== -1) nextTokenPos = blockPos;
 					if (inlinePos !== -1 && inlinePos < nextTokenPos) nextTokenPos = inlinePos;
 
-					if (nextTokenPos === stream.pos) {
-						if (blockPos === stream.pos) state.openCharacter = '$$';
-						if (inlinePos === stream.pos) state.openCharacter = '$';
-						stream.match(state.openCharacter);
-						return 'katex-marker-open';
-					}
-					// If we found a token in this stream but haven;t reached it yet, then we will
-					// pass all the characters up to our token to markdown mode
-					const oldString = stream.string;
-					stream.string = oldString.slice(0, nextTokenPos);
-					const token = markdownMode.token(stream, state.outer);
-					stream.string = oldString;
-					return token;
-				} else {
-					let nextTokenPos = stream.string.length;
-					if (state.openCharacter === '$$') { nextTokenPos = findToken(stream, blockKatexRE); } else if (state.openCharacter === '$') { nextTokenPos = findToken(stream, inlineKatexCloseRE); }
-
-					if (nextTokenPos < 0) nextTokenPos = stream.string.length;
-
-					if (nextTokenPos === stream.pos) {
-						stream.match(state.openCharacter);
-						state.openCharacter = '';
-						return 'katex-marker-close';
-					}
-					// If we found a token in this stream but haven;t reached it yet, then we will
-					// pass all the characters up to our token to markdown mode
-					const oldString = stream.string;
-					stream.string = oldString.slice(0, nextTokenPos);
-					const token = stex.token(stream, state.inner);
-					stream.string = oldString;
-					return token;
+					if (blockPos === stream.pos) state.openCharacter = '$$';
+					if (inlinePos === stream.pos) state.openCharacter = '$';
 				}
+
+				if (nextTokenPos === stream.pos) {
+					stream.match(state.openCharacter);
+
+					if (closing) state.openCharacter = '';
+
+					return tokenLabel;
+				}
+
+				// If we found a token in this stream but haven;t reached it yet, then we will
+				// pass all the characters leading up to our token to markdown mode
+				const oldString = stream.string;
+
+				stream.string = oldString.slice(0, nextTokenPos);
+				const token = currentMode.token(stream, currentState);
+				stream.string = oldString;
+
+				return token;
 			},
 
 			indent: function(state: any, textAfter: string, line: any) {