fix: columns position incorrect after lexer recovery

fixes #1969
Chevrotain · Aug 14, 2023 · d50420d · d50420d
1 parent 4cf1ac7
commit d50420d
Show file tree

Hide file tree

Showing 3 changed files with 33 additions and 0 deletions.
diff --git a/packages/chevrotain/src/scan/lexer_public.ts b/packages/chevrotain/src/scan/lexer_public.ts
@@ -732,6 +732,7 @@ export class Lexer {
         }
 
         errLength = offset - errorStartOffset;
+        column = this.computeNewColumn(column!, errLength);
         // at this point we either re-synced or reached the end of the input text
         msg = this.config.errorMessageProvider.buildUnexpectedCharactersMessage(
           orgText,

diff --git a/packages/chevrotain/test/scan/lexer_spec.ts b/packages/chevrotain/test/scan/lexer_spec.ts
@@ -277,6 +277,32 @@ function defineLexerSpecs(
         expect(result.tokens[0].image).to.equal("A");
         expect(result.tokens[0].startOffset).to.equal(0);
       });
+
+      // reproduce bug: https://github.com/Chevrotain/chevrotain/issues/1969
+      it("will report the correct column after lexer error recovery", () => {
+        const digits = createToken({
+          name: "digit",
+          pattern: /\d+/,
+        });
+
+        const myLexer = new Lexer([digits], {
+          positionTracking: "onlyStart",
+        });
+        const input = "--123++";
+        const result = myLexer.tokenize(input);
+
+        expect(result.errors).to.have.lengthOf(2);
+        expect(result.errors[0].message).to.include("-");
+        expect(result.errors[0].column).to.equal(1);
+        expect(result.errors[1].message).to.include("+");
+        expect(result.errors[1].column).to.equal(6);
+
+        expect(result.tokens).to.have.lengthOf(1);
+        const digitsToken = result.tokens[0];
+        expect(tokenMatcher(digitsToken, digits)).to.be.true;
+        expect(digitsToken.image).to.equal("123");
+        expect(digitsToken.startColumn).to.equal(3);
+      });
     });
 
     const ValidNaPattern = createToken({

diff --git a/packages/website/docs/changes/CHANGELOG.md b/packages/website/docs/changes/CHANGELOG.md
@@ -1,3 +1,9 @@
+## X.Y.Z (INSERT_DATE_HERE)
+
+#### Bug Fixes
+
+- [fix: columns position incorrect after lexer recovery](https://github.com/Chevrotain/chevrotain/issues/1969)
+
 ## 11.0.2 (8-3-2023)
 
 #### Bug Fixes