diff --git a/packages/chevrotain/src/scan/lexer_public.ts b/packages/chevrotain/src/scan/lexer_public.ts index d8927f3aa..40521ab45 100644 --- a/packages/chevrotain/src/scan/lexer_public.ts +++ b/packages/chevrotain/src/scan/lexer_public.ts @@ -732,6 +732,7 @@ export class Lexer { } errLength = offset - errorStartOffset; + column = this.computeNewColumn(column!, errLength); // at this point we either re-synced or reached the end of the input text msg = this.config.errorMessageProvider.buildUnexpectedCharactersMessage( orgText, diff --git a/packages/chevrotain/test/scan/lexer_spec.ts b/packages/chevrotain/test/scan/lexer_spec.ts index 94b02b8c0..e5d4b0b73 100644 --- a/packages/chevrotain/test/scan/lexer_spec.ts +++ b/packages/chevrotain/test/scan/lexer_spec.ts @@ -277,6 +277,32 @@ function defineLexerSpecs( expect(result.tokens[0].image).to.equal("A"); expect(result.tokens[0].startOffset).to.equal(0); }); + + // reproduce bug: https://github.com/Chevrotain/chevrotain/issues/1969 + it("will report the correct column after lexer error recovery", () => { + const digits = createToken({ + name: "digit", + pattern: /\d+/, + }); + + const myLexer = new Lexer([digits], { + positionTracking: "onlyStart", + }); + const input = "--123++"; + const result = myLexer.tokenize(input); + + expect(result.errors).to.have.lengthOf(2); + expect(result.errors[0].message).to.include("-"); + expect(result.errors[0].column).to.equal(1); + expect(result.errors[1].message).to.include("+"); + expect(result.errors[1].column).to.equal(6); + + expect(result.tokens).to.have.lengthOf(1); + const digitsToken = result.tokens[0]; + expect(tokenMatcher(digitsToken, digits)).to.be.true; + expect(digitsToken.image).to.equal("123"); + expect(digitsToken.startColumn).to.equal(3); + }); }); const ValidNaPattern = createToken({ diff --git a/packages/website/docs/changes/CHANGELOG.md b/packages/website/docs/changes/CHANGELOG.md index 05bf10c83..0d48bfbb3 100644 --- a/packages/website/docs/changes/CHANGELOG.md +++ b/packages/website/docs/changes/CHANGELOG.md @@ -1,3 +1,9 @@ +## X.Y.Z (INSERT_DATE_HERE) + +#### Bug Fixes + +- [fix: columns position incorrect after lexer recovery](https://github.com/Chevrotain/chevrotain/issues/1969) + ## 11.0.2 (8-3-2023) #### Bug Fixes