diff --git a/src/parser.js b/src/parser.js index 21588121..8edc9489 100644 --- a/src/parser.js +++ b/src/parser.js @@ -6,9 +6,24 @@ import { LiteralPrimitive, LiteralArray, LiteralObject, LiteralString } from './ast'; +/** + * Implement and register this class to support non-ascii characters in identifiers + */ +export class IdentifierValidator { + isIdentifierStart(code) { + return false; + } + + isIdentifierPart(code) { + return false; + } +} + export class Parser { - cache; - constructor() { + static inject = [IdentifierValidator]; + + constructor(identifierValidator) { + this.identifierValidator = identifierValidator || new IdentifierValidator(); this.cache = Object.create(null); } @@ -16,7 +31,7 @@ export class Parser { input = input || ''; return this.cache[input] - || (this.cache[input] = new ParserImplementation(input).parseChain()); + || (this.cache[input] = new ParserImplementation(input, this.identifierValidator).parseChain()); } } @@ -28,8 +43,9 @@ export class ParserImplementation { return this.input.slice(this.startIndex, this.index); } - constructor(input) { + constructor(input, identifierValidator) { this.index = 0; + this.identifierValidator = identifierValidator; this.startIndex = 0; this.lastIndex = 0; this.input = input; @@ -487,11 +503,14 @@ export class ParserImplementation { case $NBSP: this.nextChar(); continue; - // no default - } + default: + if (this.identifierValidator.isIdentifierStart(this.currentChar)) { + return this.scanIdentifier(); + } - this.error(`Unexpected character [${String.fromCharCode(this.currentChar)}]`); - return null; + this.error(`Unexpected character [${String.fromCharCode(this.currentChar)}]`); + return null; + } } return T_EOF; @@ -500,7 +519,7 @@ export class ParserImplementation { scanIdentifier() { this.nextChar(); - while (isIdentifierPart(this.currentChar)) { + while (isIdentifierPart(this.currentChar) || this.identifierValidator.isIdentifierPart(this.currentChar)) { this.nextChar(); } diff --git a/test/parser.spec.js b/test/parser.spec.js index 320b525e..4f4396fb 100644 --- a/test/parser.spec.js +++ b/test/parser.spec.js @@ -1,4 +1,4 @@ -import { Parser } from '../src/parser'; +import { Parser, IdentifierValidator } from '../src/parser'; import { LiteralString, LiteralPrimitive, @@ -684,8 +684,61 @@ describe('Parser', () => { }); } }); + + describe('unicode identifiers', () => { + let customParser; + + beforeAll(() => { + customParser = new Parser(new CustomIdentifierValidator()); + }); + + it('does not parse with the default IdentifierValidator', () => { + try { + parser.parse('äöüÄÖÜß'); + } catch(e) { + expect(e.message).toContain('Unexpected character'); + } + }); + + it('parses with a custom IdentifierValidator', () => { + let expression = customParser.parse('äöüÄÖÜß'); + verifyEqual(expression, new AccessScope('äöüÄÖÜß', 0)); + }); + + it('does not parse with a custom IdentifierValidator with incorrect identifierStart', () => { + try { + customParser.parse('ßäöüÄÖÜ'); + } catch(e) { + expect(e.message).toContain('Unexpected character'); + } + }); + + it('does not parse with a custom IdentifierValidator with incorrect identifierPart', () => { + try { + customParser.parse('äöüÄÖÜßಠ'); + } catch(e) { + expect(e.message).toContain('Unexpected character'); + } + }); + }); }); +class CustomIdentifierValidator extends IdentifierValidator { + constructor() { + super(); + this.identifierParts = [228,246,252,196,214,220,223]; + this.identifierStarts = [228,246,252,196,214,220]; + } + + isIdentifierStart(code) { + return this.identifierStarts.indexOf(code) > -1; + } + + isIdentifierPart(code) { + return this.identifierParts.indexOf(code) > -1; + } +} + function verifyEqual(actual, expected) { if (typeof expected !== 'object' || expected === null || expected === undefined) { expect(actual).toEqual(expected);