feat(parser): add extension point for non-ascii identifiers

aurelia · Apr 29, 2018 · 033948b · 033948b
1 parent 2ffeb84
commit 033948b
Show file tree

Hide file tree

Showing 2 changed files with 82 additions and 10 deletions.
diff --git a/src/parser.js b/src/parser.js
@@ -6,17 +6,32 @@ import {
   LiteralPrimitive, LiteralArray, LiteralObject, LiteralString
 } from './ast';
 
+/**
+ * Implement and register this class to support non-ascii characters in identifiers
+ */
+export class IdentifierValidator {
+  isIdentifierStart(code) {
+    return false;
+  }
+
+  isIdentifierPart(code) {
+    return false;
+  }
+}
+
 export class Parser {
-  cache;
-  constructor() {
+  static inject = [IdentifierValidator];
+
+  constructor(identifierValidator) {
+    this.identifierValidator = identifierValidator || new IdentifierValidator();
     this.cache = Object.create(null);
   }
 
   parse(input) {
     input = input || '';
 
     return this.cache[input]
-      || (this.cache[input] = new ParserImplementation(input).parseChain());
+      || (this.cache[input] = new ParserImplementation(input, this.identifierValidator).parseChain());
   }
 }
 
@@ -28,8 +43,9 @@ export class ParserImplementation {
     return this.input.slice(this.startIndex, this.index);
   }
 
-  constructor(input) {
+  constructor(input, identifierValidator) {
     this.index = 0;
+    this.identifierValidator = identifierValidator;
     this.startIndex = 0;
     this.lastIndex = 0;
     this.input = input;
@@ -487,11 +503,14 @@ export class ParserImplementation {
         case $NBSP:
           this.nextChar();
           continue;
-        // no default
-      }
+        default:
+          if (this.identifierValidator.isIdentifierStart(this.currentChar)) {
+            return this.scanIdentifier();
+          }
 
-      this.error(`Unexpected character [${String.fromCharCode(this.currentChar)}]`);
-      return null;
+          this.error(`Unexpected character [${String.fromCharCode(this.currentChar)}]`);
+          return null;
+      }
     }
 
     return T_EOF;
@@ -500,7 +519,7 @@ export class ParserImplementation {
   scanIdentifier() {
     this.nextChar();
 
-    while (isIdentifierPart(this.currentChar)) {
+    while (isIdentifierPart(this.currentChar) || this.identifierValidator.isIdentifierPart(this.currentChar)) {
       this.nextChar();
     }
 

diff --git a/test/parser.spec.js b/test/parser.spec.js
@@ -1,4 +1,4 @@
-import { Parser } from '../src/parser';
+import { Parser, IdentifierValidator } from '../src/parser';
 import {
   LiteralString,
   LiteralPrimitive,
@@ -684,8 +684,61 @@ describe('Parser', () => {
       });
     }
   });
+
+  describe('unicode identifiers', () => {
+    let customParser;
+
+    beforeAll(() => {
+      customParser = new Parser(new CustomIdentifierValidator());
+    });
+
+    it('does not parse with the default IdentifierValidator', () => {
+      try {
+        parser.parse('äöüÄÖÜß');
+      } catch(e) {
+        expect(e.message).toContain('Unexpected character');
+      }
+    });
+
+    it('parses with a custom IdentifierValidator', () => {
+      let expression = customParser.parse('äöüÄÖÜß');
+      verifyEqual(expression, new AccessScope('äöüÄÖÜß', 0));
+    });
+
+    it('does not parse with a custom IdentifierValidator with incorrect identifierStart', () => {
+      try {
+        customParser.parse('ßäöüÄÖÜ');
+      } catch(e) {
+        expect(e.message).toContain('Unexpected character');
+      }
+    });
+
+    it('does not parse with a custom IdentifierValidator with incorrect identifierPart', () => {
+      try {
+        customParser.parse('äöüÄÖÜßಠ');
+      } catch(e) {
+        expect(e.message).toContain('Unexpected character');
+      }
+    });
+  });
 });
 
+class CustomIdentifierValidator extends IdentifierValidator {
+  constructor() {
+    super();
+    this.identifierParts = [228,246,252,196,214,220,223];
+    this.identifierStarts = [228,246,252,196,214,220];
+  }
+
+  isIdentifierStart(code) {
+    return this.identifierStarts.indexOf(code) > -1;
+  }
+
+  isIdentifierPart(code) {
+    return this.identifierParts.indexOf(code) > -1;
+  }
+}
+
 function verifyEqual(actual, expected) {
   if (typeof expected !== 'object' || expected === null || expected === undefined) {
     expect(actual).toEqual(expected);