diff --git a/packages/message-compiler/src/tokenizer.ts b/packages/message-compiler/src/tokenizer.ts index b6b6a793f..1e12face5 100644 --- a/packages/message-compiler/src/tokenizer.ts +++ b/packages/message-compiler/src/tokenizer.ts @@ -321,7 +321,7 @@ export function createTokenizer( return fn() } else { // other characters - return isIdentifierStart(ch) + return isTextStart(scnr, false) } } @@ -680,7 +680,7 @@ export function createTokenizer( } function readLinkedRefer(scnr: Scanner): string { - const fn = (detect = false, buf: string): string => { + const fn = (buf: string): string => { const ch = scnr.currentChar() if ( ch === TokenChars.BraceLeft || @@ -697,15 +697,15 @@ export function createTokenizer( } else if (ch === NEW_LINE || ch === DOT) { buf += ch scnr.next() - return fn(detect, buf) + return fn(buf) } else { buf += ch scnr.next() - return fn(true, buf) + return fn(buf) } } - return fn(false, '') + return fn('') } function readPlural(scnr: Scanner): string { diff --git a/packages/message-compiler/test/__snapshots__/parser.test.ts.snap b/packages/message-compiler/test/__snapshots__/parser.test.ts.snap index f7f09c888..f134f37b1 100644 --- a/packages/message-compiler/test/__snapshots__/parser.test.ts.snap +++ b/packages/message-compiler/test/__snapshots__/parser.test.ts.snap @@ -525,28 +525,6 @@ exports[`parse > linked key paren error with modifier: "@.lower:(foo)" 1`] = ` } `; -exports[`parse > linked key paren error with modifier: "@.lower:(foo)" errors 1`] = ` -[ - { - "code": 13, - "domain": "parser", - "location": { - "end": { - "column": 14, - "line": 1, - "offset": 13, - }, - "start": { - "column": 8, - "line": 1, - "offset": 7, - }, - }, - "message": "Unexpected empty linked key", - }, -] -`; - exports[`parse > linked key paren error: "@:(foo)" 1`] = ` { "body": { @@ -640,28 +618,6 @@ exports[`parse > linked key paren error: "@:(foo)" 1`] = ` } `; -exports[`parse > linked key paren error: "@:(foo)" errors 1`] = ` -[ - { - "code": 13, - "domain": "parser", - "location": { - "end": { - "column": 8, - "line": 1, - "offset": 7, - }, - "start": { - "column": 2, - "line": 1, - "offset": 1, - }, - }, - "message": "Unexpected empty linked key", - }, -] -`; - exports[`parse > linked key with named and modifier: "hi @._upper:{_name} !" 1`] = ` { "body": { @@ -2024,16 +1980,6 @@ exports[`parser options > location disable > linked key paren error with modifie } `; -exports[`parser options > location disable > linked key paren error with modifier: "@.lower:(foo)" errors 1`] = ` -[ - { - "code": 13, - "domain": "parser", - "message": "Unexpected empty linked key", - }, -] -`; - exports[`parser options > location disable > linked key paren error: "@:(foo)" 1`] = ` { "body": { @@ -2056,16 +2002,6 @@ exports[`parser options > location disable > linked key paren error: "@:(foo)" 1 } `; -exports[`parser options > location disable > linked key paren error: "@:(foo)" errors 1`] = ` -[ - { - "code": 13, - "domain": "parser", - "message": "Unexpected empty linked key", - }, -] -`; - exports[`parser options > location disable > linked key with named and modifier: "hi @._upper:{_name} !" 1`] = ` { "body": { diff --git a/packages/message-compiler/test/__snapshots__/tokenizer.test.ts.snap b/packages/message-compiler/test/__snapshots__/tokenizer.test.ts.snap index e0f31387a..94eb379b1 100644 --- a/packages/message-compiler/test/__snapshots__/tokenizer.test.ts.snap +++ b/packages/message-compiler/test/__snapshots__/tokenizer.test.ts.snap @@ -4370,6 +4370,90 @@ exports[`token analysis > "hi @:name !" tokens 1`] = ` ] `; +exports[`token analysis > "hi @:名前" tokens 1`] = ` +[ + { + "loc": { + "end": { + "column": 4, + "line": 1, + "offset": 3, + }, + "start": { + "column": 1, + "line": 1, + "offset": 0, + }, + }, + "type": 0, + "value": "hi ", + }, + { + "loc": { + "end": { + "column": 5, + "line": 1, + "offset": 4, + }, + "start": { + "column": 4, + "line": 1, + "offset": 3, + }, + }, + "type": 8, + "value": "@", + }, + { + "loc": { + "end": { + "column": 6, + "line": 1, + "offset": 5, + }, + "start": { + "column": 5, + "line": 1, + "offset": 4, + }, + }, + "type": 10, + "value": ":", + }, + { + "loc": { + "end": { + "column": 8, + "line": 1, + "offset": 7, + }, + "start": { + "column": 6, + "line": 1, + "offset": 5, + }, + }, + "type": 11, + "value": "名前", + }, + { + "loc": { + "end": { + "column": 8, + "line": 1, + "offset": 7, + }, + "start": { + "column": 8, + "line": 1, + "offset": 7, + }, + }, + "type": 14, + }, +] +`; + exports[`token analysis > "hi @\\n. upper\\n: {'name'}\\n !" errors 1`] = ` [ { @@ -10520,6 +10604,30 @@ exports[`tokenize options: location disable > "hi @:name !" tokens 1`] = ` ] `; +exports[`tokenize options: location disable > "hi @:名前" tokens 1`] = ` +[ + { + "type": 0, + "value": "hi ", + }, + { + "type": 8, + "value": "@", + }, + { + "type": 10, + "value": ":", + }, + { + "type": 11, + "value": "名前", + }, + { + "type": 14, + }, +] +`; + exports[`tokenize options: location disable > "hi @\\n. upper\\n: {'name'}\\n !" errors 1`] = ` [ { diff --git a/packages/message-compiler/test/tokenizer.test.ts b/packages/message-compiler/test/tokenizer.test.ts index 01db8b8b4..303c3037d 100644 --- a/packages/message-compiler/test/tokenizer.test.ts +++ b/packages/message-compiler/test/tokenizer.test.ts @@ -80,6 +80,7 @@ const CASES = [ `hi @.upper\n{name} !`, `hi @.upper {name} !`, `hi @:\nname !`, + `hi @:名前`, `hi @: {'name'} !`, `hi @\n. upper\n: {'name'}\n !`, ` | | |`, diff --git a/packages/vue-i18n-core/test/issues.test.ts b/packages/vue-i18n-core/test/issues.test.ts index c635eef96..a742fe404 100644 --- a/packages/vue-i18n-core/test/issues.test.ts +++ b/packages/vue-i18n-core/test/issues.test.ts @@ -1437,3 +1437,16 @@ test('#1796', async () => { }) ).toEqual('My message with hello world.') }) + +test('#1809', async () => { + const i18n = createI18n({ + locale: 'en', + messages: { + en: { + hi: 'hi @:名前', + 名前: 'kazupon' + } + } + }) + expect(i18n.global.t('hi')).toEqual('hi kazupon') +}) diff --git a/spec/syntax.ebnf b/spec/syntax.ebnf index 326aaf635..f09313a3a 100644 --- a/spec/syntax.ebnf +++ b/spec/syntax.ebnf @@ -1,5 +1,5 @@ (* - * Inltify message syntax v0.3 + * Inltify message syntax v0.4.0 * (vue-i18n compatible) *) @@ -18,7 +18,7 @@ Named ::= Modulo? "{" Space? (NamedIdentifier) Space? "}"; List ::= "{" Space? (NumberLiteral) Space? "}"; Linked ::= "@" (LinkedModifier)? LinkedDelimiter LinkedRefer; LinkedRefer ::= LinkedKey | Placeholder; -LinkedKey ::= Identifier; +LinkedKey ::= Text; LinkedModifier ::= LinkedDot Identifier; LinkedDelimiter ::= ":"; LinkedDot ::= ".";