From c3d22c665db7074aca6469d5d77d5086ce3bbb04 Mon Sep 17 00:00:00 2001 From: Ary Borenszweig Date: Mon, 13 Oct 2014 14:06:45 -0300 Subject: [PATCH] Correctly lex unicode symbols --- spec/compiler/lexer/lexer_spec.cr | 2 +- src/compiler/crystal/syntax/lexer.cr | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/spec/compiler/lexer/lexer_spec.cr b/spec/compiler/lexer/lexer_spec.cr index 7ddeaa2042d5..a64cdf88284b 100755 --- a/spec/compiler/lexer/lexer_spec.cr +++ b/spec/compiler/lexer/lexer_spec.cr @@ -204,7 +204,7 @@ describe "Lexer" do it_lexes_instance_var "@foo" it_lexes_class_var "@@foo" it_lexes_globals ["$foo", "$FOO", "$_foo", "$foo123"] - it_lexes_symbols [":foo", ":foo!", ":foo?", ":\"foo\""] + it_lexes_symbols [":foo", ":foo!", ":foo?", ":\"foo\"", ":かたな"] it_lexes_global_match_data_index ["$1", "$10"] it_lexes "$~", :"$~" diff --git a/src/compiler/crystal/syntax/lexer.cr b/src/compiler/crystal/syntax/lexer.cr index 9bba8c2d40a1..7b5a2ad643a7 100644 --- a/src/compiler/crystal/syntax/lexer.cr +++ b/src/compiler/crystal/syntax/lexer.cr @@ -1935,11 +1935,11 @@ module Crystal end def ident_start?(char) - char.alpha? || char == '_' + char.alpha? || char == '_' || char.ord > 0x9F end def ident_part?(char) - ident_start?(char) || char.digit? || char.ord > 0x9F + ident_start?(char) || char.digit? end def ident_part_or_end?(char)