diff --git a/spec/compiler/lexer/lexer_spec.cr b/spec/compiler/lexer/lexer_spec.cr index d6c3a2d1fde8..239049052edc 100644 --- a/spec/compiler/lexer/lexer_spec.cr +++ b/spec/compiler/lexer/lexer_spec.cr @@ -40,7 +40,7 @@ private def it_lexes_many(values, type : Token::Kind) end end -private def it_lexes_keywords(keywords) +private def it_lexes_keywords(*keywords : Keyword) keywords.each do |keyword| it_lexes keyword.to_s, :IDENT, keyword end @@ -153,13 +153,13 @@ describe "Lexer" do it_lexes "\n", :NEWLINE it_lexes "\n\n\n", :NEWLINE it_lexes "_", :UNDERSCORE - it_lexes_keywords [:def, :if, :else, :elsif, :end, :true, :false, :class, :module, :include, - :extend, :while, :until, :nil, :do, :yield, :return, :unless, :next, :break, - :begin, :lib, :fun, :type, :struct, :union, :enum, :macro, :out, :require, - :case, :when, :select, :then, :of, :abstract, :rescue, :ensure, :is_a?, :alias, - :pointerof, :sizeof, :instance_sizeof, :offsetof, :as, :as?, :typeof, :for, :in, - :with, :self, :super, :private, :protected, :asm, :uninitialized, :nil?, - :annotation, :verbatim] + it_lexes_keywords :def, :if, :else, :elsif, :end, :true, :false, :class, :module, :include, + :extend, :while, :until, :nil, :do, :yield, :return, :unless, :next, :break, + :begin, :lib, :fun, :type, :struct, :union, :enum, :macro, :out, :require, + :case, :when, :select, :then, :of, :abstract, :rescue, :ensure, :alias, + :pointerof, :sizeof, :instance_sizeof, :offsetof, :as, :typeof, :for, :in, + :with, :self, :super, :private, :protected, :asm, :uninitialized, + :annotation, :verbatim, :is_a_question, :as_question, :nil_question, :responds_to_question it_lexes_idents ["ident", "something", "with_underscores", "with_1", "foo?", "bar!", "fooBar", "❨╯°□°❩╯︵┻━┻"] it_lexes_idents ["def?", "if?", "else?", "elsif?", "end?", "true?", "false?", "class?", "while?", @@ -443,7 +443,7 @@ describe "Lexer" do lexer = Lexer.new "end 1" token = lexer.next_token token.type.should eq(t :IDENT) - token.value.should eq(:end) + token.value.should eq(Keyword::END) token = lexer.next_token token.type.should eq(t :SPACE) end diff --git a/src/compiler/crystal/syntax/lexer.cr b/src/compiler/crystal/syntax/lexer.cr index 7f754e6fe5a7..1de25da3a125 100644 --- a/src/compiler/crystal/syntax/lexer.cr +++ b/src/compiler/crystal/syntax/lexer.cr @@ -858,7 +858,7 @@ module Crystal next_char next_char @token.type = :IDENT - @token.value = :as? + @token.value = :as_question return @token else return check_ident_or_keyword(:as, start) @@ -994,7 +994,7 @@ module Crystal end when 's' if next_char == '_' && next_char == 'a' && next_char == '?' - return check_ident_or_keyword(:is_a?, start) + return check_ident_or_keyword(:is_a_question, start) end else # scan_ident @@ -1040,7 +1040,7 @@ module Crystal when 'l' if peek_next_char == '?' next_char - return check_ident_or_keyword(:nil?, start) + return check_ident_or_keyword(:nil_question, start) else return check_ident_or_keyword(:nil, start) end @@ -1105,7 +1105,7 @@ module Crystal end when 'p' if next_char == 'o' && next_char == 'n' && next_char == 'd' && next_char == 's' && next_char == '_' && next_char == 't' && next_char == 'o' && next_char == '?' - return check_ident_or_keyword(:responds_to?, start) + return check_ident_or_keyword(:responds_to_question, start) end else # scan_ident @@ -1426,13 +1426,13 @@ module Crystal end end - def check_ident_or_keyword(symbol, start) + def check_ident_or_keyword(keyword : Keyword, start) if ident_part_or_end?(peek_next_char) scan_ident(start) else next_char @token.type = :IDENT - @token.value = symbol + @token.value = keyword end @token end diff --git a/src/compiler/crystal/syntax/parser.cr b/src/compiler/crystal/syntax/parser.cr index 48e45bbf3296..ead4e2c9ed5d 100644 --- a/src/compiler/crystal/syntax/parser.cr +++ b/src/compiler/crystal/syntax/parser.cr @@ -285,15 +285,15 @@ module Crystal next_token when .ident? case @token.value - when :if + when Keyword::IF atomic = parse_expression_suffix(location) { |exp| If.new(exp, atomic) } - when :unless + when Keyword::UNLESS atomic = parse_expression_suffix(location) { |exp| Unless.new(exp, atomic) } - when :while + when Keyword::WHILE raise "trailing `while` is not supported", @token - when :until + when Keyword::UNTIL raise "trailing `until` is not supported", @token - when :rescue + when Keyword::RESCUE next_token_skip_space rescue_body = parse_op_assign rescues = [Rescue.new(rescue_body)] of Rescue @@ -302,7 +302,7 @@ module Crystal else atomic = ExceptionHandler.new(atomic, rescues).at(location).tap { |e| e.suffix = true } end - when :ensure + when Keyword::ENSURE next_token_skip_space ensure_body = parse_op_assign if atomic.is_a?(Assign) @@ -672,15 +672,15 @@ module Crystal check AtomicWithMethodCheck name_location = @token.location - if @token.value == :is_a? + if @token.value == Keyword::IS_A_QUESTION atomic = parse_is_a(atomic).at(location) - elsif @token.value == :as + elsif @token.value == Keyword::AS atomic = parse_as(atomic).at(location) - elsif @token.value == :as? + elsif @token.value == Keyword::AS_QUESTION atomic = parse_as?(atomic).at(location) - elsif @token.value == :responds_to? + elsif @token.value == Keyword::RESPONDS_TO_QUESTION atomic = parse_responds_to(atomic).at(location) - elsif !@in_macro_expression && @token.value == :nil? + elsif !@in_macro_expression && @token.value == Keyword::NIL_QUESTION atomic = parse_nil?(atomic).at(location) elsif @token.type.op_bang? atomic = parse_negation_suffix(atomic).at(location) @@ -1014,153 +1014,152 @@ module Crystal when .ident? # NOTE: Update `Parser#invalid_internal_name?` keyword list # when adding or removing keyword to handle here. - case @token.value - when :begin - check_type_declaration { parse_begin } - when :nil - check_type_declaration { node_and_next_token NilLiteral.new } - when :true - check_type_declaration { node_and_next_token BoolLiteral.new(true) } - when :false - check_type_declaration { node_and_next_token BoolLiteral.new(false) } - when :yield - check_type_declaration { parse_yield } - when :with - check_type_declaration { parse_yield_with_scope } - when :abstract - check_type_declaration do - check_not_inside_def("can't use abstract") do - doc = @token.doc + if keyword = @token.value.as?(Keyword) + case keyword + when .begin? + check_type_declaration { parse_begin } + when Keyword::NIL + check_type_declaration { node_and_next_token NilLiteral.new } + when .true? + check_type_declaration { node_and_next_token BoolLiteral.new(true) } + when .false? + check_type_declaration { node_and_next_token BoolLiteral.new(false) } + when .yield? + check_type_declaration { parse_yield } + when .with? + check_type_declaration { parse_yield_with_scope } + when .abstract? + check_type_declaration do + check_not_inside_def("can't use abstract") do + doc = @token.doc - next_token_skip_space_or_newline - case @token.type - when .ident? + next_token_skip_space_or_newline case @token.value - when :def + when Keyword::DEF parse_def is_abstract: true, doc: doc - when :class + when Keyword::CLASS parse_class_def is_abstract: true, doc: doc - when :struct + when Keyword::STRUCT parse_class_def is_abstract: true, is_struct: true, doc: doc else unexpected_token end - else - unexpected_token end end - end - when :def - check_type_declaration do - check_not_inside_def("can't define def") do - parse_def + when .def? + check_type_declaration do + check_not_inside_def("can't define def") do + parse_def + end end - end - when :macro - check_type_declaration do - check_not_inside_def("can't define macro") do - parse_macro + when .macro? + check_type_declaration do + check_not_inside_def("can't define macro") do + parse_macro + end end - end - when :require - check_type_declaration do - check_not_inside_def("can't require") do - parse_require + when .require? + check_type_declaration do + check_not_inside_def("can't require") do + parse_require + end end - end - when :case - check_type_declaration { parse_case } - when :select - check_type_declaration { parse_select } - when :if - check_type_declaration { parse_if } - when :unless - check_type_declaration { parse_unless } - when :include - check_type_declaration do - check_not_inside_def("can't include") do - parse_include + when .case? + check_type_declaration { parse_case } + when .select? + check_type_declaration { parse_select } + when .if? + check_type_declaration { parse_if } + when .unless? + check_type_declaration { parse_unless } + when .include? + check_type_declaration do + check_not_inside_def("can't include") do + parse_include + end end - end - when :extend - check_type_declaration do - check_not_inside_def("can't extend") do - parse_extend + when .extend? + check_type_declaration do + check_not_inside_def("can't extend") do + parse_extend + end end - end - when :class - check_type_declaration do - check_not_inside_def("can't define class") do - parse_class_def + when .class? + check_type_declaration do + check_not_inside_def("can't define class") do + parse_class_def + end end - end - when :struct - check_type_declaration do - check_not_inside_def("can't define struct") do - parse_class_def is_struct: true + when .struct? + check_type_declaration do + check_not_inside_def("can't define struct") do + parse_class_def is_struct: true + end end - end - when :module - check_type_declaration do - check_not_inside_def("can't define module") do - parse_module_def + when .module? + check_type_declaration do + check_not_inside_def("can't define module") do + parse_module_def + end end - end - when :enum - check_type_declaration do - check_not_inside_def("can't define enum") do - parse_enum_def + when .enum? + check_type_declaration do + check_not_inside_def("can't define enum") do + parse_enum_def + end end - end - when :while - check_type_declaration { parse_while } - when :until - check_type_declaration { parse_until } - when :return - check_type_declaration { parse_return } - when :next - check_type_declaration { parse_next } - when :break - check_type_declaration { parse_break } - when :lib - check_type_declaration do - check_not_inside_def("can't define lib") do - parse_lib + when .while? + check_type_declaration { parse_while } + when .until? + check_type_declaration { parse_until } + when .return? + check_type_declaration { parse_return } + when .next? + check_type_declaration { parse_next } + when .break? + check_type_declaration { parse_break } + when .lib? + check_type_declaration do + check_not_inside_def("can't define lib") do + parse_lib + end end - end - when :fun - check_type_declaration do - check_not_inside_def("can't define fun") do - parse_fun_def top_level: true, require_body: true + when .fun? + check_type_declaration do + check_not_inside_def("can't define fun") do + parse_fun_def top_level: true, require_body: true + end end - end - when :alias - check_type_declaration do - check_not_inside_def("can't define alias") do - parse_alias + when .alias? + check_type_declaration do + check_not_inside_def("can't define alias") do + parse_alias + end end - end - when :pointerof - check_type_declaration { parse_pointerof } - when :sizeof - check_type_declaration { parse_sizeof } - when :instance_sizeof - check_type_declaration { parse_instance_sizeof } - when :offsetof - check_type_declaration { parse_offsetof } - when :typeof - check_type_declaration { parse_typeof } - when :private - check_type_declaration { parse_visibility_modifier Visibility::Private } - when :protected - check_type_declaration { parse_visibility_modifier Visibility::Protected } - when :asm - check_type_declaration { parse_asm } - when :annotation - check_type_declaration do - check_not_inside_def("can't define annotation") do - parse_annotation_def + when .pointerof? + check_type_declaration { parse_pointerof } + when .sizeof? + check_type_declaration { parse_sizeof } + when .instance_sizeof? + check_type_declaration { parse_instance_sizeof } + when .offsetof? + check_type_declaration { parse_offsetof } + when .typeof? + check_type_declaration { parse_typeof } + when .private? + check_type_declaration { parse_visibility_modifier Visibility::Private } + when .protected? + check_type_declaration { parse_visibility_modifier Visibility::Protected } + when .asm? + check_type_declaration { parse_asm } + when .annotation? + check_type_declaration do + check_not_inside_def("can't define annotation") do + parse_annotation_def + end end + else + set_visibility parse_var_or_call end else set_visibility parse_var_or_call @@ -1253,7 +1252,7 @@ module Crystal skip_space - if @token.keyword?(:"of") + if @token.keyword?(:of) unexpected_token end @@ -1550,19 +1549,19 @@ module Crystal check AtomicWithMethodCheck - if @token.value == :is_a? + if @token.value == Keyword::IS_A_QUESTION call = parse_is_a(obj).at(location) call = parse_atomic_method_suffix_special(call, location) - elsif @token.value == :as + elsif @token.value == Keyword::AS call = parse_as(obj).at(location) call = parse_atomic_method_suffix_special(call, location) - elsif @token.value == :as? + elsif @token.value == Keyword::AS_QUESTION call = parse_as?(obj).at(location) call = parse_atomic_method_suffix_special(call, location) - elsif @token.value == :responds_to? + elsif @token.value == Keyword::RESPONDS_TO_QUESTION call = parse_responds_to(obj).at(location) call = parse_atomic_method_suffix_special(call, location) - elsif !@in_macro_expression && @token.value == :nil? + elsif !@in_macro_expression && @token.value == Keyword::NIL_QUESTION call = parse_nil?(obj).at(location) call = parse_atomic_method_suffix_special(call, location) elsif @token.type.op_bang? @@ -2669,7 +2668,7 @@ module Crystal next_token_skip_space end - unless @token.keyword? && @token.value.in?(:when, :else, :end) + unless @token.value.in?(Keyword::WHEN, Keyword::ELSE, Keyword::END) cond = parse_op_assign_no_control skip_statement_end end @@ -2682,102 +2681,97 @@ module Crystal when_exps = Set(ASTNode).new while true - case @token.type - when .ident? - case @token.value - when :when, :in - if exhaustive.nil? - exhaustive = @token.value == :in - if exhaustive && !cond - raise "exhaustive case (case ... in) requires a case expression (case exp; in ..)" - end - elsif exhaustive && @token.value == :when - raise "expected 'in', not 'when'" - elsif !exhaustive && @token.value == :in - raise "expected 'when', not 'in'" + case @token.value + when Keyword::WHEN, Keyword::IN + if exhaustive.nil? + exhaustive = @token.value == Keyword::IN + if exhaustive && !cond + raise "exhaustive case (case ... in) requires a case expression (case exp; in ..)" end + elsif exhaustive && @token.value == Keyword::WHEN + raise "expected 'in', not 'when'" + elsif !exhaustive && @token.value == Keyword::IN + raise "expected 'when', not 'in'" + end - location = @token.location - slash_is_regex! - next_token_skip_space_or_newline - when_conds = [] of ASTNode - - if cond.is_a?(TupleLiteral) - raise "splat is not allowed inside case expression" if cond.elements.any?(Splat) + location = @token.location + slash_is_regex! + next_token_skip_space_or_newline + when_conds = [] of ASTNode - while true - if @token.type.op_lcurly? - curly_location = @token.location + if cond.is_a?(TupleLiteral) + raise "splat is not allowed inside case expression" if cond.elements.any?(Splat) - next_token_skip_space_or_newline + while true + if @token.type.op_lcurly? + curly_location = @token.location - tuple_elements = [] of ASTNode + next_token_skip_space_or_newline - while true - exp = parse_when_expression(cond, single: false, exhaustive: exhaustive) - check_valid_exhaustive_expression(exp) if exhaustive + tuple_elements = [] of ASTNode - tuple_elements << exp + while true + exp = parse_when_expression(cond, single: false, exhaustive: exhaustive) + check_valid_exhaustive_expression(exp) if exhaustive - skip_space - if @token.type.op_comma? - next_token_skip_space_or_newline - else - break - end - end + tuple_elements << exp - if tuple_elements.size != cond.elements.size - raise "wrong number of tuple elements (given #{tuple_elements.size}, expected #{cond.elements.size})", curly_location + skip_space + if @token.type.op_comma? + next_token_skip_space_or_newline + else + break end + end - tuple = TupleLiteral.new(tuple_elements).at(curly_location) - when_conds << tuple - add_when_exp(when_exps, tuple) - - check :OP_RCURLY - next_token_skip_space - else - exp = parse_when_expression(cond, single: true, exhaustive: exhaustive) - when_conds << exp - add_when_exp(when_exps, exp) - skip_space + if tuple_elements.size != cond.elements.size + raise "wrong number of tuple elements (given #{tuple_elements.size}, expected #{cond.elements.size})", curly_location end - break if when_expression_end - end - else - while true - exp = parse_when_expression(cond, single: true, exhaustive: exhaustive) - check_valid_exhaustive_expression(exp) if exhaustive + tuple = TupleLiteral.new(tuple_elements).at(curly_location) + when_conds << tuple + add_when_exp(when_exps, tuple) + check :OP_RCURLY + next_token_skip_space + else + exp = parse_when_expression(cond, single: true, exhaustive: exhaustive) when_conds << exp add_when_exp(when_exps, exp) skip_space - break if when_expression_end end + + break if when_expression_end end + else + while true + exp = parse_when_expression(cond, single: true, exhaustive: exhaustive) + check_valid_exhaustive_expression(exp) if exhaustive - when_body = parse_expressions - skip_space_or_newline - whens << When.new(when_conds, when_body).at(location) - when :else - if exhaustive - raise "exhaustive case (case ... in) doesn't allow an 'else'" + when_conds << exp + add_when_exp(when_exps, exp) + skip_space + break if when_expression_end end + end - next_token_skip_statement_end - a_else = parse_expressions - skip_statement_end - check_ident :end - next_token - break - when :end - next_token - break - else - unexpected_token "expecting when, else or end" + when_body = parse_expressions + skip_space_or_newline + whens << When.new(when_conds, when_body).at(location) + when Keyword::ELSE + if exhaustive + raise "exhaustive case (case ... in) doesn't allow an 'else'" end + + next_token_skip_statement_end + a_else = parse_expressions + skip_statement_end + check_ident :end + next_token + break + when Keyword::END + next_token + break else unexpected_token "expecting when, else or end" end @@ -2898,49 +2892,44 @@ module Crystal whens = [] of Select::When while true - case @token.type - when .ident? - case @token.value - when :when - slash_is_regex! - next_token_skip_space_or_newline + case @token.value + when Keyword::WHEN + slash_is_regex! + next_token_skip_space_or_newline - location = @token.location - condition = parse_op_assign_no_control - unless valid_select_when?(condition) - raise "invalid select when expression: must be an assignment or call", location - end + location = @token.location + condition = parse_op_assign_no_control + unless valid_select_when?(condition) + raise "invalid select when expression: must be an assignment or call", location + end - skip_space - unless when_expression_end - unexpected_token "expecting then, ';' or newline" - end - skip_statement_end + skip_space + unless when_expression_end + unexpected_token "expecting then, ';' or newline" + end + skip_statement_end - body = parse_expressions - skip_space_or_newline + body = parse_expressions + skip_space_or_newline - whens << Select::When.new(condition, body) - when :else - if whens.size == 0 - unexpected_token "expecting when" - end - slash_is_regex! - next_token_skip_statement_end - a_else = parse_expressions - skip_statement_end - check_ident :end - next_token - break - when :end - if whens.empty? - unexpected_token "expecting when, else or end" - end - next_token - break - else + whens << Select::When.new(condition, body) + when Keyword::ELSE + if whens.size == 0 + unexpected_token "expecting when" + end + slash_is_regex! + next_token_skip_statement_end + a_else = parse_expressions + skip_statement_end + check_ident :end + next_token + break + when Keyword::END + if whens.empty? unexpected_token "expecting when, else or end" end + next_token + break else unexpected_token "expecting when, else or end" end @@ -3287,87 +3276,85 @@ module Crystal def parse_macro_control(start_location, macro_state = Token::MacroState.default) next_token_skip_space_or_newline - if @token.type.ident? - case @token.value - when :for - next_token_skip_space + case @token.value + when Keyword::FOR + next_token_skip_space - vars = [] of Var + vars = [] of Var - while true - var = case @token.type - when .underscore? - "_" - when .ident? - @token.value.to_s - else - unexpected_token "expecting ident or underscore" - end - vars << Var.new(var).at(@token.location) + while true + var = case @token.type + when .underscore? + "_" + when .ident? + @token.value.to_s + else + unexpected_token "expecting ident or underscore" + end + vars << Var.new(var).at(@token.location) + next_token_skip_space + if @token.type.op_comma? next_token_skip_space - if @token.type.op_comma? - next_token_skip_space - else - break - end + else + break end + end - check_ident :in - next_token_skip_space + check_ident :in + next_token_skip_space - exp = parse_expression_inside_macro + exp = parse_expression_inside_macro - check :OP_PERCENT_RCURLY + check :OP_PERCENT_RCURLY - macro_state.control_nest += 1 - body, end_location = parse_macro_body(start_location, macro_state) - macro_state.control_nest -= 1 + macro_state.control_nest += 1 + body, end_location = parse_macro_body(start_location, macro_state) + macro_state.control_nest -= 1 - check_ident :end - next_token_skip_space - check :OP_PERCENT_RCURLY + check_ident :end + next_token_skip_space + check :OP_PERCENT_RCURLY - return MacroFor.new(vars, exp, body).at_end(token_end_location) - when :if - return parse_macro_if(start_location, macro_state) - when :unless - return parse_macro_if(start_location, macro_state, is_unless: true) - when :begin - next_token_skip_space - check :OP_PERCENT_RCURLY + return MacroFor.new(vars, exp, body).at_end(token_end_location) + when Keyword::IF + return parse_macro_if(start_location, macro_state) + when Keyword::UNLESS + return parse_macro_if(start_location, macro_state, is_unless: true) + when Keyword::BEGIN + next_token_skip_space + check :OP_PERCENT_RCURLY - macro_state.control_nest += 1 - body, end_location = parse_macro_body(start_location, macro_state) - macro_state.control_nest -= 1 + macro_state.control_nest += 1 + body, end_location = parse_macro_body(start_location, macro_state) + macro_state.control_nest -= 1 - check_ident :end - next_token_skip_space - check :OP_PERCENT_RCURLY + check_ident :end + next_token_skip_space + check :OP_PERCENT_RCURLY - return MacroIf.new(BoolLiteral.new(true), body).at_end(token_end_location) - when :else, :elsif, :end - return nil - when :verbatim - next_token_skip_space - unless @token.keyword?(:do) - unexpected_token(msg: "expecting 'do'") - end - next_token_skip_space - check :OP_PERCENT_RCURLY + return MacroIf.new(BoolLiteral.new(true), body).at_end(token_end_location) + when Keyword::ELSE, Keyword::ELSIF, Keyword::END + return nil + when Keyword::VERBATIM + next_token_skip_space + unless @token.keyword?(:do) + unexpected_token(msg: "expecting 'do'") + end + next_token_skip_space + check :OP_PERCENT_RCURLY - macro_state.control_nest += 1 - body, end_location = parse_macro_body(start_location, macro_state) - macro_state.control_nest -= 1 + macro_state.control_nest += 1 + body, end_location = parse_macro_body(start_location, macro_state) + macro_state.control_nest -= 1 - check_ident :end - next_token_skip_space - check :OP_PERCENT_RCURLY + check_ident :end + next_token_skip_space + check :OP_PERCENT_RCURLY - return MacroVerbatim.new(body).at_end(token_end_location) - else - # will be parsed as a normal expression - end + return MacroVerbatim.new(body).at_end(token_end_location) + else + # will be parsed as a normal expression end @in_macro_expression = true @@ -3401,37 +3388,33 @@ module Crystal a_then, end_location = parse_macro_body(start_location, macro_state) macro_state.control_nest -= 1 - if @token.type.ident? - case @token.value - when :else - next_token_skip_space - check :OP_PERCENT_RCURLY + case @token.value + when Keyword::ELSE + next_token_skip_space + check :OP_PERCENT_RCURLY - macro_state.control_nest += 1 - a_else, end_location = parse_macro_body(start_location, macro_state) - macro_state.control_nest -= 1 + macro_state.control_nest += 1 + a_else, end_location = parse_macro_body(start_location, macro_state) + macro_state.control_nest -= 1 - if check_end - check_ident :end - next_token_skip_space - check :OP_PERCENT_RCURLY - end - when :elsif - unexpected_token if is_unless - a_else = parse_macro_if(start_location, macro_state, false) + if check_end + check_ident :end + next_token_skip_space + check :OP_PERCENT_RCURLY + end + when Keyword::ELSIF + unexpected_token if is_unless + a_else = parse_macro_if(start_location, macro_state, false) - if check_end - check_ident :end - next_token_skip_space - check :OP_PERCENT_RCURLY - end - when :end - if check_end - next_token_skip_space - check :OP_PERCENT_RCURLY - end - else - unexpected_token + if check_end + check_ident :end + next_token_skip_space + check :OP_PERCENT_RCURLY + end + when Keyword::END + if check_end + next_token_skip_space + check :OP_PERCENT_RCURLY end else unexpected_token @@ -3632,7 +3615,7 @@ module Crystal end skip_space - if @token.keyword?("forall") + if @token.type.ident? && @token.value == "forall" next_token_skip_space free_vars = parse_def_free_vars end @@ -3675,7 +3658,7 @@ module Crystal end def check_valid_def_name - if @token.value.in?(:is_a?, :as, :as?, :responds_to?, :nil?) + if @token.value.in?(Keyword::IS_A_QUESTION, Keyword::AS, Keyword::AS_QUESTION, Keyword::RESPONDS_TO_QUESTION, Keyword::NIL_QUESTION) raise "'#{@token.value}' is a pseudo-method and can't be redefined", @token end end @@ -4018,17 +4001,17 @@ module Crystal def invalid_internal_name?(keyword) case keyword - when Symbol + when Keyword case keyword # These names are handled as keyword by `Parser#parse_atomic_without_location`. # We cannot assign value into them and never reference them, # so they are invalid internal name. - when :begin, :nil, :true, :false, :yield, :with, :abstract, - :def, :macro, :require, :case, :select, :if, :unless, :include, - :extend, :class, :struct, :module, :enum, :while, :until, :return, - :next, :break, :lib, :fun, :alias, :pointerof, :sizeof, :offsetof, - :instance_sizeof, :typeof, :private, :protected, :asm, :out, - :self, :in, :end + when .begin?, Keyword::NIL, .true?, .false?, .yield?, .with?, .abstract?, + .def?, .macro?, .require?, .case?, .select?, .if?, .unless?, .include?, + .extend?, .class?, .struct?, .module?, .enum?, .while?, .until?, .return?, + .next?, .break?, .lib?, .fun?, .alias?, .pointerof?, .sizeof?, .offsetof?, + .instance_sizeof?, .typeof?, .private?, .protected?, .asm?, .out?, + .self?, Keyword::IN, .end? true else false @@ -4071,10 +4054,10 @@ module Crystal if @token.type.ident? else_location = @token.location case @token.value - when :else + when Keyword::ELSE next_token_skip_statement_end a_else = parse_expressions - when :elsif + when Keyword::ELSIF a_else = parse_if check_end: false end end @@ -4142,19 +4125,19 @@ module Crystal end case @token.value - when :is_a? + when Keyword::IS_A_QUESTION obj = Var.new("self").at(location) return parse_is_a(obj) - when :as + when Keyword::AS obj = Var.new("self").at(location) return parse_as(obj) - when :as? + when Keyword::AS_QUESTION obj = Var.new("self").at(location) return parse_as?(obj) - when :responds_to? + when Keyword::RESPONDS_TO_QUESTION obj = Var.new("self").at(location) return parse_responds_to(obj) - when :nil? + when Keyword::NIL_QUESTION unless @in_macro_expression obj = Var.new("self").at(location) return parse_nil?(obj) @@ -4595,9 +4578,9 @@ module Crystal end case @token.value - when :if, :unless, :while, :until, :rescue, :ensure + when Keyword::IF, Keyword::UNLESS, Keyword::WHILE, Keyword::UNTIL, Keyword::RESCUE, Keyword::ENSURE return nil unless next_comes_colon_space? - when :yield + when Keyword::YIELD return nil if @stop_on_yield > 0 && !next_comes_colon_space? else # keep going @@ -4871,13 +4854,13 @@ module Crystal case @token.type when .ident? case @token.value - when :self + when Keyword::SELF next_token_skip_space Self.new.at(location) when "self?" next_token_skip_space make_nilable_type Self.new.at(location) - when :typeof + when Keyword::TYPEOF parse_typeof else unexpected_token @@ -5219,9 +5202,9 @@ module Crystal when .ident? return false if named_tuple_start? case @token.value - when :typeof + when Keyword::TYPEOF true - when :self, "self?" + when Keyword::SELF, "self?" next_token_skip_space delimiter_or_type_suffix? else @@ -5540,20 +5523,20 @@ module Crystal parse_annotation when .ident? case @token.value - when :alias + when Keyword::ALIAS parse_alias - when :fun + when Keyword::FUN parse_fun_def(top_level: false) - when :type + when Keyword::TYPE parse_type_def - when :struct + when Keyword::STRUCT @inside_c_struct = true node = parse_c_struct_or_union union: false @inside_c_struct = false node - when :union + when Keyword::UNION parse_c_struct_or_union union: true - when :enum + when Keyword::ENUM parse_enum_def else unexpected_token @@ -5849,16 +5832,16 @@ module Crystal case @token.type when .ident? case @token.value - when :include + when Keyword::INCLUDE if @inside_c_struct location = @token.location exps << parse_include.at(location) else parse_c_struct_or_union_fields exps end - when :else + when Keyword::ELSE break - when :end + when Keyword::END break else parse_c_struct_or_union_fields exps @@ -5972,10 +5955,10 @@ module Crystal visibility = nil case @token.value - when :private + when Keyword::PRIVATE visibility = Visibility::Private next_token_skip_space - when :protected + when Keyword::PROTECTED visibility = Visibility::Protected next_token_skip_space else @@ -5985,11 +5968,11 @@ module Crystal def_location = @token.location case @token.value - when :def + when Keyword::DEF member = parse_def.at(def_location) member = VisibilityModifier.new(visibility, member) if visibility members << member - when :macro + when Keyword::MACRO member = parse_macro.at(def_location) member = VisibilityModifier.new(visibility, member) if visibility members << member @@ -6030,10 +6013,12 @@ module Crystal def end_token? case @token.type when .op_rcurly?, .op_rsquare?, .op_percent_rcurly?, .eof? - true - when .ident? - case @token.value - when :do, :end, :else, :elsif, :when, :in, :rescue, :ensure, :then + return true + end + + if keyword = @token.value.as?(Keyword) + case keyword + when .do?, .end?, .else?, .elsif?, .when?, Keyword::IN, .rescue?, .ensure?, .then? !next_comes_colon_space? else false @@ -6162,18 +6147,11 @@ module Crystal end def check_void_expression_keyword - case @token.type - when .ident? - case @token.value - when :break, :next, :return - unless next_comes_colon_space? - raise "void value expression", @token, @token.value.to_s.size - end - else - # not a void expression + case @token.value + when Keyword::BREAK, Keyword::NEXT, Keyword::RETURN + unless next_comes_colon_space? + raise "void value expression", @token, @token.value.to_s.size end - else - # not a void expression end end @@ -6185,11 +6163,7 @@ module Crystal raise "expecting token '#{token_type}', not '#{@token}'", @token unless token_type == @token.type end - def check_token(value) - raise "expecting token '#{value}', not '#{@token}'", @token unless @token.type.token? && @token.value == value - end - - def check_ident(value) + def check_ident(value : Keyword) raise "expecting identifier '#{value}', not '#{@token}'", @token unless @token.keyword?(value) end diff --git a/src/compiler/crystal/syntax/token.cr b/src/compiler/crystal/syntax/token.cr index dc1d41fec6f0..db85b0f697a7 100644 --- a/src/compiler/crystal/syntax/token.cr +++ b/src/compiler/crystal/syntax/token.cr @@ -1,6 +1,85 @@ require "./location" module Crystal + # All possible identifiers that may be considered keywords. + enum Keyword + ABSTRACT + ALIAS + ANNOTATION + AS + AS_QUESTION + ASM + BEGIN + BREAK + CASE + CLASS + DEF + DO + ELSE + ELSIF + END + ENSURE + ENUM + EXTEND + FALSE + FOR + FUN + IF + IN + INCLUDE + INSTANCE_SIZEOF + IS_A_QUESTION + LIB + MACRO + MODULE + NEXT + NIL + NIL_QUESTION + OF + OFFSETOF + OUT + POINTEROF + PRIVATE + PROTECTED + REQUIRE + RESCUE + RESPONDS_TO_QUESTION + RETURN + SELECT + SELF + SIZEOF + STRUCT + SUPER + THEN + TRUE + TYPE + TYPEOF + UNINITIALIZED + UNION + UNLESS + UNTIL + VERBATIM + WHEN + WHILE + WITH + YIELD + + def to_s + case self + when AS_QUESTION + "as?" + when IS_A_QUESTION + "is_a?" + when NIL_QUESTION + "nil?" + when RESPONDS_TO_QUESTION + "responds_to?" + else + super.downcase + end + end + end + class Token enum Kind EOF @@ -174,7 +253,7 @@ module Crystal end property type : Kind - property value : Char | String | Symbol | Nil + property value : Char | String | Keyword | Nil property number_kind : NumberKind property line_number : Int32 property column_number : Int32 @@ -279,10 +358,10 @@ module Crystal end def keyword? - @type.ident? && @value.is_a?(Symbol) + @type.ident? && @value.is_a?(Keyword) end - def keyword?(keyword) + def keyword?(keyword : Keyword) @type.ident? && @value == keyword end diff --git a/src/compiler/crystal/tools/formatter.cr b/src/compiler/crystal/tools/formatter.cr index e2bb0a94306f..eef9ac5364d1 100644 --- a/src/compiler/crystal/tools/formatter.cr +++ b/src/compiler/crystal/tools/formatter.cr @@ -1321,7 +1321,7 @@ module Crystal visit_if_or_unless node, :unless end - def visit_if_or_unless(node, keyword) + def visit_if_or_unless(node, keyword : Keyword) if !@token.keyword?(keyword) && node.else.is_a?(Nop) # Suffix if/unless accept node.then @@ -1338,7 +1338,7 @@ module Crystal false end - def format_if_at_cond(node, keyword, check_end = true) + def format_if_at_cond(node, keyword : Keyword, check_end = true) inside_cond do indent(@column, node.cond) end @@ -1367,7 +1367,7 @@ module Crystal end end - def format_elsif(node_else, keyword) + def format_elsif(node_else, keyword : Keyword) write_indent write "elsif " next_token_skip_space_or_newline @@ -1382,7 +1382,7 @@ module Crystal format_while_or_until node, :until end - def format_while_or_until(node, keyword) + def format_while_or_until(node, keyword : Keyword) write_keyword keyword, " " inside_cond do indent(@column, node.cond) @@ -3497,7 +3497,7 @@ module Crystal def visit(node : ClassDef) write_keyword :abstract, " " if node.abstract? - write_keyword (node.struct? ? :struct : :class), " " + write_keyword (node.struct? ? Keyword::STRUCT : Keyword::CLASS), " " accept node.name format_type_vars node.type_vars, node.splat_index @@ -3534,7 +3534,7 @@ module Crystal end def visit(node : CStructOrUnionDef) - keyword = node.union? ? :union : :struct + keyword = node.union? ? Keyword::UNION : Keyword::STRUCT write_keyword keyword, " " write node.name @@ -3644,7 +3644,7 @@ module Crystal format_control_expression node, :next end - def format_control_expression(node, keyword) + def format_control_expression(node, keyword : Keyword) write_keyword keyword has_parentheses = false @@ -3778,7 +3778,7 @@ module Crystal slash_is_regex! write_indent - write_keyword(node.exhaustive? ? :in : :when, " ") + write_keyword(node.exhaustive? ? Keyword::IN : Keyword::WHEN, " ") base_indent = @column when_start_line = @line when_start_column = @column @@ -4140,7 +4140,7 @@ module Crystal format_alias_or_typedef node, :type, node.type_spec end - def format_alias_or_typedef(node, keyword, value) + def format_alias_or_typedef(node, keyword : Keyword, value) write_keyword keyword, " " name = node.name @@ -5083,24 +5083,24 @@ module Crystal end end - def write_keyword(keyword : Symbol) + def write_keyword(keyword : Keyword) check_keyword keyword write keyword next_token end - def write_keyword(before : String, keyword : Symbol) + def write_keyword(before : String, keyword : Keyword) write before write_keyword keyword end - def write_keyword(keyword : Symbol, after : String, skip_space_or_newline = true) + def write_keyword(keyword : Keyword, after : String, skip_space_or_newline = true) write_keyword keyword write after skip_space_or_newline() if skip_space_or_newline end - def write_keyword(before : String, keyword : Symbol, after : String) + def write_keyword(before : String, keyword : Keyword, after : String) passed_backslash_newline = @token.passed_backslash_newline skip_space if passed_backslash_newline && before == " " @@ -5135,7 +5135,7 @@ module Crystal write after end - def check_keyword(*keywords) + def check_keyword(*keywords : Keyword) raise "expecting keyword #{keywords.join " or "}, not `#{@token.type}, #{@token.value}`, at #{@token.location}" unless keywords.any? { |k| @token.keyword?(k) } end diff --git a/src/crystal/syntax_highlighter.cr b/src/crystal/syntax_highlighter.cr index 8aa7f9d1d152..60db6a71dbac 100644 --- a/src/crystal/syntax_highlighter.cr +++ b/src/crystal/syntax_highlighter.cr @@ -148,19 +148,12 @@ abstract class Crystal::SyntaxHighlighter render :IDENT, token.to_s else case token.value - when :def, :if, :else, :elsif, :end, - :class, :module, :include, :extend, - :while, :until, :do, :yield, :return, :unless, :next, :break, :begin, - :lib, :fun, :type, :struct, :union, :enum, :macro, :out, :require, - :case, :when, :select, :then, :of, :abstract, :rescue, :ensure, :is_a?, - :alias, :pointerof, :sizeof, :instance_sizeof, :offsetof, :as, :as?, :typeof, :for, :in, - :with, :super, :private, :asm, :nil?, :protected, :uninitialized, "new", - :annotation, :verbatim - render :KEYWORD, token.to_s - when :true, :false, :nil + when Keyword::TRUE, Keyword::FALSE, Keyword::NIL render :PRIMITIVE_LITERAL, token.to_s - when :self + when Keyword::SELF render :SELF, token.to_s + when Keyword + render :KEYWORD, token.to_s else render :UNKNOWN, token.to_s end