Skip to content

Commit

Permalink
Use enum instead of symbols for keywords in the lexer (#11871)
Browse files Browse the repository at this point in the history
  • Loading branch information
HertzDevil authored Sep 10, 2022
1 parent 64bee9f commit 2166a8b
Show file tree
Hide file tree
Showing 6 changed files with 488 additions and 442 deletions.
18 changes: 9 additions & 9 deletions spec/compiler/lexer/lexer_spec.cr
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ private def it_lexes_many(values, type : Token::Kind)
end
end

private def it_lexes_keywords(keywords)
private def it_lexes_keywords(*keywords : Keyword)
keywords.each do |keyword|
it_lexes keyword.to_s, :IDENT, keyword
end
Expand Down Expand Up @@ -153,13 +153,13 @@ describe "Lexer" do
it_lexes "\n", :NEWLINE
it_lexes "\n\n\n", :NEWLINE
it_lexes "_", :UNDERSCORE
it_lexes_keywords [:def, :if, :else, :elsif, :end, :true, :false, :class, :module, :include,
:extend, :while, :until, :nil, :do, :yield, :return, :unless, :next, :break,
:begin, :lib, :fun, :type, :struct, :union, :enum, :macro, :out, :require,
:case, :when, :select, :then, :of, :abstract, :rescue, :ensure, :is_a?, :alias,
:pointerof, :sizeof, :instance_sizeof, :offsetof, :as, :as?, :typeof, :for, :in,
:with, :self, :super, :private, :protected, :asm, :uninitialized, :nil?,
:annotation, :verbatim]
it_lexes_keywords :def, :if, :else, :elsif, :end, :true, :false, :class, :module, :include,
:extend, :while, :until, :nil, :do, :yield, :return, :unless, :next, :break,
:begin, :lib, :fun, :type, :struct, :union, :enum, :macro, :out, :require,
:case, :when, :select, :then, :of, :abstract, :rescue, :ensure, :alias,
:pointerof, :sizeof, :instance_sizeof, :offsetof, :as, :typeof, :for, :in,
:with, :self, :super, :private, :protected, :asm, :uninitialized,
:annotation, :verbatim, :is_a_question, :as_question, :nil_question, :responds_to_question
it_lexes_idents ["ident", "something", "with_underscores", "with_1", "foo?", "bar!", "fooBar",
"❨╯°□°❩╯︵┻━┻"]
it_lexes_idents ["def?", "if?", "else?", "elsif?", "end?", "true?", "false?", "class?", "while?",
Expand Down Expand Up @@ -443,7 +443,7 @@ describe "Lexer" do
lexer = Lexer.new "end 1"
token = lexer.next_token
token.type.should eq(t :IDENT)
token.value.should eq(:end)
token.value.should eq(Keyword::END)
token = lexer.next_token
token.type.should eq(t :SPACE)
end
Expand Down
12 changes: 6 additions & 6 deletions src/compiler/crystal/syntax/lexer.cr
Original file line number Diff line number Diff line change
Expand Up @@ -858,7 +858,7 @@ module Crystal
next_char
next_char
@token.type = :IDENT
@token.value = :as?
@token.value = :as_question
return @token
else
return check_ident_or_keyword(:as, start)
Expand Down Expand Up @@ -994,7 +994,7 @@ module Crystal
end
when 's'
if next_char == '_' && next_char == 'a' && next_char == '?'
return check_ident_or_keyword(:is_a?, start)
return check_ident_or_keyword(:is_a_question, start)
end
else
# scan_ident
Expand Down Expand Up @@ -1040,7 +1040,7 @@ module Crystal
when 'l'
if peek_next_char == '?'
next_char
return check_ident_or_keyword(:nil?, start)
return check_ident_or_keyword(:nil_question, start)
else
return check_ident_or_keyword(:nil, start)
end
Expand Down Expand Up @@ -1105,7 +1105,7 @@ module Crystal
end
when 'p'
if next_char == 'o' && next_char == 'n' && next_char == 'd' && next_char == 's' && next_char == '_' && next_char == 't' && next_char == 'o' && next_char == '?'
return check_ident_or_keyword(:responds_to?, start)
return check_ident_or_keyword(:responds_to_question, start)
end
else
# scan_ident
Expand Down Expand Up @@ -1426,13 +1426,13 @@ module Crystal
end
end

def check_ident_or_keyword(symbol, start)
def check_ident_or_keyword(keyword : Keyword, start)
if ident_part_or_end?(peek_next_char)
scan_ident(start)
else
next_char
@token.type = :IDENT
@token.value = symbol
@token.value = keyword
end
@token
end
Expand Down
Loading

0 comments on commit 2166a8b

Please sign in to comment.