Skip to content

Commit

Permalink
Make lexer discard comments entirely
Browse files Browse the repository at this point in the history
Alternate solution to #388 (previous solution at #393). This one skips over comments entirely, so no comment tokens are passed to callers.
  • Loading branch information
gromgit authored and odino committed Aug 14, 2020
1 parent 393ef73 commit 48c2462
Show file tree
Hide file tree
Showing 4 changed files with 8 additions and 10 deletions.
14 changes: 8 additions & 6 deletions lexer/lexer.go
Original file line number Diff line number Diff line change
Expand Up @@ -138,9 +138,10 @@ func (l *Lexer) NextToken() token.Token {
}
case '/':
if l.peekChar() == '/' {
tok.Type = token.COMMENT
tok.Position = l.position
tok.Literal = l.readLine()
// comment chunk - skip it
_ = l.readLine()
l.readChar()
return l.NextToken()
} else if l.peekChar() == '=' {
tok.Type = token.COMP_SLASH
tok.Position = l.position
Expand All @@ -150,9 +151,10 @@ func (l *Lexer) NextToken() token.Token {
tok = l.newToken(token.SLASH)
}
case '#':
tok.Type = token.COMMENT
tok.Position = l.position
tok.Literal = l.readLine()
// comment chunk - skip it
_ = l.readLine()
l.readChar()
return l.NextToken()
case '&':
if l.peekChar() == '&' {
tok.Type = token.AND
Expand Down
2 changes: 0 additions & 2 deletions lexer/lexer_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -286,8 +286,6 @@ f hello(x, y) {
{token.IDENT, "a"},
{token.DOT, "."},
{token.IDENT, "prop"},
{token.COMMENT, "# Comment"},
{token.COMMENT, "// Comment"},
{token.IDENT, "hello"},
{token.COMMAND, "command; command"},
{token.COMMAND, "command2; command2"},
Expand Down
1 change: 0 additions & 1 deletion parser/parser.go
Original file line number Diff line number Diff line change
Expand Up @@ -109,7 +109,6 @@ func New(l *lexer.Lexer) *Parser {
p.registerPrefix(token.LBRACKET, p.ParseArrayLiteral)
p.registerPrefix(token.LBRACE, p.ParseHashLiteral)
p.registerPrefix(token.COMMAND, p.parseCommand)
p.registerPrefix(token.COMMENT, p.parseComment)
p.registerPrefix(token.BREAK, p.parseBreak)
p.registerPrefix(token.CONTINUE, p.parseContinue)
p.registerPrefix(token.CURRENT_ARGS, p.parseCurrentArgsLiteral)
Expand Down
1 change: 0 additions & 1 deletion token/token.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@ const (
IDENT = "IDENT" // add, foobar, x, y, ...
NUMBER = "NUMBER" // 1343456, 1.23456
STRING = "STRING" // "foobar"
COMMENT = "#" // # Comment
AT = "@" // @ At symbol
NULL = "NULL" // # null
CURRENT_ARGS = "..." // # ... function args
Expand Down

0 comments on commit 48c2462

Please sign in to comment.