From 69705c8f5a6ede9b1f1b32403d8fc0d4e131a052 Mon Sep 17 00:00:00 2001
From: Adrian Ho <the.gromgit@gmail.com>
Date: Wed, 5 Aug 2020 19:11:14 +0800
Subject: [PATCH] Make lexer discard comments entirely

Alternate solution to #388 (previous solution at #393). This one skips over comments entirely, so no comment tokens are passed to callers.
---
 lexer/lexer.go      | 14 ++++++++------
 lexer/lexer_test.go |  2 --
 parser/parser.go    |  1 -
 token/token.go      |  1 -
 4 files changed, 8 insertions(+), 10 deletions(-)

diff --git a/lexer/lexer.go b/lexer/lexer.go
index 43659a38..9c18a4a3 100644
--- a/lexer/lexer.go
+++ b/lexer/lexer.go
@@ -138,9 +138,10 @@ func (l *Lexer) NextToken() token.Token {
 		}
 	case '/':
 		if l.peekChar() == '/' {
-			tok.Type = token.COMMENT
-			tok.Position = l.position
-			tok.Literal = l.readLine()
+			// comment chunk - skip it
+			_ = l.readLine()
+			l.readChar()
+			return l.NextToken()
 		} else if l.peekChar() == '=' {
 			tok.Type = token.COMP_SLASH
 			tok.Position = l.position
@@ -150,9 +151,10 @@ func (l *Lexer) NextToken() token.Token {
 			tok = l.newToken(token.SLASH)
 		}
 	case '#':
-		tok.Type = token.COMMENT
-		tok.Position = l.position
-		tok.Literal = l.readLine()
+		// comment chunk - skip it
+		_ = l.readLine()
+		l.readChar()
+		return l.NextToken()
 	case '&':
 		if l.peekChar() == '&' {
 			tok.Type = token.AND
diff --git a/lexer/lexer_test.go b/lexer/lexer_test.go
index 1e1b331a..5e7e317b 100644
--- a/lexer/lexer_test.go
+++ b/lexer/lexer_test.go
@@ -286,8 +286,6 @@ f hello(x, y) {
 		{token.IDENT, "a"},
 		{token.DOT, "."},
 		{token.IDENT, "prop"},
-		{token.COMMENT, "# Comment"},
-		{token.COMMENT, "// Comment"},
 		{token.IDENT, "hello"},
 		{token.COMMAND, "command; command"},
 		{token.COMMAND, "command2; command2"},
diff --git a/parser/parser.go b/parser/parser.go
index c0abbf6d..ac12e0d9 100644
--- a/parser/parser.go
+++ b/parser/parser.go
@@ -109,7 +109,6 @@ func New(l *lexer.Lexer) *Parser {
 	p.registerPrefix(token.LBRACKET, p.ParseArrayLiteral)
 	p.registerPrefix(token.LBRACE, p.ParseHashLiteral)
 	p.registerPrefix(token.COMMAND, p.parseCommand)
-	p.registerPrefix(token.COMMENT, p.parseComment)
 	p.registerPrefix(token.BREAK, p.parseBreak)
 	p.registerPrefix(token.CONTINUE, p.parseContinue)
 	p.registerPrefix(token.CURRENT_ARGS, p.parseCurrentArgsLiteral)
diff --git a/token/token.go b/token/token.go
index 733f931a..8761079e 100644
--- a/token/token.go
+++ b/token/token.go
@@ -10,7 +10,6 @@ const (
 	IDENT        = "IDENT"  // add, foobar, x, y, ...
 	NUMBER       = "NUMBER" // 1343456, 1.23456
 	STRING       = "STRING" // "foobar"
-	COMMENT      = "#"      // # Comment
 	AT           = "@"      // @ At symbol
 	NULL         = "NULL"   // # null
 	CURRENT_ARGS = "..."    // # ... function args