Make lexer discard comments entirely

Alternate solution to abs-lang#388 (previous solution at abs-lang#393). This one skips over comments entirely, so no comment tokens are passed to callers.
gromgit · Aug 6, 2020 · 69705c8 · 69705c8
1 parent 5e90b88
commit 69705c8
Show file tree

Hide file tree

Showing 4 changed files with 8 additions and 10 deletions.
diff --git a/lexer/lexer.go b/lexer/lexer.go
@@ -138,9 +138,10 @@ func (l *Lexer) NextToken() token.Token {
 		}
 	case '/':
 		if l.peekChar() == '/' {
-			tok.Type = token.COMMENT
-			tok.Position = l.position
-			tok.Literal = l.readLine()
+			// comment chunk - skip it
+			_ = l.readLine()
+			l.readChar()
+			return l.NextToken()
 		} else if l.peekChar() == '=' {
 			tok.Type = token.COMP_SLASH
 			tok.Position = l.position
@@ -150,9 +151,10 @@ func (l *Lexer) NextToken() token.Token {
 			tok = l.newToken(token.SLASH)
 		}
 	case '#':
-		tok.Type = token.COMMENT
-		tok.Position = l.position
-		tok.Literal = l.readLine()
+		// comment chunk - skip it
+		_ = l.readLine()
+		l.readChar()
+		return l.NextToken()
 	case '&':
 		if l.peekChar() == '&' {
 			tok.Type = token.AND

diff --git a/lexer/lexer_test.go b/lexer/lexer_test.go
@@ -286,8 +286,6 @@ f hello(x, y) {
 		{token.IDENT, "a"},
 		{token.DOT, "."},
 		{token.IDENT, "prop"},
-		{token.COMMENT, "# Comment"},
-		{token.COMMENT, "// Comment"},
 		{token.IDENT, "hello"},
 		{token.COMMAND, "command; command"},
 		{token.COMMAND, "command2; command2"},

diff --git a/parser/parser.go b/parser/parser.go
@@ -109,7 +109,6 @@ func New(l *lexer.Lexer) *Parser {
 	p.registerPrefix(token.LBRACKET, p.ParseArrayLiteral)
 	p.registerPrefix(token.LBRACE, p.ParseHashLiteral)
 	p.registerPrefix(token.COMMAND, p.parseCommand)
-	p.registerPrefix(token.COMMENT, p.parseComment)
 	p.registerPrefix(token.BREAK, p.parseBreak)
 	p.registerPrefix(token.CONTINUE, p.parseContinue)
 	p.registerPrefix(token.CURRENT_ARGS, p.parseCurrentArgsLiteral)

diff --git a/token/token.go b/token/token.go
@@ -10,7 +10,6 @@ const (
 	IDENT        = "IDENT"  // add, foobar, x, y, ...
 	NUMBER       = "NUMBER" // 1343456, 1.23456
 	STRING       = "STRING" // "foobar"
-	COMMENT      = "#"      // # Comment
 	AT           = "@"      // @ At symbol
 	NULL         = "NULL"   // # null
 	CURRENT_ARGS = "..."    // # ... function args