Introduce skipWhitespace

luau-lang · Dec 16, 2024 · bba1afc · bba1afc
1 parent a14ee98
commit bba1afc
Show file tree

Hide file tree

Showing 4 changed files with 16 additions and 6 deletions.
diff --git a/Ast/include/Luau/Lexer.h b/Ast/include/Luau/Lexer.h
@@ -157,6 +157,7 @@ class Lexer
     Lexer(const char* buffer, std::size_t bufferSize, AstNameTable& names, Position startPosition = {0, 0});
 
     void setSkipComments(bool skip);
+    void setSkipWhitespace(bool skip);
     void setReadNames(bool read);
 
     const Location& previousLocation() const
@@ -165,7 +166,7 @@ class Lexer
     }
 
     const Lexeme& next();
-    const Lexeme& next(bool skipComments, bool updatePrevLocation);
+    const Lexeme& next(bool skipComments, bool skipWhitespace, bool updatePrevLocation);
     void nextline();
 
     Lexeme lookahead();
@@ -229,6 +230,7 @@ class Lexer
     AstNameTable& names;
 
     bool skipComments;
+    bool skipWhitespace;
     bool readNames;
 
     enum class BraceType

diff --git a/Ast/src/Lexer.cpp b/Ast/src/Lexer.cpp
@@ -317,6 +317,7 @@ Lexer::Lexer(const char* buffer, size_t bufferSize, AstNameTable& names, Positio
       )
     , names(names)
     , skipComments(false)
+    , skipWhitespace(true)
     , readNames(true)
 {
 }
@@ -326,17 +327,22 @@ void Lexer::setSkipComments(bool skip)
     skipComments = skip;
 }
 
+void Lexer::setSkipWhitespace(bool skip)
+{
+    skipWhitespace = skip;
+}
+
 void Lexer::setReadNames(bool read)
 {
     readNames = read;
 }
 
 const Lexeme& Lexer::next()
 {
-    return next(this->skipComments, true);
+    return next(this->skipComments, this->skipWhitespace, true);
 }
 
-const Lexeme& Lexer::next(bool skipComments, bool updatePrevLocation)
+const Lexeme& Lexer::next(bool skipComments, bool skipWhitespace, bool updatePrevLocation)
 {
     // in skipComments mode we reject valid comments
     do
@@ -353,7 +359,7 @@ const Lexeme& Lexer::next(bool skipComments, bool updatePrevLocation)
 
         lexeme = readNext();
         updatePrevLocation = false;
-    } while (skipComments && (lexeme.type == Lexeme::Comment || lexeme.type == Lexeme::BlockComment || lexeme.type == Lexeme::Whitespace));
+    } while ((skipComments && (lexeme.type == Lexeme::Comment || lexeme.type == Lexeme::BlockComment)) || (skipWhitespace && lexeme.type == Lexeme::Whitespace));
 
     return lexeme;
 }

diff --git a/Ast/src/Parser.cpp b/Ast/src/Parser.cpp
@@ -3572,7 +3572,7 @@ AstTypeError* Parser::reportMissingTypeError(const Location& parseErrorLocation,
 
 void Parser::nextLexeme()
 {
-    Lexeme::Type type = lexer.next(/* skipComments= */ false, true).type;
+    Lexeme::Type type = lexer.next(/* skipComments= */ false, /* skipWhitespace= */ false, true).type;
 
     while (type == Lexeme::BrokenComment || type == Lexeme::Comment || type == Lexeme::BlockComment || type == Lexeme::Whitespace)
     {
@@ -3598,7 +3598,7 @@ void Parser::nextLexeme()
             hotcomments.push_back({hotcommentHeader, lexeme.location, std::string(text + 1, text + end)});
         }
 
-        type = lexer.next(/* skipComments= */ false, /* updatePrevLocation= */ false).type;
+        type = lexer.next(/* skipComments= */ false, /* skipWhitespace= */ false, /* updatePrevLocation= */ false).type;
     }
 }
 

diff --git a/tests/Lexer.test.cpp b/tests/Lexer.test.cpp
@@ -252,6 +252,7 @@ TEST_CASE("lexer_tokenizes_whitespace")
     Luau::Allocator alloc;
     AstNameTable table(alloc);
     Lexer lexer(testInput.c_str(), testInput.size(), table);
+    lexer.setSkipWhitespace(false);
 
     CHECK_EQ(lexer.next().type, Lexeme::ReservedLocal);
     CHECK_EQ(lexer.next().type, Lexeme::Whitespace);
@@ -282,6 +283,7 @@ TEST_CASE("lexer_tokenizes_multiline_whitespace")
     Luau::Allocator alloc;
     AstNameTable table(alloc);
     Lexer lexer(testInput.c_str(), testInput.size(), table);
+    lexer.setSkipWhitespace(false);
 
     CHECK_EQ(lexer.next().type, Lexeme::ReservedLocal);
     CHECK_EQ(lexer.next().type, Lexeme::Whitespace);