Merge pull request #256 from lepsa/module-parser-bug

Fixing the lexer bug where newlines were part of identifiers.
Haskell-Things · Apr 10, 2020 · cbfe207 · cbfe207
2 parents ce7870a + 2c676ed
commit cbfe207
Show file tree

Hide file tree

Showing 2 changed files with 13 additions and 4 deletions.
diff --git a/Graphics/Implicit/ExtOpenScad/Parser/Lexer.hs b/Graphics/Implicit/ExtOpenScad/Parser/Lexer.hs
@@ -9,7 +9,11 @@
 
 module Graphics.Implicit.ExtOpenScad.Parser.Lexer (whiteSpace, matchTrue, matchFalse, matchFunction, matchInclude, matchUse, matchIf, matchElse, matchModule, matchLet, matchUndef, matchTok, matchColon, matchSemi, matchComma, matchIdentifier, surroundedBy, matchLT, matchLE, matchGT, matchGE, matchEQ, matchNE, matchCAT, matchOR, matchAND, matchEXP, matchEach, lexer) where
 
-import Prelude (String, Char, Bool(True), (>>), pure)
+import Prelude (String, Char, Bool(True), (>>), pure, not, (&&), ($))
+
+import Data.List (notElem)
+
+import Data.Char (isSpace)
 
 import Data.Functor.Identity (Identity)
 
@@ -23,7 +27,7 @@ import Text.Parsec.Token (GenTokenParser, makeTokenParser, commentStart, comment
 
 import Text.Parsec (char, between)
 
-import Text.Parsec.Char (noneOf)
+import Text.Parsec.Char (noneOf, satisfy)
 
 import Data.Text.Lazy (Text)
 
@@ -35,8 +39,8 @@ openScadStyle
     , commentEnd = "*/"
     , commentLine = "//"
     , nestedComments = True
-    , identStart =  noneOf " ,|[]{}()+-*&^%#@!~`'\"\\/;:.,<>?=1234567890"
-    , identLetter = noneOf " ,|[]{}()+-*&^%#@!~`'\"\\/;:.,<>?="
+    , identStart =  satisfy $ \c -> notElem c (",|[]{}()+-*&^%#@!~`'\"\\/;:.,<>?=1234567890" :: String) && not (isSpace c)
+    , identLetter = satisfy $ \c -> notElem c (",|[]{}()+-*&^%#@!~`'\"\\/;:.,<>?=" :: String) && not (isSpace c)
     , reservedNames = ["module", "function", "if", "else", "let", "each", "true", "false", "undef", "include", "use"]
     , reservedOpNames= ["<=", ">=", "==", "!=", "&&", "||", "++", "^", "<", ">"]
     , caseSensitive = True

diff --git a/tests/ParserSpec/Statement.hs b/tests/ParserSpec/Statement.hs
@@ -118,3 +118,8 @@ statementSpec = do
       "module foo_bar ( x ) { }" --> single (NewModule (Symbol "foo_bar") [(Symbol "x", Nothing)] [])
     it "accepts argument with default" $
       "module foo_bar ( x = 1) { }" --> single (NewModule (Symbol "foo_bar") [(Symbol "x", Just $ num 1)] [])
+    it "accepts split lines" $ do
+      "module foo\n(\nbar\n)\n{}" --> single (NewModule (Symbol "foo") [(Symbol "bar", Nothing)] [])
+  describe "identifiers" $ do
+    it "accepts unicode" $
+      "module 💩 () { }" --> single (NewModule (Symbol "💩") [] [])