fix(lexer) recognize numbers without a leading 0 (#333)

Numbers like "-.123" would not be recognized as a number.

fixes #315

CHANGELOG.md

@@ -27,6 +27,8 @@
     available the original coroutine functions are now used [#329](https://github.com/Tieske/Penlight/pull/329)
   - Fix: in `pl.strict` also predefine global `_PROMPT2`
   - Fix: in `pl.strict` apply `tostring` to the given name, in case it is not a string.
+  - Fix: the lexer would not recognize numbers without leading zero; "-.123".
+    See [#315](https://github.com/Tieske/Penlight/issues/315)
 
 ## 1.7.0 (2019-10-14)
 

docs_topics/06-data.md

@@ -665,7 +665,7 @@ nicely delimited by newlines. For example, here is a snippet of a in-house file
 format which it was my task to maintain:
 
     points
-(818344.1,-20389.7,-0.1),(818337.9,-20389.3,-0.1),(818332.5,-20387.8,-0.1)
+        (818344.1,-20389.7,-0.1),(818337.9,-20389.3,-0.1),(818332.5,-20387.8,-0.1)
         ,(818327.4,-20388,-0.1),(818322,-20387.7,-0.1),(818316.3,-20388.6,-0.1)
         ,(818309.7,-20389.4,-0.1),(818303.5,-20390.6,-0.1),(818295.8,-20388.3,-0.1)
         ,(818290.5,-20386.9,-0.1),(818285.2,-20386.1,-0.1),(818279.3,-20383.6,-0.1)

lua/pl/lexer.lua

@@ -37,11 +37,15 @@ end
 
 local lexer = {}
 
-local NUMBER1 = '^[%+%-]?%d+%.?%d*[eE][%+%-]?%d+'
-local NUMBER2 = '^[%+%-]?%d+%.?%d*'
-local NUMBER3 = '^0x[%da-fA-F]+'
-local NUMBER4 = '^%d+%.?%d*[eE][%+%-]?%d+'
-local NUMBER5 = '^%d+%.?%d*'
+local NUMBER1  = '^[%+%-]?%d+%.?%d*[eE][%+%-]?%d+'
+local NUMBER1a = '^[%+%-]?%d*%.%d+[eE][%+%-]?%d+'
+local NUMBER2  = '^[%+%-]?%d+%.?%d*'
+local NUMBER2a = '^[%+%-]?%d*%.%d+'
+local NUMBER3  = '^0x[%da-fA-F]+'
+local NUMBER4  = '^%d+%.?%d*[eE][%+%-]?%d+'
+local NUMBER4a = '^%d*%.%d+[eE][%+%-]?%d+'
+local NUMBER5  = '^%d+%.?%d*'
+local NUMBER5a = '^%d*%.%d+'
 local IDEN = '^[%a_][%w_]*'
 local WSPACE = '^%s+'
 local STRING1 = "^(['\"])%1" -- empty string
@@ -153,7 +157,9 @@ function lexer.scan(s,matches,filter,options)
                 {NUMBER3,ndump},
                 {IDEN,plain_vdump},
                 {NUMBER1,ndump},
+                {NUMBER1a,ndump},
                 {NUMBER2,ndump},
+                {NUMBER2a,ndump},
                 {STRING1,sdump},
                 {STRING2,sdump},
                 {STRING3,sdump},
@@ -325,7 +331,9 @@ function lexer.lua(s,filter,options)
             {NUMBER3,ndump},
             {IDEN,lua_vdump},
             {NUMBER4,ndump},
+            {NUMBER4a,ndump},
             {NUMBER5,ndump},
+            {NUMBER5a,ndump},
             {STRING1,sdump},
             {STRING2,sdump},
             {STRING3,sdump},
@@ -375,7 +383,9 @@ function lexer.cpp(s,filter,options)
             {NUMBER3,ndump},
             {IDEN,cpp_vdump},
             {NUMBER4,ndump},
+            {NUMBER4a,ndump},
             {NUMBER5,ndump},
+            {NUMBER5a,ndump},
             {CHAR1,chdump},
             {CHAR2,chdump},
             {CHAR3,chdump},

tests/test-lexer.lua

@@ -137,3 +137,10 @@ asserteq(lexer.lineno(iter), 3)
 iter()
 iter()
 asserteq(lexer.lineno(iter), 3)
+
+do  -- numbers without leading zero; ".123"
+  local s = 'hello = +.234'
+  test_scan(s, {space=true}, {number=true}, {
+    {'iden', 'hello'}, {'=', '='}, {'number', .234}
+  })
+end