Merge pull request #497 from b3b00/bugix/#495

Bugix/#495

src/samples/ParserExample/Program.cs

@@ -22,6 +22,7 @@
 using ParserTests.Issue239;
 using ParserTests.Issue332;
 using ParserTests.Issue414;
+using ParserTests.Issue495;
 using ParserTests.lexer;
 using simpleExpressionParser;
 using SimpleTemplate;
@@ -1351,6 +1352,7 @@ print a
         }
         private static void Main(string[] args)
         {
+            TestIssue495();
             //testGenericLexerJson();
             // TestIssue487();
             //BenchSimpleExpression();
@@ -1372,7 +1374,7 @@ private static void Main(string[] args)
             //testJSONEscaped();
             //testJSONNotEscaped();
             // testProfileJSONEscaping(escape:true);
-            testProfileJSONEscaping(escape:false);
+            //testProfileJSONEscaping(escape:false);
             //TestGrammarParser();
             // TestGraphViz();
             // TestGraphViz();
@@ -1803,6 +1805,35 @@ private static void TestIssue487()
                   Console.Write(lexed.Error);
               }
           }
+
+          private static void TestIssue495()
+          {
+              Parser<Issue495Token, string> parser;
+
+              ParserBuilder<Issue495Token, string> builder = new ParserBuilder<Issue495Token, string>("en");
+              var build = builder.BuildParser(new Issue495Parser(), ParserType.EBNF_LL_RECURSIVE_DESCENT, "program");
+              Check.That(build).IsOk();
+              parser = build.Result;
+
+              Check.That(parser).IsNotNull();
+              Check.That(parser.Lexer).IsNotNull();
+              Check.That(parser.Lexer).IsInstanceOf<GenericLexer<Issue495Token>>();
+              var lexer = parser.Lexer as GenericLexer<Issue495Token>;
+              string source = "test = \"3 3\";";
+              var tokenized = lexer.Tokenize(source);
+              Check.That(tokenized).IsOkLexing();
+              var tokens = tokenized.Tokens.MainTokens();
+              Check.That(tokens).CountIs(7);
+              var stringValue = tokens[3];
+              Check.That(stringValue).IsNotNull();
+              Check.That(stringValue.TokenID).IsEqualTo(Issue495Token.StringValue);
+
+
+              var parsed = parser.Parse(source);
+              Check.That(parsed).IsOkParsing();
+              Check.That(parsed.Result).IsEqualTo("test=3 3");
+
+          }
     }
 
     public enum TestGrammarToken

src/sly/lexer/LexerBuilder.cs

@@ -446,8 +446,17 @@ private static BuildResult<ILexer<IN>> BuildGenericSubLexers<IN>(
             var subLexers = GetSubLexers(attributes);
             foreach (var subLexer in subLexers)
             {
-                var x = BuildGenericLexer(subLexer.Value, extensionBuilder, result, lang, explicitTokens);
-                var currentGenericLexer = x.Result as GenericLexer<IN>;
+                BuildResult<ILexer<IN>> b = null;
+                if (subLexer.Key == ModeAttribute.DefaultLexerMode)
+                {
+                    b = BuildGenericLexer(subLexer.Value, extensionBuilder, result, lang, explicitTokens);
+                }
+                else
+                {
+                    b = BuildGenericLexer(subLexer.Value, extensionBuilder, result, lang, null);
+                }
+
+                var currentGenericLexer = b.Result as GenericLexer<IN>;
                 if (genLexer == null)
                 {
                     genLexer = currentGenericLexer;

tests/ParserTests/Issue495/Issue495Parser.cs

@@ -0,0 +1,35 @@
+using System.Collections.Generic;
+using System.Linq;
+using sly.lexer;
+using sly.parser.generator;
+
+namespace ParserTests.Issue495;
+
+public class Issue495Parser
+{
+
+    public Issue495Parser()
+    {
+
+    }
+
+    [Production("STRING: StartQuote StringValue* EndQuote")]
+    public string stringValue(Token<Issue495Token> open, List<Token<Issue495Token>> values, Token<Issue495Token> close)
+    {
+        return string.Join(", ", values.Select(x => x.Value.ToString()));
+    }
+
+    [Production("statement : Identifier Assign STRING End")]
+    [Production("statement : '$'[d] Identifier Assign STRING End")]
+    public string Statement(Token<Issue495Token> id, Token<Issue495Token> assign, string value,
+        Token<Issue495Token> end)
+    {
+        return $"{id.Value}{assign.Value}{value}";
+    }
+
+    [Production("program: statement*")]
+    public string Program(List<string> statements)
+    {
+        return string.Join("\n", statements);
+    }
+}

tests/ParserTests/Issue495/Issue495Tests.cs

@@ -0,0 +1,49 @@
+using NFluent;
+using sly.lexer;
+using sly.parser;
+using sly.parser.generator;
+using Xunit;
+
+namespace ParserTests.Issue495;
+
+public class Issue495Tests
+{
+    public Parser<Issue495Token,string> _parser { get; set; }
+
+    public Parser<Issue495Token, string> GetParser()
+    {
+        if (_parser == null)
+        {
+            ParserBuilder<Issue495Token, string> builder = new ParserBuilder<Issue495Token, string>("en");
+            var build = builder.BuildParser(new Issue495Parser(), ParserType.EBNF_LL_RECURSIVE_DESCENT, "program");
+            Check.That(build).IsOk();
+            _parser = build.Result;
+        }
+
+        return _parser;
+    }
+
+    [Fact]
+    public void TestIssue495()
+    {
+        var parser = GetParser();
+        Check.That(parser).IsNotNull();
+        Check.That(parser.Lexer).IsNotNull();
+        Check.That(parser.Lexer).IsInstanceOf<GenericLexer<Issue495Token>>();
+        var lexer = parser.Lexer as GenericLexer<Issue495Token>;
+        string source = "test = \"3 3\";";
+        var tokenized = lexer.Tokenize(source);
+        Check.That(tokenized).IsOkLexing();
+        var tokens = tokenized.Tokens.MainTokens();
+        Check.That(tokens).CountIs(7);
+        var stringValue = tokens[3];
+        Check.That(stringValue).IsNotNull();
+        Check.That(stringValue.TokenID).IsEqualTo(Issue495Token.StringValue);
+
+
+        var parsed = parser.Parse(source);
+        Check.That(parsed).IsOkParsing();
+        Check.That(parsed.Result).IsEqualTo("test=3 3");
+    } 
+
+}

tests/ParserTests/Issue495/Issue495Token.cs

@@ -0,0 +1,155 @@
+using sly.lexer;
+
+namespace ParserTests.Issue495;
+
+public enum Issue495Token
+{
+    EOF = 0,
+
+    #region Sugar
+    [Comment("//", "/*", "*/")]
+    Comment,
+    [Sugar("++")]
+    Increment,
+    [Sugar("--")]
+    Decrement,
+    [Sugar("+")]
+    Plus,
+    [Sugar("-")]
+    Minus,
+    [Sugar("*")]
+    Multiply,
+    [Sugar("/")]
+    Divide,
+    [Sugar("%")]
+    Remainder,
+    [Sugar(">")]
+    GreaterThan,
+    [Sugar("<")]
+    LessThan,
+    [Sugar("==")]
+    Equal,
+    [Sugar("!=")]
+    NotEqual,
+    [Sugar(">=")]
+    GreaterThanOrEqual,
+    [Sugar("<=")]
+    LessThanOrEqual,
+    [Sugar("=")]
+    Assign,
+    [Sugar("?=")]
+    BooleanAssign,
+    [Sugar("%=")]
+    RemainderAssign,
+    [Sugar("+=")]
+    PlusAssign,
+    [Sugar("-=")]
+    MinusAssign,
+    [Sugar("*=")]
+    MultiplyAssign,
+    [Sugar("/=")]
+    DivideAssign,
+    [Sugar("><")]
+    CompareAssign,
+    [Sugar("??=")]
+    NullColesleAssign,
+    [Sugar("=>")]
+    Arrow,
+    [Sugar("{")]
+    BlockStart,
+    [Sugar("}")]
+    BlockEnd,
+    [Sugar("[")]
+    ListStart,
+    [Sugar("]")]
+    ListEnd,
+    [Sugar("(")]
+    ParenStart,
+    [Sugar(")")]
+    ParenEnd,
+    [Sugar(":")]
+    Colon,
+    [Sugar("::")]
+    Imply,
+    [Sugar(",")]
+    Comma,
+    [Sugar(";")]
+    End,
+    [Sugar("!")]
+    Not,
+    [Sugar("$")]
+    DollarSign,
+    [Sugar(".")]
+    Dot,
+    [Sugar("&")]
+    Deref,
+    [Sugar("||")]
+    Or,
+    [Sugar("&&")]
+    And,
+    [Sugar("\\")]
+    Escape,
+    [Sugar("\"")]
+    [Push("defaultString")]
+    StartQuote,
+    [Sugar("\"")]
+    [Mode("defaultString")]
+    [Pop]
+    EndQuote,
+
+    [UpTo("\"")]
+    [Mode("defaultString")]
+    StringValue,
+    #endregion
+
+    #region Keywords
+    [Keyword("true")]
+    True,
+    [Keyword("false")]
+    False,
+    [Keyword("run")]
+    RunKeyword,
+    [Keyword("import")]
+    ImportKeyword,
+    [Keyword("function")]
+    FunctionKeyword,
+    [Keyword("while")]
+    WhileKeyword,
+    [Keyword("if")]
+    IfKeyword,
+    [Keyword("else")]
+    ElseKeyword,
+    [Keyword("do")]
+    DoKeyword,
+    [Keyword("class")]
+    ClassKeyword,
+    [Keyword("new")]
+    NewKeyword,
+    [Keyword("null")]
+    NullKeyword,
+    [Keyword("command")]
+    CommandKeyword,
+    [Sugar("@s")]
+    SelectorSelf,
+    [Sugar("@p")]
+    SelectorNearest,
+    [Sugar("@a")]
+    SelectorAllPlayers,
+    [Sugar("@r")]
+    SelectorRandomPlayer,
+    [Sugar("@e")]
+    SelectorAllEntities,
+    #endregion
+
+    [Int]
+    Int,
+    [Double]
+    Double,
+    [AlphaNumDashId]
+    Identifier,
+
+
+
+    Variable,
+    Namespace,
+}