Merge pull request #489 from b3b00/fix/explicit-groups

- groups with explicit tokens.
b3b00 · Oct 14, 2024 · e32365d · e32365d
2 parents 632bb1f + ac88eb0
commit e32365d
Show file tree

Hide file tree

Showing 3 changed files with 64 additions and 4 deletions.
diff --git a/src/sly/parser/generator/RuleParser.cs b/src/sly/parser/generator/RuleParser.cs
@@ -203,7 +203,7 @@ public GroupClause<IN> GroupClause(Token<EbnfTokenGeneric> id)
         [Production("groupclause : STRING ")]
         public GroupClause<IN> GroupClauseExplicit(Token<EbnfTokenGeneric> explicitToken)
         {
-            var clause = BuildTerminalOrNonTerimal(explicitToken.Value,discard:true);
+            var clause = BuildTerminalOrNonTerimal(explicitToken.Value,discard:false);
             return new GroupClause<IN>(clause);
         }
 

diff --git a/src/sly/parser/parser/llparser/bnf/RecursiveDescentSyntaxParser.NonTerminal.cs b/src/sly/parser/parser/llparser/bnf/RecursiveDescentSyntaxParser.NonTerminal.cs
@@ -38,8 +38,9 @@ public SyntaxParseResult<IN> ParseNonTerminal(IList<Token<IN>> tokens, string no
         while (i < rules.Count)
         {
             var innerrule = rules[i];
-            if (startPosition < tokens.Count && !tokens[startPosition].IsEOS &&
-                innerrule.Match(tokens, startPosition, Configuration))
+            if (startPosition < tokens.Count 
+                && (!tokens[startPosition].IsEOS || (tokens[startPosition].IsEOS && innerrule.MayBeEmpty)) 
+                && innerrule.Match(tokens, startPosition, Configuration))
             {
                 var innerRuleRes = Parse(tokens, innerrule, startPosition, nonTerminalName, parsingContext);
                 rulesResults.Add(innerRuleRes);
@@ -49,9 +50,10 @@ public SyntaxParseResult<IN> ParseNonTerminal(IList<Token<IN>> tokens, string no
                     innerRuleRes.GetErrors().Count == 0 || other)
                 {
                     greaterIndex = innerRuleRes.EndingPosition;
-                    if (innerRuleRes.GetErrors() != null) 
+                    if (innerRuleRes.GetErrors() != null)
                         innerRuleErrors.AddRange(innerRuleRes.GetErrors());
                 }
+
                 if (innerRuleRes.GetErrors() != null)
                     innerRuleErrors.AddRange(innerRuleRes.GetErrors());
             }

diff --git a/tests/ParserTests/ExplicitTokensTests.cs b/tests/ParserTests/ExplicitTokensTests.cs
@@ -9,6 +9,7 @@
 using sly.parser;
 using sly.parser.generator;
 using sly.parser.generator.visitor;
+using sly.parser.parser;
 using Xunit;
 
 namespace ParserTests
@@ -24,6 +25,31 @@ public enum Lex
         Dbl = 2
     }
 
+    public class ExplicitTokenGroupParser
+    {
+        [Production("root : ('a' 'b')* discard")]
+        public string Root(List<Group<Lex, string>> groups, string discard)
+        {
+
+           return string.Join(" ",groups.Select(group =>
+           {
+               var r = group.Token(0).Value + "-" + group.Token(1).Value;
+               return r;
+           }).ToList())+"_"+discard; 
+        }
+
+        [Production("discard : ('c' 'd'[d])?")]
+        public string Discard(ValueOption<Group<Lex, string>> option)
+        {
+            return option.Match<string>(group =>  group.Token(0).Value + (group.Count == 1),
+                    () => "empty");
+        }
+    }
+
+    public class list<T>
+    {
+    }
+
     public class Parse
     {
         [Production("program : statement*")]
@@ -185,6 +211,38 @@ public void Test()
             //"(condition:(a == 1.0,(b = ( 1.0 + ( 2.0 * 3.0 ) )),(b = ( 2.0 + a ))))(c = 3.0)"
             Check.That(r.Result).IsEqualTo("((condition:(a == 1.0,(b = ( 1.0 + ( 2.0 * 3.0 ) )),(b = ( 2.0 + a )))),(c = 3.0))");
         }
+
+        [Fact]
+        public void TestExplicitGroups()
+        {
+            var instance = new ExplicitTokenGroupParser();
+            var builder = new ParserBuilder<Lex, string>();
+            var build = builder.BuildParser(instance, ParserType.EBNF_LL_RECURSIVE_DESCENT, "root");
+            Check.That(build).IsOk();
+            var parser = build.Result;
+            Check.That(parser).IsNotNull();
+            var parsed = parser.Parse("a b a b c d");
+            Check.That(parsed).IsOkParsing();
+            var result = parsed.Result;
+            Check.That(result).IsNotNull();
+            Check.That(result).IsEqualTo("a-b a-b_cTrue");
+        }
+
+        [Fact]
+        public void TestExplicitGroups2()
+        {
+            var instance = new ExplicitTokenGroupParser();
+            var builder = new ParserBuilder<Lex, string>();
+            var build = builder.BuildParser(instance, ParserType.EBNF_LL_RECURSIVE_DESCENT, "root");
+            Check.That(build).IsOk();
+            var parser = build.Result;
+            Check.That(parser).IsNotNull();
+            var parsed = parser.Parse("a b a b");
+            Check.That(parsed).IsOkParsing();
+            var result = parsed.Result;
+            Check.That(result).IsNotNull();
+            Check.That(result).IsEqualTo("a-b a-b_empty");
+        }
 
     }