diff --git a/src/sly/parser/generator/RuleParser.cs b/src/sly/parser/generator/RuleParser.cs index 03a83883..3f764b14 100644 --- a/src/sly/parser/generator/RuleParser.cs +++ b/src/sly/parser/generator/RuleParser.cs @@ -203,7 +203,7 @@ public GroupClause GroupClause(Token id) [Production("groupclause : STRING ")] public GroupClause GroupClauseExplicit(Token explicitToken) { - var clause = BuildTerminalOrNonTerimal(explicitToken.Value,discard:true); + var clause = BuildTerminalOrNonTerimal(explicitToken.Value,discard:false); return new GroupClause(clause); } diff --git a/src/sly/parser/parser/llparser/bnf/RecursiveDescentSyntaxParser.NonTerminal.cs b/src/sly/parser/parser/llparser/bnf/RecursiveDescentSyntaxParser.NonTerminal.cs index 7a15c6f1..fdc9533b 100644 --- a/src/sly/parser/parser/llparser/bnf/RecursiveDescentSyntaxParser.NonTerminal.cs +++ b/src/sly/parser/parser/llparser/bnf/RecursiveDescentSyntaxParser.NonTerminal.cs @@ -38,8 +38,9 @@ public SyntaxParseResult ParseNonTerminal(IList> tokens, string no while (i < rules.Count) { var innerrule = rules[i]; - if (startPosition < tokens.Count && !tokens[startPosition].IsEOS && - innerrule.Match(tokens, startPosition, Configuration)) + if (startPosition < tokens.Count + && (!tokens[startPosition].IsEOS || (tokens[startPosition].IsEOS && innerrule.MayBeEmpty)) + && innerrule.Match(tokens, startPosition, Configuration)) { var innerRuleRes = Parse(tokens, innerrule, startPosition, nonTerminalName, parsingContext); rulesResults.Add(innerRuleRes); @@ -49,9 +50,10 @@ public SyntaxParseResult ParseNonTerminal(IList> tokens, string no innerRuleRes.GetErrors().Count == 0 || other) { greaterIndex = innerRuleRes.EndingPosition; - if (innerRuleRes.GetErrors() != null) + if (innerRuleRes.GetErrors() != null) innerRuleErrors.AddRange(innerRuleRes.GetErrors()); } + if (innerRuleRes.GetErrors() != null) innerRuleErrors.AddRange(innerRuleRes.GetErrors()); } diff --git a/tests/ParserTests/ExplicitTokensTests.cs b/tests/ParserTests/ExplicitTokensTests.cs index a952110f..6b5afa11 100644 --- a/tests/ParserTests/ExplicitTokensTests.cs +++ b/tests/ParserTests/ExplicitTokensTests.cs @@ -9,6 +9,7 @@ using sly.parser; using sly.parser.generator; using sly.parser.generator.visitor; +using sly.parser.parser; using Xunit; namespace ParserTests @@ -24,6 +25,31 @@ public enum Lex Dbl = 2 } + public class ExplicitTokenGroupParser + { + [Production("root : ('a' 'b')* discard")] + public string Root(List> groups, string discard) + { + + return string.Join(" ",groups.Select(group => + { + var r = group.Token(0).Value + "-" + group.Token(1).Value; + return r; + }).ToList())+"_"+discard; + } + + [Production("discard : ('c' 'd'[d])?")] + public string Discard(ValueOption> option) + { + return option.Match(group => group.Token(0).Value + (group.Count == 1), + () => "empty"); + } + } + + public class list + { + } + public class Parse { [Production("program : statement*")] @@ -185,6 +211,38 @@ public void Test() //"(condition:(a == 1.0,(b = ( 1.0 + ( 2.0 * 3.0 ) )),(b = ( 2.0 + a ))))(c = 3.0)" Check.That(r.Result).IsEqualTo("((condition:(a == 1.0,(b = ( 1.0 + ( 2.0 * 3.0 ) )),(b = ( 2.0 + a )))),(c = 3.0))"); } + + [Fact] + public void TestExplicitGroups() + { + var instance = new ExplicitTokenGroupParser(); + var builder = new ParserBuilder(); + var build = builder.BuildParser(instance, ParserType.EBNF_LL_RECURSIVE_DESCENT, "root"); + Check.That(build).IsOk(); + var parser = build.Result; + Check.That(parser).IsNotNull(); + var parsed = parser.Parse("a b a b c d"); + Check.That(parsed).IsOkParsing(); + var result = parsed.Result; + Check.That(result).IsNotNull(); + Check.That(result).IsEqualTo("a-b a-b_cTrue"); + } + + [Fact] + public void TestExplicitGroups2() + { + var instance = new ExplicitTokenGroupParser(); + var builder = new ParserBuilder(); + var build = builder.BuildParser(instance, ParserType.EBNF_LL_RECURSIVE_DESCENT, "root"); + Check.That(build).IsOk(); + var parser = build.Result; + Check.That(parser).IsNotNull(); + var parsed = parser.Parse("a b a b"); + Check.That(parsed).IsOkParsing(); + var result = parsed.Result; + Check.That(result).IsNotNull(); + Check.That(result).IsEqualTo("a-b a-b_empty"); + } }