From 33ccc38eff4c5873576f4f436b7da0e5d24a1f2e Mon Sep 17 00:00:00 2001 From: b3b00 Date: Sat, 21 Dec 2024 08:39:50 +0100 Subject: [PATCH 1/7] wip --- src/sly/parser/generator/OperationMetaData.cs | 6 +- src/sly/parser/generator/ParserBuilder.cs | 2 +- ...ecursiveDescentSyntaxParser.Expressions.cs | 22 +++--- ...EBNFRecursiveDescentSyntaxParser.Option.cs | 6 +- src/sly/parser/syntax/grammar/Rule.cs | 54 ++++++++++---- tests/ParserTests/FluentTests.cs | 72 ++++++++++++++++++- 6 files changed, 127 insertions(+), 35 deletions(-) diff --git a/src/sly/parser/generator/OperationMetaData.cs b/src/sly/parser/generator/OperationMetaData.cs index acabdaaa..cfa6f069 100644 --- a/src/sly/parser/generator/OperationMetaData.cs +++ b/src/sly/parser/generator/OperationMetaData.cs @@ -30,7 +30,7 @@ public OperationMetaData(int precedence, Associativity assoc, Func { Precedence = precedence; Associativity = assoc; - VisitorLambda = lambda; + LambdaVisitor = lambda; OperatorToken = oper; Affix = affix; NodeName = nodeName; @@ -40,7 +40,7 @@ public OperationMetaData(int precedence, Associativity assoc, Func { Precedence = precedence; Associativity = assoc; - VisitorLambda = lambda; + LambdaVisitor = lambda; ExplicitOperatorToken = oper; Affix = affix; NodeName = nodeName; @@ -52,7 +52,7 @@ public OperationMetaData(int precedence, Associativity assoc, Func public MethodInfo VisitorMethod { get; set; } - public Func VisitorLambda { get; set; } + public Func LambdaVisitor { get; set; } public IN OperatorToken { get; set; } diff --git a/src/sly/parser/generator/ParserBuilder.cs b/src/sly/parser/generator/ParserBuilder.cs index a8139033..6d3b2fe3 100644 --- a/src/sly/parser/generator/ParserBuilder.cs +++ b/src/sly/parser/generator/ParserBuilder.cs @@ -471,7 +471,7 @@ private BuildResult> CheckVisitorSignature(BuildResult ManageExpressionRules(Rule rule, SyntaxNo if (operatorIndex >= 0 && node.Children[operatorIndex] is SyntaxLeaf operatorNode) { - var visitor = rule.GetVisitor(operatorNode.Token.TokenID); - if (visitor != null) + var token = operatorNode.Token; + string key = token.IsExplicit ? $"'{token.Value}'" : token.TokenID.ToString(); + var visitor = rule.GetVisitorMethod(key); + var operation = rule.GetOperation(key); + if (operation != null) { - node.Visitor = visitor; - node.Operation = rule.GetOperation(operatorNode.Token.TokenID); - } - var lmabdaVisitor = rule.getLambdaVisitor(operatorNode.Token.TokenID); - if (lmabdaVisitor != null) - { - node.LambdaVisitor = lmabdaVisitor; - node.Operation = rule.GetOperation(operatorNode.Token.TokenID); + node.Visitor = operation.VisitorMethod; + node.LambdaVisitor = operation.LambdaVisitor; + node.Operation = rule.GetOperation(key); } } break; } case false: - node.LambdaVisitor = rule.getLambdaVisitor(); - node.Visitor = rule.GetVisitor(); + node.LambdaVisitor = rule.getLambdaVisitor(null); + node.Visitor = rule.GetVisitorMethod(null); break; } diff --git a/src/sly/parser/parser/llparser/ebnf/EBNFRecursiveDescentSyntaxParser.Option.cs b/src/sly/parser/parser/llparser/ebnf/EBNFRecursiveDescentSyntaxParser.Option.cs index 9a6ac12d..e43edd59 100644 --- a/src/sly/parser/parser/llparser/ebnf/EBNFRecursiveDescentSyntaxParser.Option.cs +++ b/src/sly/parser/parser/llparser/ebnf/EBNFRecursiveDescentSyntaxParser.Option.cs @@ -65,7 +65,7 @@ public SyntaxParseResult ParseOption(IList> tokens, OptionCla { IsError = false, Root = new OptionSyntaxNode(rule.NonTerminalName, new List>(), - rule.GetVisitor()), + rule.GetVisitorMethod()), EndingPosition = position }; } @@ -79,7 +79,7 @@ public SyntaxParseResult ParseOption(IList> tokens, OptionCla var children = new List> { innerResult.Root }; if (innerResult.IsError) children.Clear(); result.Root = new OptionSyntaxNode(rule.NonTerminalName, children, - rule.GetVisitor()); + rule.GetVisitorMethod()); (result.Root as OptionSyntaxNode).IsGroupOption = clause.IsGroupOption; result.EndingPosition = position; break; @@ -90,7 +90,7 @@ public SyntaxParseResult ParseOption(IList> tokens, OptionCla { var children = new List> { innerResult.Root }; result.Root = - new OptionSyntaxNode(rule.NonTerminalName, children, rule.GetVisitor()); + new OptionSyntaxNode(rule.NonTerminalName, children, rule.GetVisitorMethod()); result.EndingPosition = innerResult.EndingPosition; result.HasByPassNodes = innerResult.HasByPassNodes; } diff --git a/src/sly/parser/syntax/grammar/Rule.cs b/src/sly/parser/syntax/grammar/Rule.cs index f3b35106..b224b334 100644 --- a/src/sly/parser/syntax/grammar/Rule.cs +++ b/src/sly/parser/syntax/grammar/Rule.cs @@ -14,7 +14,8 @@ public class Rule : GrammarNode where IN : struct public Rule() { Clauses = new List>(); - VisitorMethodsForOperation = new Dictionary>(); + VisitorMethodsForOperation = new Dictionary>(); + LambdaVisitorsForOperation = new Dictionary>(); Visitor = null; IsSubRule = false; NodeName = ""; @@ -27,7 +28,9 @@ public Rule() public bool IsByPassRule { get; set; } = false; // visitors for operation rules - private Dictionary> VisitorMethodsForOperation { get; } + private Dictionary> VisitorMethodsForOperation { get; } + + private Dictionary> LambdaVisitorsForOperation { get; } // visitor for classical rules private MethodInfo Visitor { get; set; } @@ -87,17 +90,15 @@ public bool ContainsSubRule || Clauses.Count == 1 && Clauses[0].MayBeEmpty(); - public OperationMetaData GetOperation(IN token = default) + public OperationMetaData GetOperation(string token) { + OperationMetaData operation = null; if (IsExpressionRule) { - var operation = VisitorMethodsForOperation.TryGetValue(token, out var value) - ? value - : null; - return operation; + LambdaVisitorsForOperation.TryGetValue(token, out operation); } - return null; + return operation; } public List> GetOperations() @@ -110,15 +111,15 @@ public List> GetOperations() return null; } - public Func getLambdaVisitor(IN token = default) + public Func getLambdaVisitor(string token ) { Func visitor = null; - if (IsExpressionRule) + if (IsExpressionRule && !string.IsNullOrEmpty(token)) { - var operation = VisitorMethodsForOperation.TryGetValue(token, out var value) + var operation = LambdaVisitorsForOperation.TryGetValue(token.ToString(), out var value) ? value : null; - visitor = operation?.VisitorLambda; + visitor = operation?.LambdaVisitor; } else { @@ -133,10 +134,10 @@ public void SetLambdaVisitor(Func lambdaVisitor) LambdaVisitor = lambdaVisitor; } - public MethodInfo GetVisitor(IN token = default) + public MethodInfo GetVisitorMethod(string token = null) { MethodInfo visitor = null; - if (IsExpressionRule) + if (IsExpressionRule && !string.IsNullOrEmpty(token)) { var operation = VisitorMethodsForOperation.TryGetValue(token, out var value) ? value @@ -151,6 +152,24 @@ public MethodInfo GetVisitor(IN token = default) return visitor; } + public Func GetLambdaVisitor(string token = null) + { + Func visitor = null; + if (IsExpressionRule && !string.IsNullOrEmpty(token)) + { + var operation = LambdaVisitorsForOperation.TryGetValue(token, out var value) + ? value + : null; + visitor = operation?.LambdaVisitor; + } + else + { + visitor = LambdaVisitor; + } + + return visitor; + } + public void SetVisitor(MethodInfo visitor) { Visitor = visitor; @@ -163,7 +182,12 @@ public void SetVisitor(Func visitorLambda) public void SetVisitor(OperationMetaData operation) { - VisitorMethodsForOperation[operation.OperatorToken] = operation; + if (operation.VisitorMethod != null) + VisitorMethodsForOperation[operation.Operatorkey] = operation; + else if (operation.LambdaVisitor != null) + { + LambdaVisitorsForOperation[operation.Operatorkey] = operation; + } } diff --git a/tests/ParserTests/FluentTests.cs b/tests/ParserTests/FluentTests.cs index e046ba49..f2db41c9 100644 --- a/tests/ParserTests/FluentTests.cs +++ b/tests/ParserTests/FluentTests.cs @@ -190,7 +190,7 @@ public void TestRegexFluentLexer() .IgnoreEol(true) .Regex(FluentToken.HELLO, "hello") .Regex(FluentToken.WORLD, "world") - .Regex(FluentToken.THREE_DOT, ".{3}") + .Regex(FluentToken.THREE_DOT, "\\.{3}") .Regex(FluentToken.EOL, "[\\r\\n]*", true, true) .Build("en"); Check.That(lexer).IsOk(); @@ -326,4 +326,74 @@ public void TestFluentExpressionParser() Check.That(result.Result).IsEqualTo(-1 + 2); } + + [Fact] + public void TestFluentExpressionParserWithExplicits() + { + var lexer = FluentLexerBuilder.NewBuilder() + .IgnoreEol(true) + .IgnoreWhiteSpace(true) + .IgnoreKeywordCase(true) + .Int(ExpressionToken.INT) + .Sugar(ExpressionToken.PLUS, "+") + .Sugar(ExpressionToken.MINUS, "-") + .Sugar(ExpressionToken.TIMES, "*") + .Sugar(ExpressionToken.DIVIDE, "/"); + + var build = FluentEBNFParserBuilder.NewBuilder(new FluentTests(), "expression","en") + .Production($"expression : {nameof(FluentTests)}_expressions", args => + { + return (int)args[0]; + }) + .Left("'moins'", 10, (object[] args) => + { + var l = (int)args[0]; + var r = (int)args[2]; + return l - r; + }) + .Left("'plus'", 10, (object[] args) => + { + var l = (int)args[0]; + var r = (int)args[2]; + return l + r; + }) + .Right("'fois'", 50, (object[] args) => + { + var l = (int)args[0]; + var r = (int)args[2]; + return l * r; + }) + .Left("'div'", 50, (object[] args) => + { + var l = (int)args[0]; + var r = (int)args[2]; + return l / r; + }) + .Prefix("'moins'", 100, (object[] args) => + { + return -(int)args[1]; + }) + .Operand("operand : INT", args => + { + var v = args[0] as Token; + return v.IntValue; + }) + .WithLexerbuilder(lexer) + .BuildParser(); + Check.That(build).IsOk(); + var parser = build.Result; + var result = parser.Parse("2 plus 2"); + Check.That(result).IsOkParsing(); + Check.That(result.Result).IsEqualTo(2 + 2); + result = parser.Parse("1 moins 2 moins 3"); + Check.That(result).IsOkParsing(); + Check.That(result.Result).IsEqualTo(1 - 2 - 3); + result = parser.Parse("1 plus 2 fois 3"); + Check.That(result).IsOkParsing(); + Check.That(result.Result).IsEqualTo(1 + 2 * 3); + result = parser.Parse("moins 1 plus 2 "); + Check.That(result).IsOkParsing(); + Check.That(result.Result).IsEqualTo(-1 + 2); + + } } \ No newline at end of file From 720e2e35ca3d26e13cac505c345960adddabf37f Mon Sep 17 00:00:00 2001 From: b3b00 Date: Sat, 21 Dec 2024 09:15:36 +0100 Subject: [PATCH 2/7] wip --- .../llparser/bnf/RecursiveDescentSyntaxParser.Expressions.cs | 5 ++--- src/sly/parser/syntax/grammar/Rule.cs | 5 ++++- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/sly/parser/parser/llparser/bnf/RecursiveDescentSyntaxParser.Expressions.cs b/src/sly/parser/parser/llparser/bnf/RecursiveDescentSyntaxParser.Expressions.cs index 909d4e8d..bd4b3cc7 100644 --- a/src/sly/parser/parser/llparser/bnf/RecursiveDescentSyntaxParser.Expressions.cs +++ b/src/sly/parser/parser/llparser/bnf/RecursiveDescentSyntaxParser.Expressions.cs @@ -37,14 +37,13 @@ protected SyntaxNode ManageExpressionRules(Rule rule, SyntaxNo if (operatorIndex >= 0 && node.Children[operatorIndex] is SyntaxLeaf operatorNode) { var token = operatorNode.Token; - string key = token.IsExplicit ? $"'{token.Value}'" : token.TokenID.ToString(); - var visitor = rule.GetVisitorMethod(key); + string key = token.IsExplicit ? $"'{token.Value}'" : token.TokenID.ToString(); var operation = rule.GetOperation(key); if (operation != null) { node.Visitor = operation.VisitorMethod; node.LambdaVisitor = operation.LambdaVisitor; - node.Operation = rule.GetOperation(key); + node.Operation = operation; } } diff --git a/src/sly/parser/syntax/grammar/Rule.cs b/src/sly/parser/syntax/grammar/Rule.cs index b224b334..2072d71d 100644 --- a/src/sly/parser/syntax/grammar/Rule.cs +++ b/src/sly/parser/syntax/grammar/Rule.cs @@ -95,7 +95,10 @@ public OperationMetaData GetOperation(string token) OperationMetaData operation = null; if (IsExpressionRule) { - LambdaVisitorsForOperation.TryGetValue(token, out operation); + if (!LambdaVisitorsForOperation.TryGetValue(token, out operation)) + { + VisitorMethodsForOperation.TryGetValue(token, out operation); + } } return operation; From 1f8592d335e051971869e1f456dbd44d2a668f5e Mon Sep 17 00:00:00 2001 From: b3b00 Date: Sun, 22 Dec 2024 18:44:43 +0100 Subject: [PATCH 3/7] fixes --- src/sly/parser/generator/EBNFParserBuilder.cs | 1 + .../parser/generator/ExpressionRulesGenerator.cs | 1 + src/sly/parser/generator/OperationMetaData.cs | 2 +- .../generator/visitor/EBNFSyntaxTreeVisitor.cs | 8 +++++++- .../RecursiveDescentSyntaxParser.Expressions.cs | 5 +++-- .../ebnf/EBNFRecursiveDescentSyntaxParser.cs | 1 + src/sly/parser/syntax/grammar/Rule.cs | 16 ++++++++++++++-- src/sly/parser/syntax/tree/SyntaxNode.cs | 4 +++- tests/ParserTests/ExpressionGeneratorTests.cs | 5 +++-- 9 files changed, 34 insertions(+), 9 deletions(-) diff --git a/src/sly/parser/generator/EBNFParserBuilder.cs b/src/sly/parser/generator/EBNFParserBuilder.cs index 2875aaa2..b2c49fc1 100644 --- a/src/sly/parser/generator/EBNFParserBuilder.cs +++ b/src/sly/parser/generator/EBNFParserBuilder.cs @@ -178,6 +178,7 @@ protected virtual ParserConfiguration ExtractEbnfParserConfiguration(Ty if (!parseResult.IsError) { var rule = (Rule)parseResult.Result; + rule.ForcedName = nodeName != null; rule.NodeName = nodeName; rule.SubNodeNames = subNodeNames; rule.RuleString = ruleString; diff --git a/src/sly/parser/generator/ExpressionRulesGenerator.cs b/src/sly/parser/generator/ExpressionRulesGenerator.cs index 741d637a..46fcc3ee 100644 --- a/src/sly/parser/generator/ExpressionRulesGenerator.cs +++ b/src/sly/parser/generator/ExpressionRulesGenerator.cs @@ -256,6 +256,7 @@ private void GenerateExpressionParser(ParserConfiguration configuration rule.IsByPassRule = true; rule.IsExpressionRule = true; rule.ExpressionAffix = Affix.NotOperator; + rule.SetLambdaVisitor((args) => (OUT)args[0]); configuration.NonTerminals[entrypoint.Name] = entrypoint; entrypoint.Rules.Add(rule); } diff --git a/src/sly/parser/generator/OperationMetaData.cs b/src/sly/parser/generator/OperationMetaData.cs index cfa6f069..6b5af43c 100644 --- a/src/sly/parser/generator/OperationMetaData.cs +++ b/src/sly/parser/generator/OperationMetaData.cs @@ -56,7 +56,7 @@ public OperationMetaData(int precedence, Associativity assoc, Func public IN OperatorToken { get; set; } - public string Operatorkey => NodeName ?? (IsExplicitOperatorToken ? ExplicitOperatorToken : OperatorToken.ToString()); + public string Operatorkey => (IsExplicitOperatorToken ? ExplicitOperatorToken : OperatorToken.ToString()); public Affix Affix { get; set; } diff --git a/src/sly/parser/generator/visitor/EBNFSyntaxTreeVisitor.cs b/src/sly/parser/generator/visitor/EBNFSyntaxTreeVisitor.cs index aed33e10..e9d04a9c 100644 --- a/src/sly/parser/generator/visitor/EBNFSyntaxTreeVisitor.cs +++ b/src/sly/parser/generator/visitor/EBNFSyntaxTreeVisitor.cs @@ -86,7 +86,7 @@ private SyntaxVisitorResult Visit(OptionSyntaxNode node, objec private SyntaxVisitorResult Visit(SyntaxNode node, object context = null) { var result = SyntaxVisitorResult.NoneResult(); - if (node.LambdaVisitor != null || node.Visitor != null || node.IsByPassNode) + if (!node .IsByPassNode && (node.LambdaVisitor != null || node.Visitor != null || node.IsByPassNode)) { int parametersArrayLength = node.Children.Count + (context is NoContext ? 0 : 1); var parameters = new object[parametersArrayLength]; @@ -176,6 +176,12 @@ private SyntaxVisitorResult Visit(SyntaxNode node, object cont } } } + else if (node.IsByPassNode) + { + var child = node.Children[0]; + var v = Visit(child, context); + return v; + } return result; } diff --git a/src/sly/parser/parser/llparser/bnf/RecursiveDescentSyntaxParser.Expressions.cs b/src/sly/parser/parser/llparser/bnf/RecursiveDescentSyntaxParser.Expressions.cs index bd4b3cc7..32d1dbde 100644 --- a/src/sly/parser/parser/llparser/bnf/RecursiveDescentSyntaxParser.Expressions.cs +++ b/src/sly/parser/parser/llparser/bnf/RecursiveDescentSyntaxParser.Expressions.cs @@ -37,7 +37,7 @@ protected SyntaxNode ManageExpressionRules(Rule rule, SyntaxNo if (operatorIndex >= 0 && node.Children[operatorIndex] is SyntaxLeaf operatorNode) { var token = operatorNode.Token; - string key = token.IsExplicit ? $"'{token.Value}'" : token.TokenID.ToString(); + string key = node.ForcedName && node.Name != null ? node.Name : (token.IsExplicit ? $"'{token.Value}'" : token.TokenID.ToString()); var operation = rule.GetOperation(key); if (operation != null) { @@ -46,13 +46,14 @@ protected SyntaxNode ManageExpressionRules(Rule rule, SyntaxNo node.Operation = operation; } } - break; } case false: + { node.LambdaVisitor = rule.getLambdaVisitor(null); node.Visitor = rule.GetVisitorMethod(null); break; + } } return node; diff --git a/src/sly/parser/parser/llparser/ebnf/EBNFRecursiveDescentSyntaxParser.cs b/src/sly/parser/parser/llparser/ebnf/EBNFRecursiveDescentSyntaxParser.cs index a8a456f3..9cecf411 100644 --- a/src/sly/parser/parser/llparser/ebnf/EBNFRecursiveDescentSyntaxParser.cs +++ b/src/sly/parser/parser/llparser/ebnf/EBNFRecursiveDescentSyntaxParser.cs @@ -162,6 +162,7 @@ public override SyntaxParseResult Parse(IList> tokens, Rule( nonTerminalName, children); + node.ForcedName = rule.ForcedName; node.Name = string.IsNullOrEmpty(rule.NodeName) ? nonTerminalName : rule.NodeName; node.ExpressionAffix = rule.ExpressionAffix; node = ManageExpressionRules(rule, node); diff --git a/src/sly/parser/syntax/grammar/Rule.cs b/src/sly/parser/syntax/grammar/Rule.cs index 2072d71d..12eeabf4 100644 --- a/src/sly/parser/syntax/grammar/Rule.cs +++ b/src/sly/parser/syntax/grammar/Rule.cs @@ -89,15 +89,27 @@ public bool ContainsSubRule || Clauses.Count == 0 || Clauses.Count == 1 && Clauses[0].MayBeEmpty(); + public bool ForcedName { get; set; } + public OperationMetaData GetOperation(string token) { OperationMetaData operation = null; if (IsExpressionRule) { - if (!LambdaVisitorsForOperation.TryGetValue(token, out operation)) + if (LambdaVisitorsForOperation.ContainsKey(token)) + { + return LambdaVisitorsForOperation[token]; + } + + if (VisitorMethodsForOperation.ContainsKey(token)) + { + return VisitorMethodsForOperation[token]; + } + else { - VisitorMethodsForOperation.TryGetValue(token, out operation); + ; + } } diff --git a/src/sly/parser/syntax/tree/SyntaxNode.cs b/src/sly/parser/syntax/tree/SyntaxNode.cs index 0997d1b2..9d045183 100644 --- a/src/sly/parser/syntax/tree/SyntaxNode.cs +++ b/src/sly/parser/syntax/tree/SyntaxNode.cs @@ -109,7 +109,9 @@ public ISyntaxNode Right } } } - + + public bool ForcedName { get; set; } + public string Dump(string tab) { StringBuilder builder = new StringBuilder(); diff --git a/tests/ParserTests/ExpressionGeneratorTests.cs b/tests/ParserTests/ExpressionGeneratorTests.cs index 140b6ece..3b10dee0 100644 --- a/tests/ParserTests/ExpressionGeneratorTests.cs +++ b/tests/ParserTests/ExpressionGeneratorTests.cs @@ -8,6 +8,7 @@ using sly.parser.generator; using Xunit; using ParserTests.Issue184; +using sly.parser.generator.visitor; using ExpressionToken = expressionparser.ExpressionToken; namespace ParserTests @@ -236,12 +237,12 @@ public void TestBuild() Check.That(nt.Rules[1].NodeName).IsEqualTo("integer"); Check.That(nt.Rules[2].NodeName).IsEqualTo("group"); nt = nonterminals[3]; - Check.That(nt.Rules).IsSingle(); + Check.That(nt.Rules).CountIs(1); Check.That(nt.Name).Contains("PLUS"); Check.That(nt.Name).Contains("MINUS"); nt = nonterminals[4]; Check.That(nt.Rules).IsSingle(); - Check.That(nt.Name).IsEqualTo("multiplication_or_division"); + Check.That(nt.Name).IsEqualTo("TIMES_DIVIDE"); nt = nonterminals[5]; Check.That(nt.Rules).CountIs(3); Check.That(nt.Name).Contains("MINUS"); From b615229d6ad94a241951e56e38a2e0a5599721d4 Mon Sep 17 00:00:00 2001 From: b3b00 Date: Mon, 23 Dec 2024 10:29:18 +0100 Subject: [PATCH 4/7] fixes --- .../SimpleExpressionParser.cs | 11 +++++--- src/sly/parser/generator/EBNFParserBuilder.cs | 1 - .../generator/ExpressionRulesGenerator.cs | 17 +++++++---- .../visitor/EBNFSyntaxTreeVisitor.cs | 2 +- ...ecursiveDescentSyntaxParser.Expressions.cs | 8 +++--- ...ecursiveDescentSyntaxParser.Expressions.cs | 13 ++++++++- .../ebnf/EBNFRecursiveDescentSyntaxParser.cs | 2 +- src/sly/parser/syntax/grammar/Rule.cs | 2 +- tests/ParserTests/ExpressionGeneratorTests.cs | 28 ++++++++++++++----- 9 files changed, 59 insertions(+), 25 deletions(-) diff --git a/src/samples/SimpleExpressionParser/SimpleExpressionParser.cs b/src/samples/SimpleExpressionParser/SimpleExpressionParser.cs index a52bb024..2898a1aa 100644 --- a/src/samples/SimpleExpressionParser/SimpleExpressionParser.cs +++ b/src/samples/SimpleExpressionParser/SimpleExpressionParser.cs @@ -15,6 +15,7 @@ public class SimpleExpressionParser [Operation((int) ExpressionToken.PLUS, Affix.InFix, Associativity.Right, 10)] [Operation("MINUS", Affix.InFix, Associativity.Left, 10)] + [NodeName("addition_or_substraction")] public double BinaryTermExpression(double left, Token operation, double right) { double result = 0; @@ -61,11 +62,13 @@ public double BinaryFactorExpression(double left, Token operati [Prefix((int) ExpressionToken.MINUS, Associativity.Right, 100)] + [NodeName("minus")] public double PreFixExpression(Token operation, double value) { return -value; } + [NodeName("factorial")] [Postfix((int) ExpressionToken.FACTORIAL, Associativity.Right, 100)] public double PostFixExpression(double value, Token operation) { @@ -76,7 +79,7 @@ public double PostFixExpression(double value, Token operation) [Operand] [Production("operand : primary_value")] - [NodeName("double")] + [NodeName("op_prim")] public double OperandValue(double value) { return value; @@ -84,21 +87,21 @@ public double OperandValue(double value) [Production("primary_value : DOUBLE")] - [NodeName("double")] + [NodeName("prim_double")] public double OperandDouble(Token value) { return value.DoubleValue; } [Production("primary_value : INT")] - [NodeName("integer")] + [NodeName("prim_int")] public double OperandInt(Token value) { return value.DoubleValue; } [Production("primary_value : LPAREN SimpleExpressionParser_expressions RPAREN")] - [NodeName("group")] + [NodeName("prim_group")] public double OperandParens(Token lparen, double value, Token rparen) { return value; diff --git a/src/sly/parser/generator/EBNFParserBuilder.cs b/src/sly/parser/generator/EBNFParserBuilder.cs index b2c49fc1..b6501d7f 100644 --- a/src/sly/parser/generator/EBNFParserBuilder.cs +++ b/src/sly/parser/generator/EBNFParserBuilder.cs @@ -167,7 +167,6 @@ protected virtual ParserConfiguration ExtractEbnfParserConfiguration(Ty var nodeNames = (NodeNameAttribute[])m.GetCustomAttributes(typeof(NodeNameAttribute), true); string nodeName = nodeNames != null && nodeNames.Any() ? nodeNames[0].Name : null; - var subNodeNamesAttributes = (SubNodeNamesAttribute[])m.GetCustomAttributes(typeof(SubNodeNamesAttribute), true); string[] subNodeNames = subNodeNamesAttributes != null && subNodeNamesAttributes.Any() ? subNodeNamesAttributes[0].Names : null; diff --git a/src/sly/parser/generator/ExpressionRulesGenerator.cs b/src/sly/parser/generator/ExpressionRulesGenerator.cs index 46fcc3ee..99bea51f 100644 --- a/src/sly/parser/generator/ExpressionRulesGenerator.cs +++ b/src/sly/parser/generator/ExpressionRulesGenerator.cs @@ -284,7 +284,9 @@ private NonTerminal BuildPrecedenceNonTerminal(string name, string next { ExpressionAffix = Affix.InFix, IsExpressionRule = true, - NonTerminalName = name + NonTerminalName = name, + NodeName = InFixOps[0].NodeName, + ForcedName = !string.IsNullOrEmpty(InFixOps[0].NodeName) }; rule.Clauses.Add(new NonTerminalClause(nextName)); @@ -296,7 +298,8 @@ private NonTerminal BuildPrecedenceNonTerminal(string name, string next rule.SetVisitor(x); rule.IsExpressionRule = true; rule.IsInfixExpressionRule = true; - + rule.NodeName = x.NodeName; + rule.ForcedName = !string.IsNullOrEmpty(x.NodeName); }); nonTerminal.Rules.Add(rule); if (isLastLevel) @@ -307,7 +310,7 @@ private NonTerminal BuildPrecedenceNonTerminal(string name, string next ExpressionAffix = Affix.NotOperator, IsExpressionRule = true, NonTerminalName = name, - IsByPassRule = true + IsByPassRule = true, }; rule2.Clauses.Add(new NonTerminalClause(nextName)); @@ -337,7 +340,9 @@ private NonTerminal BuildPrecedenceNonTerminal(string name, string next var rule = new Rule { ExpressionAffix = Affix.PreFix, - IsExpressionRule = true + IsExpressionRule = true, + NodeName = PreFixOps[0].NodeName, + ForcedName = !string.IsNullOrEmpty(PreFixOps[0].NodeName) }; rule.Clauses.Add(PreFixClauses.Count == 1 ? PreFixClauses[0] : new ChoiceClause(PreFixClauses)); @@ -366,7 +371,9 @@ private NonTerminal BuildPrecedenceNonTerminal(string name, string next var rule = new Rule { ExpressionAffix = Affix.PostFix, - IsExpressionRule = true + IsExpressionRule = true, + NodeName = PostFixOps[0].NodeName, + ForcedName = !string.IsNullOrEmpty(PostFixOps[0].NodeName) }; rule.Clauses.Add(new NonTerminalClause(nextName)); diff --git a/src/sly/parser/generator/visitor/EBNFSyntaxTreeVisitor.cs b/src/sly/parser/generator/visitor/EBNFSyntaxTreeVisitor.cs index e9d04a9c..351e5a38 100644 --- a/src/sly/parser/generator/visitor/EBNFSyntaxTreeVisitor.cs +++ b/src/sly/parser/generator/visitor/EBNFSyntaxTreeVisitor.cs @@ -86,7 +86,7 @@ private SyntaxVisitorResult Visit(OptionSyntaxNode node, objec private SyntaxVisitorResult Visit(SyntaxNode node, object context = null) { var result = SyntaxVisitorResult.NoneResult(); - if (!node .IsByPassNode && (node.LambdaVisitor != null || node.Visitor != null || node.IsByPassNode)) + if (!node .IsByPassNode && (node.LambdaVisitor != null || node.Visitor != null )) { int parametersArrayLength = node.Children.Count + (context is NoContext ? 0 : 1); var parameters = new object[parametersArrayLength]; diff --git a/src/sly/parser/parser/llparser/bnf/RecursiveDescentSyntaxParser.Expressions.cs b/src/sly/parser/parser/llparser/bnf/RecursiveDescentSyntaxParser.Expressions.cs index 32d1dbde..13e2e391 100644 --- a/src/sly/parser/parser/llparser/bnf/RecursiveDescentSyntaxParser.Expressions.cs +++ b/src/sly/parser/parser/llparser/bnf/RecursiveDescentSyntaxParser.Expressions.cs @@ -27,9 +27,9 @@ protected SyntaxNode ManageExpressionRules(Rule rule, SyntaxNo case 2 when node.ExpressionAffix == Affix.PreFix: operatorIndex = 0; break; - case 2: + case 2 when node.ExpressionAffix == Affix.PostFix: { - if (node.ExpressionAffix == Affix.PostFix) operatorIndex = 1; + operatorIndex = 1; break; } } @@ -37,13 +37,14 @@ protected SyntaxNode ManageExpressionRules(Rule rule, SyntaxNo if (operatorIndex >= 0 && node.Children[operatorIndex] is SyntaxLeaf operatorNode) { var token = operatorNode.Token; - string key = node.ForcedName && node.Name != null ? node.Name : (token.IsExplicit ? $"'{token.Value}'" : token.TokenID.ToString()); + string key = (token.IsExplicit ? $"'{token.Value}'" : token.TokenID.ToString()); var operation = rule.GetOperation(key); if (operation != null) { node.Visitor = operation.VisitorMethod; node.LambdaVisitor = operation.LambdaVisitor; node.Operation = operation; + node.Name = operation.NodeName; } } break; @@ -55,7 +56,6 @@ protected SyntaxNode ManageExpressionRules(Rule rule, SyntaxNo break; } } - return node; } diff --git a/src/sly/parser/parser/llparser/ebnf/EBNFRecursiveDescentSyntaxParser.Expressions.cs b/src/sly/parser/parser/llparser/ebnf/EBNFRecursiveDescentSyntaxParser.Expressions.cs index acdf206b..6c14b426 100644 --- a/src/sly/parser/parser/llparser/ebnf/EBNFRecursiveDescentSyntaxParser.Expressions.cs +++ b/src/sly/parser/parser/llparser/ebnf/EBNFRecursiveDescentSyntaxParser.Expressions.cs @@ -1,6 +1,7 @@ using System.Collections.Generic; using sly.lexer; using sly.parser; +using sly.parser.generator; using sly.parser.syntax.grammar; using sly.parser.syntax.tree; @@ -14,6 +15,7 @@ public virtual SyntaxParseResult ParseInfixExpressionRule(IList parsingContext) { + OperationMetaData operation = null; var currentPosition = position; var children = new List>(); if (!tokens[position].IsEOS && rule.Match(tokens, position, Configuration) && rule.Clauses != null && @@ -34,6 +36,15 @@ public virtual SyntaxParseResult ParseInfixExpressionRule(IList secondResult = null; + + var op = tokens[position]; + string key = op.TokenID.ToString(); + if (op.IsExplicit) + { + key = op.Value.ToString(); + } + operation = rule.GetOperation(key); + switch (second) { case ChoiceClause secondChoice: @@ -90,7 +101,7 @@ public virtual SyntaxParseResult ParseInfixExpressionRule(IList(nonTerminalName, children); + var finalNode = new SyntaxNode(rule.NodeName ?? nonTerminalName, children); finalNode.ExpressionAffix = rule.ExpressionAffix; finalNode = ManageExpressionRules(rule, finalNode); var finalResult = new SyntaxParseResult(); diff --git a/src/sly/parser/parser/llparser/ebnf/EBNFRecursiveDescentSyntaxParser.cs b/src/sly/parser/parser/llparser/ebnf/EBNFRecursiveDescentSyntaxParser.cs index 9cecf411..8b5b95c8 100644 --- a/src/sly/parser/parser/llparser/ebnf/EBNFRecursiveDescentSyntaxParser.cs +++ b/src/sly/parser/parser/llparser/ebnf/EBNFRecursiveDescentSyntaxParser.cs @@ -162,7 +162,7 @@ public override SyntaxParseResult Parse(IList> tokens, Rule( nonTerminalName, children); - node.ForcedName = rule.ForcedName; + node.ForcedName = rule.ForcedName; node.Name = string.IsNullOrEmpty(rule.NodeName) ? nonTerminalName : rule.NodeName; node.ExpressionAffix = rule.ExpressionAffix; node = ManageExpressionRules(rule, node); diff --git a/src/sly/parser/syntax/grammar/Rule.cs b/src/sly/parser/syntax/grammar/Rule.cs index 12eeabf4..7e87f97a 100644 --- a/src/sly/parser/syntax/grammar/Rule.cs +++ b/src/sly/parser/syntax/grammar/Rule.cs @@ -109,7 +109,7 @@ public OperationMetaData GetOperation(string token) else { ; - + } } diff --git a/tests/ParserTests/ExpressionGeneratorTests.cs b/tests/ParserTests/ExpressionGeneratorTests.cs index 3b10dee0..72e7d251 100644 --- a/tests/ParserTests/ExpressionGeneratorTests.cs +++ b/tests/ParserTests/ExpressionGeneratorTests.cs @@ -1,5 +1,5 @@ -using System.Collections.Generic; -using expressionparser; +using System; +using System.Collections.Generic; using NFluent; using simpleExpressionParser; using sly.buildresult; @@ -233,9 +233,9 @@ public void TestBuild() nt = nonterminals[2]; Check.That(nt.Rules).CountIs(3); Check.That(nt.Name).Contains("primary_value"); - Check.That(nt.Rules[0].NodeName).IsEqualTo("double"); - Check.That(nt.Rules[1].NodeName).IsEqualTo("integer"); - Check.That(nt.Rules[2].NodeName).IsEqualTo("group"); + Check.That(nt.Rules[0].NodeName).IsEqualTo("prim_double"); + Check.That(nt.Rules[1].NodeName).IsEqualTo("prim_int"); + Check.That(nt.Rules[2].NodeName).IsEqualTo("prim_group"); nt = nonterminals[3]; Check.That(nt.Rules).CountIs(1); Check.That(nt.Name).Contains("PLUS"); @@ -274,9 +274,23 @@ public void TestFactorTimes() public void TestGroup() { BuildParser(); - var r = Parser.Result.Parse("(-1 + 2) * 3", StartingRule); + var r = Parser.Result.Parse("(-1 + 2) * 3!", StartingRule); Check.That(r).IsOkParsing();; - Check.That(r.Result).IsEqualTo(3.0); + Check.That(r.Result).IsEqualTo((-1 + 2 ) * 6.0); + + var d = r.SyntaxTree.Dump(" "); + Check.That(d).Contains("addition_or_substraction") + .And.Contains("multiplication_or_division") + .And.DoesNotContain("TIMES_DIVIDE"); + + GraphVizEBNFSyntaxTreeVisitor gvViz = + new GraphVizEBNFSyntaxTreeVisitor(); + gvViz.VisitTree(r.SyntaxTree); + string graph = gvViz.Graph.Compile(); + + Check.That(graph).Contains("addition_or_substraction") + .And.Contains("multiplication_or_division") + .And.DoesNotContain("TIMES_DIVIDE"); } [Fact] From dcacb3ff32ca46d792dc5187ce41190de6aa8809 Mon Sep 17 00:00:00 2001 From: b3b00 Date: Mon, 23 Dec 2024 14:14:20 +0100 Subject: [PATCH 5/7] many node names for same precedence level --- ...ecursiveDescentSyntaxParser.Expressions.cs | 5 +- ...ecursiveDescentSyntaxParser.Expressions.cs | 8 --- src/sly/parser/syntax/grammar/Rule.cs | 3 + tests/ParserTests/ExpressionGeneratorTests.cs | 65 +++++++++++++++++++ 4 files changed, 72 insertions(+), 9 deletions(-) diff --git a/src/sly/parser/parser/llparser/bnf/RecursiveDescentSyntaxParser.Expressions.cs b/src/sly/parser/parser/llparser/bnf/RecursiveDescentSyntaxParser.Expressions.cs index 13e2e391..bd480766 100644 --- a/src/sly/parser/parser/llparser/bnf/RecursiveDescentSyntaxParser.Expressions.cs +++ b/src/sly/parser/parser/llparser/bnf/RecursiveDescentSyntaxParser.Expressions.cs @@ -44,7 +44,10 @@ protected SyntaxNode ManageExpressionRules(Rule rule, SyntaxNo node.Visitor = operation.VisitorMethod; node.LambdaVisitor = operation.LambdaVisitor; node.Operation = operation; - node.Name = operation.NodeName; + if (!string.IsNullOrEmpty(operation.NodeName)) + { + node.Name = operation.NodeName; + } } } break; diff --git a/src/sly/parser/parser/llparser/ebnf/EBNFRecursiveDescentSyntaxParser.Expressions.cs b/src/sly/parser/parser/llparser/ebnf/EBNFRecursiveDescentSyntaxParser.Expressions.cs index 6c14b426..477fe726 100644 --- a/src/sly/parser/parser/llparser/ebnf/EBNFRecursiveDescentSyntaxParser.Expressions.cs +++ b/src/sly/parser/parser/llparser/ebnf/EBNFRecursiveDescentSyntaxParser.Expressions.cs @@ -37,14 +37,6 @@ public virtual SyntaxParseResult ParseInfixExpressionRule(IList secondResult = null; - var op = tokens[position]; - string key = op.TokenID.ToString(); - if (op.IsExplicit) - { - key = op.Value.ToString(); - } - operation = rule.GetOperation(key); - switch (second) { case ChoiceClause secondChoice: diff --git a/src/sly/parser/syntax/grammar/Rule.cs b/src/sly/parser/syntax/grammar/Rule.cs index 7e87f97a..2757993c 100644 --- a/src/sly/parser/syntax/grammar/Rule.cs +++ b/src/sly/parser/syntax/grammar/Rule.cs @@ -25,6 +25,8 @@ public Rule() public string[] SubNodeNames { get; set; } = null; + private Dictionary NodeNamesMap { get; set; } = new Dictionary(); + public bool IsByPassRule { get; set; } = false; // visitors for operation rules @@ -197,6 +199,7 @@ public void SetVisitor(Func visitorLambda) public void SetVisitor(OperationMetaData operation) { + NodeNamesMap[operation.Operatorkey] = operation.NodeName; if (operation.VisitorMethod != null) VisitorMethodsForOperation[operation.Operatorkey] = operation; else if (operation.LambdaVisitor != null) diff --git a/tests/ParserTests/ExpressionGeneratorTests.cs b/tests/ParserTests/ExpressionGeneratorTests.cs index 72e7d251..67a70a8b 100644 --- a/tests/ParserTests/ExpressionGeneratorTests.cs +++ b/tests/ParserTests/ExpressionGeneratorTests.cs @@ -9,6 +9,7 @@ using Xunit; using ParserTests.Issue184; using sly.parser.generator.visitor; +using sly.parser.syntax.tree; using ExpressionToken = expressionparser.ExpressionToken; namespace ParserTests @@ -26,6 +27,32 @@ public double BinaryTermExpression(double left, Token operation } } + + public class PrefixOperation + { + [Prefix("'##'",Associativity.Left,10)] + [NodeName("hash-hash")] + public string prefixHashHash(Token token, string operand) + { + return $"{token.Value}::{operand}"; + } + + [Prefix("'@@'",Associativity.Left,10)] + [NodeName("at-at")] + public string prefixAtAt(Token token, string operand) + { + return $"{token.Value}::{operand}"; + } + + + + [Operand] + [Production("testOperand: INT")] + public string Operand(Token token) + { + return token.Value; + } + } public class ExpressionGeneratorExplicitOperatorAsNames { @@ -48,6 +75,7 @@ public double BinaryTermExpression(double left, Token ope } [Prefix((int) SimpleExpressionToken.MINUS, Associativity.Right, 100)] + [NodeName("hash-hash")] public double PreFixExpression(Token operation, double value) { return -value; @@ -430,5 +458,42 @@ public void TestExplicitOperatorAsNames() Check.That(result).IsOkParsing(); Check.That(result.Result).IsEqualTo(-1+2*(5+6)-4); } + + [Fact] + public void TestPrefix() + { + StartingRule = $"{nameof(PrefixOperation)}_expressions"; + var parserInstance = new PrefixOperation(); + var builder = new ParserBuilder(); + var buildResult = builder.BuildParser(parserInstance, ParserType.EBNF_LL_RECURSIVE_DESCENT, StartingRule); + Check.That(buildResult).IsOk(); + var parser = buildResult.Result; + + var result = parser.Parse("## 42"); + Check.That(result).IsOkParsing(); + Check.That(result.Result).IsEqualTo("##::42"); + var tree = result.SyntaxTree; + var syntaxTree = tree as SyntaxNode; + Check.That(syntaxTree).IsNotNull(); + Check.That(syntaxTree.Children).CountIs(1); + Check.That(syntaxTree.Name).IsEqualTo($"{nameof(PrefixOperation)}_expressions"); + var child = syntaxTree.Children[0] as SyntaxNode; + Check.That(child).IsNotNull(); + Check.That(child.Name).IsEqualTo("hash-hash"); + Check.That(child.Children).CountIs(2); + + result = parser.Parse("@@ 42"); + Check.That(result).IsOkParsing(); + Check.That(result.Result).IsEqualTo("@@::42"); + tree = result.SyntaxTree; + syntaxTree = tree as SyntaxNode; + Check.That(syntaxTree).IsNotNull(); + Check.That(syntaxTree.Children).CountIs(1); + Check.That(syntaxTree.Name).IsEqualTo($"{nameof(PrefixOperation)}_expressions"); + child = syntaxTree.Children[0] as SyntaxNode; + Check.That(child).IsNotNull(); + Check.That(child.Name).IsEqualTo("at-at"); + Check.That(child.Children).CountIs(2); + } } } From f9aaac92e472e8d96450416a40efb79a8135bd10 Mon Sep 17 00:00:00 2001 From: b3b00 Date: Mon, 23 Dec 2024 16:00:41 +0100 Subject: [PATCH 6/7] fluent : sub node names --- .../parser/fluent/FluentEBNFParserBuilder.cs | 9 +++ .../parser/fluent/IFluentEbnfParserBuilder.cs | 2 + tests/ParserTests/ExpressionGeneratorTests.cs | 64 +++++++++++++++++++ tests/ParserTests/FluentTests.cs | 54 ++++++++++++++++ 4 files changed, 129 insertions(+) diff --git a/src/sly/parser/fluent/FluentEBNFParserBuilder.cs b/src/sly/parser/fluent/FluentEBNFParserBuilder.cs index 3ae858e2..ec0c5d5b 100644 --- a/src/sly/parser/fluent/FluentEBNFParserBuilder.cs +++ b/src/sly/parser/fluent/FluentEBNFParserBuilder.cs @@ -368,4 +368,13 @@ public IFluentEbnfParserBuilder Named(string name) } return this; } + + public IFluentEbnfParserBuilder WithSubNodeNamed(params string[] subNodeNames) + { + if (_currentRule != null) + { + _currentRule.SubNodeNames = subNodeNames; + } + return this; + } } \ No newline at end of file diff --git a/src/sly/parser/fluent/IFluentEbnfParserBuilder.cs b/src/sly/parser/fluent/IFluentEbnfParserBuilder.cs index e91b9452..fa73a60a 100644 --- a/src/sly/parser/fluent/IFluentEbnfParserBuilder.cs +++ b/src/sly/parser/fluent/IFluentEbnfParserBuilder.cs @@ -7,6 +7,8 @@ namespace sly.parser.generator; public interface IFluentEbnfRuleBuilder : IFluentEbnfParserBuilder where IN : struct { public IFluentEbnfParserBuilder Named(string name); + + public IFluentEbnfParserBuilder WithSubNodeNamed(params string[] subNodeNames); } public interface IFluentEbnfParserBuilder where IN : struct diff --git a/tests/ParserTests/ExpressionGeneratorTests.cs b/tests/ParserTests/ExpressionGeneratorTests.cs index 67a70a8b..f68da55c 100644 --- a/tests/ParserTests/ExpressionGeneratorTests.cs +++ b/tests/ParserTests/ExpressionGeneratorTests.cs @@ -8,6 +8,7 @@ using sly.parser.generator; using Xunit; using ParserTests.Issue184; +using sly.lexer.fluent; using sly.parser.generator.visitor; using sly.parser.syntax.tree; using ExpressionToken = expressionparser.ExpressionToken; @@ -469,6 +470,69 @@ public void TestPrefix() Check.That(buildResult).IsOk(); var parser = buildResult.Result; + var result = parser.Parse("## 42"); + Check.That(result).IsOkParsing(); + Check.That(result.Result).IsEqualTo("##::42"); + var tree = result.SyntaxTree; + var syntaxTree = tree as SyntaxNode; + Check.That(syntaxTree).IsNotNull(); + Check.That(syntaxTree.Children).CountIs(1); + Check.That(syntaxTree.Name).IsEqualTo($"{nameof(PrefixOperation)}_expressions"); + var child = syntaxTree.Children[0] as SyntaxNode; + Check.That(child).IsNotNull(); + Check.That(child.Name).IsEqualTo("hash-hash"); + Check.That(child.Children).CountIs(2); + + result = parser.Parse("@@ 42"); + Check.That(result).IsOkParsing(); + Check.That(result.Result).IsEqualTo("@@::42"); + tree = result.SyntaxTree; + syntaxTree = tree as SyntaxNode; + Check.That(syntaxTree).IsNotNull(); + Check.That(syntaxTree.Children).CountIs(1); + Check.That(syntaxTree.Name).IsEqualTo($"{nameof(PrefixOperation)}_expressions"); + child = syntaxTree.Children[0] as SyntaxNode; + Check.That(child).IsNotNull(); + Check.That(child.Name).IsEqualTo("at-at"); + Check.That(child.Children).CountIs(2); + } + + [Fact] + public void TestPrefixFluent() + { + StartingRule = $"{nameof(PrefixOperation)}_expressions"; + + var lexer = FluentLexerBuilder.NewBuilder() + .IgnoreEol(true) + .IgnoreWhiteSpace(true) + .IgnoreKeywordCase(true) + .Int(SimpleExpressionToken.INT); + + var build = FluentEBNFParserBuilder.NewBuilder(new PrefixOperation(), + $"{nameof(PrefixOperation)}_expressions", "en") + .Operand("testOperand: INT", (args => + { + return (args[0] as Token)?.Value; + })) + .Prefix("'##'", 10, args => + { + var token = args[0] as Token; + var operand = args[1] as string; + return $"{token?.Value}::{operand}"; + }).Named("hash-hash") + .Prefix("'@@'", 10, args => + { + var token = args[0] as Token; + var operand = args[1] as string; + return $"{token?.Value}::{operand}"; + }).Named("at-at") + .WithLexerbuilder(lexer) + .BuildParser(); + + Check.That(build).IsOk(); + var parser = build.Result; + + var result = parser.Parse("## 42"); Check.That(result).IsOkParsing(); Check.That(result.Result).IsEqualTo("##::42"); diff --git a/tests/ParserTests/FluentTests.cs b/tests/ParserTests/FluentTests.cs index f2db41c9..a152021f 100644 --- a/tests/ParserTests/FluentTests.cs +++ b/tests/ParserTests/FluentTests.cs @@ -7,6 +7,8 @@ using sly.lexer.fluent; using sly.lexer.fsm; using sly.parser.generator; +using sly.parser.parser; +using sly.parser.syntax.tree; using Xunit; namespace ParserTests; @@ -256,6 +258,8 @@ public void TestFluentParser() Check.That(r).IsOkParsing(); Check.That(r.Result).IsEqualTo(@"hello(hello, olivier) - world(world, 1977-03-30)"); } + + [Fact] public void TestFluentExpressionParser() @@ -396,4 +400,54 @@ public void TestFluentExpressionParserWithExplicits() Check.That(result.Result).IsEqualTo(-1 + 2); } + + [Fact] + public void TestFluentSubNodeNames() + { + var lexer = FluentLexerBuilder.NewBuilder() + .IgnoreEol(true) + .IgnoreWhiteSpace(true) + .IgnoreKeywordCase(true) + .AlphaNumDashId(FluentToken.ID) + .Date(FluentToken.DATE, DateFormat.YYYYMMDD, '-') + .AlphaNumDashId(FluentToken.ID); + + var build = FluentEBNFParserBuilder.NewBuilder(new FluentTests(), "root", "en") + .Production("root : (l 'SEP'[d] r)*", args => + { + var items = args[0] as List>; + var r = string.Join(",", items.Select(x => $"{x.Value(0)}|{x.Value(1)}")); + return r; + + }).WithSubNodeNamed("items") + .Production("l : ID", args => + { + return (args[0] as Token)?.Value; + }).Named("left") + .Production("r : ID", args => + { + return (args[0] as Token)?.Value; + }).Named("right") + .WithLexerbuilder(lexer) + .BuildParser(); + Check.That(build).IsOk(); + var parsed = build.Result.Parse("a SEP b c SEP d "); + Check.That(parsed).IsOkParsing(); + Check.That(parsed.Result).IsEqualTo("a|b,c|d"); + var tree = parsed.SyntaxTree as SyntaxNode; + Check.That(tree).IsNotNull(); + Check.That(tree.Children).IsSingle(); + var child = tree.Children.FirstOrDefault(); + Check.That(child).IsNotNull(); + Check.That(child.Name).IsEqualTo("items"); + var childNode = child as SyntaxNode; + Check.That(childNode).IsNotNull(); + Check.That(childNode.Children).CountIs(2); + var first = childNode.Children.FirstOrDefault() as SyntaxNode; + Check.That(first).IsNotNull(); + Check.That(first.Children).CountIs(3); + var l = first.Children.FirstOrDefault() as SyntaxNode; + Check.That(l).IsNotNull(); + Check.That(l.Name).IsEqualTo("left"); + } } \ No newline at end of file From a58cc843c92b0147fc4b47400a999815b0c4c021 Mon Sep 17 00:00:00 2001 From: b3b00 Date: Mon, 23 Dec 2024 16:34:42 +0100 Subject: [PATCH 7/7] fluent : test lexer post processing --- tests/ParserTests/FluentTests.cs | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/tests/ParserTests/FluentTests.cs b/tests/ParserTests/FluentTests.cs index a152021f..41549dfd 100644 --- a/tests/ParserTests/FluentTests.cs +++ b/tests/ParserTests/FluentTests.cs @@ -410,7 +410,19 @@ public void TestFluentSubNodeNames() .IgnoreKeywordCase(true) .AlphaNumDashId(FluentToken.ID) .Date(FluentToken.DATE, DateFormat.YYYYMMDD, '-') - .AlphaNumDashId(FluentToken.ID); + .AlphaNumDashId(FluentToken.ID) + .UseLexerPostProcessor(tokens => + { + return tokens.Select, Token>(x => + { + if (x.TokenID == FluentToken.ID) + { + x.SpanValue = x.Value.ToUpper().AsMemory(); + } + + return x; + }).ToList(); + }); var build = FluentEBNFParserBuilder.NewBuilder(new FluentTests(), "root", "en") .Production("root : (l 'SEP'[d] r)*", args => @@ -433,7 +445,7 @@ public void TestFluentSubNodeNames() Check.That(build).IsOk(); var parsed = build.Result.Parse("a SEP b c SEP d "); Check.That(parsed).IsOkParsing(); - Check.That(parsed.Result).IsEqualTo("a|b,c|d"); + Check.That(parsed.Result).IsEqualTo("A|B,C|D"); var tree = parsed.SyntaxTree as SyntaxNode; Check.That(tree).IsNotNull(); Check.That(tree.Children).IsSingle();