From 1978d75fdc56563022ba25ae245fac2e9f755ce8 Mon Sep 17 00:00:00 2001 From: b3b00 Date: Fri, 1 Nov 2024 10:43:27 +0200 Subject: [PATCH 1/6] issue #495 unit test --- tests/ParserTests/Issue495/Issue495Parser.cs | 33 +++++++++++++++++++ tests/ParserTests/Issue495/Issue495Tests.cs | 34 ++++++++++++++++++++ tests/ParserTests/Issue495/Issue495Token.cs | 23 +++++++++++++ 3 files changed, 90 insertions(+) create mode 100644 tests/ParserTests/Issue495/Issue495Parser.cs create mode 100644 tests/ParserTests/Issue495/Issue495Tests.cs create mode 100644 tests/ParserTests/Issue495/Issue495Token.cs diff --git a/tests/ParserTests/Issue495/Issue495Parser.cs b/tests/ParserTests/Issue495/Issue495Parser.cs new file mode 100644 index 00000000..7d7a1900 --- /dev/null +++ b/tests/ParserTests/Issue495/Issue495Parser.cs @@ -0,0 +1,33 @@ +using System.Collections.Generic; +using sly.lexer; +using sly.parser.generator; + +namespace ParserTests.Issue495; + +public class Issue495Parser +{ + + public Issue495Parser() + { + + } + + [Production("STRING: StartQuote StringValue* EndQuote")] + public object stringValue(Token open, Token value, Token close) + { + return null; + } + + [Production("statement : Identifier Assign STRING End")] + public object Statement(Token id, Token assign, object value, + Token end) + { + return null; + } + + [Production("program: statement*")] + public object Program(List statements) + { + return null; + } +} \ No newline at end of file diff --git a/tests/ParserTests/Issue495/Issue495Tests.cs b/tests/ParserTests/Issue495/Issue495Tests.cs new file mode 100644 index 00000000..82a5f100 --- /dev/null +++ b/tests/ParserTests/Issue495/Issue495Tests.cs @@ -0,0 +1,34 @@ +using NFluent; +using sly.parser; +using sly.parser.generator; +using Xunit; + +namespace ParserTests.Issue495; + +public class Issue495Tests +{ + public Parser _parser { get; set; } + + public Parser GetParser() + { + if (_parser == null) + { + ParserBuilder builder = new ParserBuilder("en"); + var build = builder.BuildParser(new Issue495Parser(), ParserType.EBNF_LL_RECURSIVE_DESCENT, "program"); + Check.That(build).IsOk(); + _parser = build.Result; + } + + return _parser; + } + + [Fact] + public void TestIssue495() + { + var parser = GetParser(); + Check.That(parser).IsNotNull(); + var parsed = parser.Parse("test = \"3 3\""); + Check.That(parsed).IsOkParsing(); + } + +} \ No newline at end of file diff --git a/tests/ParserTests/Issue495/Issue495Token.cs b/tests/ParserTests/Issue495/Issue495Token.cs new file mode 100644 index 00000000..647cef2e --- /dev/null +++ b/tests/ParserTests/Issue495/Issue495Token.cs @@ -0,0 +1,23 @@ +using sly.lexer; + +namespace ParserTests.Issue495; + +public enum Issue495Token +{ + [Sugar("\"")] + [Push("defaultString")] + StartQuote, + [Sugar("\"")] + [Mode("defaultString")] + [Pop] + EndQuote, + [UpTo("\"")] + [Mode("defaultString")] + StringValue, + [Sugar(";")] + End, + [AlphaNumDashId] + Identifier, + [Sugar("=")] + Assign +} \ No newline at end of file From 1e7818ebdb976adbb7c6cb7a651bddc1bdbfb95a Mon Sep 17 00:00:00 2001 From: b3b00 Date: Fri, 1 Nov 2024 10:55:52 +0200 Subject: [PATCH 2/6] . --- tests/ParserTests/Issue495/Issue495Parser.cs | 2 +- tests/ParserTests/Issue495/Issue495Tests.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/ParserTests/Issue495/Issue495Parser.cs b/tests/ParserTests/Issue495/Issue495Parser.cs index 7d7a1900..ed267f88 100644 --- a/tests/ParserTests/Issue495/Issue495Parser.cs +++ b/tests/ParserTests/Issue495/Issue495Parser.cs @@ -13,7 +13,7 @@ public Issue495Parser() } [Production("STRING: StartQuote StringValue* EndQuote")] - public object stringValue(Token open, Token value, Token close) + public object stringValue(Token open, List> values, Token close) { return null; } diff --git a/tests/ParserTests/Issue495/Issue495Tests.cs b/tests/ParserTests/Issue495/Issue495Tests.cs index 82a5f100..3f234c72 100644 --- a/tests/ParserTests/Issue495/Issue495Tests.cs +++ b/tests/ParserTests/Issue495/Issue495Tests.cs @@ -27,7 +27,7 @@ public void TestIssue495() { var parser = GetParser(); Check.That(parser).IsNotNull(); - var parsed = parser.Parse("test = \"3 3\""); + var parsed = parser.Parse("test = \"3 3\";"); Check.That(parsed).IsOkParsing(); } From 9b0b3e4216dc8ecd9fd98922b981599edfec717f Mon Sep 17 00:00:00 2001 From: b3b00 Date: Fri, 1 Nov 2024 11:01:38 +0200 Subject: [PATCH 3/6] unit test #495 --- tests/ParserTests/Issue495/Issue495Parser.cs | 13 +++++++------ tests/ParserTests/Issue495/Issue495Tests.cs | 7 ++++--- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/tests/ParserTests/Issue495/Issue495Parser.cs b/tests/ParserTests/Issue495/Issue495Parser.cs index ed267f88..5c5e3acc 100644 --- a/tests/ParserTests/Issue495/Issue495Parser.cs +++ b/tests/ParserTests/Issue495/Issue495Parser.cs @@ -1,4 +1,5 @@ using System.Collections.Generic; +using System.Linq; using sly.lexer; using sly.parser.generator; @@ -13,21 +14,21 @@ public Issue495Parser() } [Production("STRING: StartQuote StringValue* EndQuote")] - public object stringValue(Token open, List> values, Token close) + public string stringValue(Token open, List> values, Token close) { - return null; + return string.Join(", ", values.Select(x => x.Value.ToString())); } [Production("statement : Identifier Assign STRING End")] - public object Statement(Token id, Token assign, object value, + public string Statement(Token id, Token assign, string value, Token end) { - return null; + return $"{id.Value}{assign.Value}{value}"; } [Production("program: statement*")] - public object Program(List statements) + public string Program(List statements) { - return null; + return string.Join("\n", statements); } } \ No newline at end of file diff --git a/tests/ParserTests/Issue495/Issue495Tests.cs b/tests/ParserTests/Issue495/Issue495Tests.cs index 3f234c72..3b4ab789 100644 --- a/tests/ParserTests/Issue495/Issue495Tests.cs +++ b/tests/ParserTests/Issue495/Issue495Tests.cs @@ -7,13 +7,13 @@ namespace ParserTests.Issue495; public class Issue495Tests { - public Parser _parser { get; set; } + public Parser _parser { get; set; } - public Parser GetParser() + public Parser GetParser() { if (_parser == null) { - ParserBuilder builder = new ParserBuilder("en"); + ParserBuilder builder = new ParserBuilder("en"); var build = builder.BuildParser(new Issue495Parser(), ParserType.EBNF_LL_RECURSIVE_DESCENT, "program"); Check.That(build).IsOk(); _parser = build.Result; @@ -29,6 +29,7 @@ public void TestIssue495() Check.That(parser).IsNotNull(); var parsed = parser.Parse("test = \"3 3\";"); Check.That(parsed).IsOkParsing(); + Check.That(parsed.Result).IsEqualTo("test=3 3"); } } \ No newline at end of file From f2ef0191f25872f6a9240d7ff47303a5b93440c6 Mon Sep 17 00:00:00 2001 From: bd k Date: Fri, 1 Nov 2024 19:47:12 +0800 Subject: [PATCH 4/6] Make issue reproducable --- tests/ParserTests/Issue495/Issue495Parser.cs | 1 + tests/ParserTests/Issue495/Issue495Token.cs | 142 ++++++++++++++++++- 2 files changed, 137 insertions(+), 6 deletions(-) diff --git a/tests/ParserTests/Issue495/Issue495Parser.cs b/tests/ParserTests/Issue495/Issue495Parser.cs index 5c5e3acc..ead051a4 100644 --- a/tests/ParserTests/Issue495/Issue495Parser.cs +++ b/tests/ParserTests/Issue495/Issue495Parser.cs @@ -20,6 +20,7 @@ public string stringValue(Token open, List> } [Production("statement : Identifier Assign STRING End")] + [Production("statement : '$'[d] Identifier Assign STRING End")] public string Statement(Token id, Token assign, string value, Token end) { diff --git a/tests/ParserTests/Issue495/Issue495Token.cs b/tests/ParserTests/Issue495/Issue495Token.cs index 647cef2e..fdc3bb50 100644 --- a/tests/ParserTests/Issue495/Issue495Token.cs +++ b/tests/ParserTests/Issue495/Issue495Token.cs @@ -4,6 +4,91 @@ namespace ParserTests.Issue495; public enum Issue495Token { + EOF = 0, + + #region Sugar + [Comment("//", "/*", "*/")] + Comment, + [Sugar("++")] + Increment, + [Sugar("--")] + Decrement, + [Sugar("+")] + Plus, + [Sugar("-")] + Minus, + [Sugar("*")] + Multiply, + [Sugar("/")] + Divide, + [Sugar("%")] + Remainder, + [Sugar(">")] + GreaterThan, + [Sugar("<")] + LessThan, + [Sugar("==")] + Equal, + [Sugar("!=")] + NotEqual, + [Sugar(">=")] + GreaterThanOrEqual, + [Sugar("<=")] + LessThanOrEqual, + [Sugar("=")] + Assign, + [Sugar("?=")] + BooleanAssign, + [Sugar("%=")] + RemainderAssign, + [Sugar("+=")] + PlusAssign, + [Sugar("-=")] + MinusAssign, + [Sugar("*=")] + MultiplyAssign, + [Sugar("/=")] + DivideAssign, + [Sugar("><")] + CompareAssign, + [Sugar("??=")] + NullColesleAssign, + [Sugar("=>")] + Arrow, + [Sugar("{")] + BlockStart, + [Sugar("}")] + BlockEnd, + [Sugar("[")] + ListStart, + [Sugar("]")] + ListEnd, + [Sugar("(")] + ParenStart, + [Sugar(")")] + ParenEnd, + [Sugar(":")] + Colon, + [Sugar("::")] + Imply, + [Sugar(",")] + Comma, + [Sugar(";")] + End, + [Sugar("!")] + Not, + [Sugar("$")] + DollarSign, + [Sugar(".")] + Dot, + [Sugar("&")] + Deref, + [Sugar("||")] + Or, + [Sugar("&&")] + And, + [Sugar("\\")] + Escape, [Sugar("\"")] [Push("defaultString")] StartQuote, @@ -11,13 +96,58 @@ public enum Issue495Token [Mode("defaultString")] [Pop] EndQuote, + #endregion + + #region Keywords + [Keyword("true")] + True, + [Keyword("false")] + False, + [Keyword("run")] + RunKeyword, + [Keyword("import")] + ImportKeyword, + [Keyword("function")] + FunctionKeyword, + [Keyword("while")] + WhileKeyword, + [Keyword("if")] + IfKeyword, + [Keyword("else")] + ElseKeyword, + [Keyword("do")] + DoKeyword, + [Keyword("class")] + ClassKeyword, + [Keyword("new")] + NewKeyword, + [Keyword("null")] + NullKeyword, + [Keyword("command")] + CommandKeyword, + [Sugar("@s")] + SelectorSelf, + [Sugar("@p")] + SelectorNearest, + [Sugar("@a")] + SelectorAllPlayers, + [Sugar("@r")] + SelectorRandomPlayer, + [Sugar("@e")] + SelectorAllEntities, + #endregion + + [Int] + Int, + [Double] + Double, + [AlphaNumDashId] + Identifier, + [UpTo("\"")] [Mode("defaultString")] StringValue, - [Sugar(";")] - End, - [AlphaNumDashId] - Identifier, - [Sugar("=")] - Assign + + Variable, + Namespace, } \ No newline at end of file From 2670194fe03324599eb80c713365ef9668af68c8 Mon Sep 17 00:00:00 2001 From: b3b00 Date: Sat, 2 Nov 2024 10:09:27 +0200 Subject: [PATCH 5/6] #495 : debugging --- src/samples/ParserExample/Program.cs | 33 ++++++++++++++++++++- tests/ParserTests/Issue495/Issue495Tests.cs | 16 +++++++++- tests/ParserTests/Issue495/Issue495Token.cs | 8 +++-- 3 files changed, 52 insertions(+), 5 deletions(-) diff --git a/src/samples/ParserExample/Program.cs b/src/samples/ParserExample/Program.cs index 1507b404..6ad65431 100644 --- a/src/samples/ParserExample/Program.cs +++ b/src/samples/ParserExample/Program.cs @@ -22,6 +22,7 @@ using ParserTests.Issue239; using ParserTests.Issue332; using ParserTests.Issue414; +using ParserTests.Issue495; using ParserTests.lexer; using simpleExpressionParser; using SimpleTemplate; @@ -1351,6 +1352,7 @@ print a } private static void Main(string[] args) { + TestIssue495(); //testGenericLexerJson(); // TestIssue487(); //BenchSimpleExpression(); @@ -1372,7 +1374,7 @@ private static void Main(string[] args) //testJSONEscaped(); //testJSONNotEscaped(); // testProfileJSONEscaping(escape:true); - testProfileJSONEscaping(escape:false); + //testProfileJSONEscaping(escape:false); //TestGrammarParser(); // TestGraphViz(); // TestGraphViz(); @@ -1803,6 +1805,35 @@ private static void TestIssue487() Console.Write(lexed.Error); } } + + private static void TestIssue495() + { + Parser parser; + + ParserBuilder builder = new ParserBuilder("en"); + var build = builder.BuildParser(new Issue495Parser(), ParserType.EBNF_LL_RECURSIVE_DESCENT, "program"); + Check.That(build).IsOk(); + parser = build.Result; + + Check.That(parser).IsNotNull(); + Check.That(parser.Lexer).IsNotNull(); + Check.That(parser.Lexer).IsInstanceOf>(); + var lexer = parser.Lexer as GenericLexer; + string source = "test = \"3 3\";"; + var tokenized = lexer.Tokenize(source); + Check.That(tokenized).IsOkLexing(); + var tokens = tokenized.Tokens.MainTokens(); + Check.That(tokens).CountIs(7); + var stringValue = tokens[3]; + Check.That(stringValue).IsNotNull(); + Check.That(stringValue.TokenID).IsEqualTo(Issue495Token.StringValue); + + + var parsed = parser.Parse(source); + Check.That(parsed).IsOkParsing(); + Check.That(parsed.Result).IsEqualTo("test=3 3"); + + } } public enum TestGrammarToken diff --git a/tests/ParserTests/Issue495/Issue495Tests.cs b/tests/ParserTests/Issue495/Issue495Tests.cs index 3b4ab789..cd37222e 100644 --- a/tests/ParserTests/Issue495/Issue495Tests.cs +++ b/tests/ParserTests/Issue495/Issue495Tests.cs @@ -1,4 +1,5 @@ using NFluent; +using sly.lexer; using sly.parser; using sly.parser.generator; using Xunit; @@ -27,7 +28,20 @@ public void TestIssue495() { var parser = GetParser(); Check.That(parser).IsNotNull(); - var parsed = parser.Parse("test = \"3 3\";"); + Check.That(parser.Lexer).IsNotNull(); + Check.That(parser.Lexer).IsInstanceOf>(); + var lexer = parser.Lexer as GenericLexer; + string source = "test = \"3 3\";"; + var tokenized = lexer.Tokenize(source); + Check.That(tokenized).IsOkLexing(); + var tokens = tokenized.Tokens.MainTokens(); + Check.That(tokens).CountIs(7); + var stringValue = tokens[3]; + Check.That(stringValue).IsNotNull(); + Check.That(stringValue.TokenID).IsEqualTo(Issue495Token.StringValue); + + + var parsed = parser.Parse(source); Check.That(parsed).IsOkParsing(); Check.That(parsed.Result).IsEqualTo("test=3 3"); } diff --git a/tests/ParserTests/Issue495/Issue495Token.cs b/tests/ParserTests/Issue495/Issue495Token.cs index fdc3bb50..66044c87 100644 --- a/tests/ParserTests/Issue495/Issue495Token.cs +++ b/tests/ParserTests/Issue495/Issue495Token.cs @@ -96,6 +96,10 @@ public enum Issue495Token [Mode("defaultString")] [Pop] EndQuote, + + [UpTo("\"")] + [Mode("defaultString")] + StringValue, #endregion #region Keywords @@ -144,9 +148,7 @@ public enum Issue495Token [AlphaNumDashId] Identifier, - [UpTo("\"")] - [Mode("defaultString")] - StringValue, + Variable, Namespace, From 60f7a7531328b396d681c7fa188d21740d64da86 Mon Sep 17 00:00:00 2001 From: b3b00 Date: Sun, 3 Nov 2024 19:15:28 +0200 Subject: [PATCH 6/6] fix #495 --- src/sly/lexer/LexerBuilder.cs | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/sly/lexer/LexerBuilder.cs b/src/sly/lexer/LexerBuilder.cs index d6946279..7c4514f1 100644 --- a/src/sly/lexer/LexerBuilder.cs +++ b/src/sly/lexer/LexerBuilder.cs @@ -446,8 +446,17 @@ private static BuildResult> BuildGenericSubLexers( var subLexers = GetSubLexers(attributes); foreach (var subLexer in subLexers) { - var x = BuildGenericLexer(subLexer.Value, extensionBuilder, result, lang, explicitTokens); - var currentGenericLexer = x.Result as GenericLexer; + BuildResult> b = null; + if (subLexer.Key == ModeAttribute.DefaultLexerMode) + { + b = BuildGenericLexer(subLexer.Value, extensionBuilder, result, lang, explicitTokens); + } + else + { + b = BuildGenericLexer(subLexer.Value, extensionBuilder, result, lang, null); + } + + var currentGenericLexer = b.Result as GenericLexer; if (genLexer == null) { genLexer = currentGenericLexer;