Merge pull request #486 from b3b00/bugfix/#485

fix #485

src/sly/lexer/CallBacksBuilder.cs

@@ -11,8 +11,11 @@ public static void BuildCallbacks<IN>(GenericLexer<IN> lexer) where IN : struct
         {
             var attributes =
                 (CallBacksAttribute[]) typeof(IN).GetCustomAttributes(typeof(CallBacksAttribute), true);
-            Type callbackClass = attributes[0].CallBacksClass;
-            ExtractCallBacks<IN>(callbackClass,lexer);
+            if (attributes.Length > 0)
+            {
+                Type callbackClass = attributes[0].CallBacksClass;
+                ExtractCallBacks<IN>(callbackClass, lexer);
+            }
 
         }
 

src/sly/lexer/GenericLexer.cs

@@ -794,15 +794,31 @@ public ReadOnlyMemory<char> diffCharEscaper(char escapeStringDelimiterChar, char
             var value = stringValue;
             string newValue = "";
             int i = 0;
+            bool escaping = false;
             while (i < value.Length)
             {
                 char current = value.At(i);
-                if (current == escapeStringDelimiterChar)
+                if (current == escapeStringDelimiterChar && !escaping)
+                {
+                    escaping = true;
+                }
+                else if (escaping)
+                {
+                    if (escaping)
+                    {
+                        if (current != stringDelimiterChar)
+                        {
+                            newValue += escapeStringDelimiterChar;
+                        }
+                    }
+                    escaping = false;
+                }
+
+                if (!escaping)
                 {
-                    i++;
+                    newValue += value.At(i);
                 }
 
-                newValue += value.At(i);
                 i++;
 
             }

src/sly/parser/generator/ParserBuilder.cs

@@ -178,7 +178,10 @@ protected virtual BuildResult<ILexer<IN>> BuildLexer(
         {
             var lexer = LexerBuilder.BuildLexer<IN>(new BuildResult<ILexer<IN>>(), extensionBuilder, I18N,
                 lexerPostProcess, explicitTokens);
-
+            if (lexer.IsOk && lexer.Result != null && lexer.Result is GenericLexer<IN> genericLexer)
+            {
+                CallBacksBuilder.BuildCallbacks(genericLexer);
+            }
             return lexer;
         }
 

tests/ParserTests/Issue485/Issue485Lexer.cs

@@ -0,0 +1,10 @@
+using sly.lexer;
+
+namespace ParserTests.Issue485;
+
+public enum Issue485Lexer
+{
+    [Sugar(":")] COLON,
+    [Keyword("Property")] PROPERTY, 
+    [String] STRING
+}

tests/ParserTests/Issue485/Issue485Parser.cs

@@ -0,0 +1,14 @@
+using sly.lexer;
+using sly.parser.generator;
+
+namespace ParserTests.Issue485;
+
+[ParserRoot("root")]
+public class Issue485Parser
+{
+    [Production("root : PROPERTY[d] COLON[d] STRING")]
+    public string Parse(Token<Issue485Lexer> str)
+    {
+        return str.StringWithoutQuotes;
+    }
+}

tests/ParserTests/Issue485/Issue485SelfEscapeLexer.cs

@@ -0,0 +1,10 @@
+using sly.lexer;
+
+namespace ParserTests.Issue485;
+
+public enum Issue485SelfEscapeLexer
+{
+    [Sugar(":")] COLON,
+    [Keyword("Property")] PROPERTY, 
+    [String("\"","\"")] STRING
+}

tests/ParserTests/Issue485/Issue485SelfEscapeParser.cs

@@ -0,0 +1,14 @@
+using sly.lexer;
+using sly.parser.generator;
+
+namespace ParserTests.Issue485;
+
+[ParserRoot("root")]
+public class Issue485SelfEscapeParser
+{
+    [Production("root : PROPERTY[d] COLON[d] STRING")]
+    public string Parse(Token<Issue485SelfEscapeLexer> str)
+    {
+        return str.StringWithoutQuotes;
+    }
+}

tests/ParserTests/Issue485/Issue485Tests.cs

@@ -0,0 +1,55 @@
+using NFluent;
+using sly.parser.generator;
+using Xunit;
+
+namespace ParserTests.Issue485;
+
+public class Issue485Tests
+{
+    [Fact]
+    public void TestIssue485()
+    {
+        var builder = new ParserBuilder<Issue485Lexer, string>();
+        var build = builder.BuildParser(new Issue485Parser(), ParserType.EBNF_LL_RECURSIVE_DESCENT,"root");
+        Check.That(build).IsOk();
+        var parser = build.Result;
+        Check.That(parser).IsNotNull();
+        var parsed = parser.Parse("Property: \"Hello \\\"There\\\"\\nSecond line\"");
+        Check.That(parsed).IsOkParsing();
+        var result = parsed.Result;
+        Check.That(result).IsNotNull();
+        Check.That(result).IsNotEmpty();
+        Check.That(result).Equals("Hello \"There\"\\nSecond line");
+    }
+    [Fact]
+    public void TestIssue485SelfEscape()
+    {
+        var builder = new ParserBuilder<Issue485SelfEscapeLexer, string>();
+        var build = builder.BuildParser(new Issue485SelfEscapeParser(), ParserType.EBNF_LL_RECURSIVE_DESCENT,"root");
+        Check.That(build).IsOk();
+        var parser = build.Result;
+        Check.That(parser).IsNotNull();
+        var parsed = parser.Parse("Property: \"Hello \"\"There\"\"\\nSecond line\"");
+        Check.That(parsed).IsOkParsing();
+        var result = parsed.Result;
+        Check.That(result).IsNotNull();
+        Check.That(result).IsNotEmpty();
+        Check.That(result).Equals("Hello \"There\"\\nSecond line");
+    }
+
+    [Fact]
+    public void TestIssue485WithCallback()
+    {
+        var builder = new ParserBuilder<Issue485WithCallbackLexer, string>();
+        var build = builder.BuildParser(new Issue485WithCallbackParser(), ParserType.EBNF_LL_RECURSIVE_DESCENT,"root");
+        Check.That(build).IsOk();
+        var parser = build.Result;
+        Check.That(parser).IsNotNull();
+        var parsed = parser.Parse("Property: \"Hello \\\"There\\\"\\nSecond line\"");
+        Check.That(parsed).IsOkParsing();
+        var result = parsed.Result;
+        Check.That(result).IsNotNull();
+        Check.That(result).IsNotEmpty();
+        Check.That(result).Equals("Hello \"There\"\nSecond line");
+    }
+}

tests/ParserTests/Issue485/Issue485WithCallbackLexer.cs

@@ -0,0 +1,29 @@
+using System;
+using sly.lexer;
+
+namespace ParserTests.Issue485;
+
+[CallBacks(typeof(Issue485Callbacks))]
+public enum Issue485WithCallbackLexer
+{
+    [Sugar(":")] COLON,
+    [Keyword("Property")] PROPERTY, 
+    [String] STRING
+}
+
+public class Issue485Callbacks
+{
+
+    [TokenCallback((int)Issue485WithCallbackLexer.STRING)]
+    public static Token<Issue485WithCallbackLexer> EscapeEscapedSequences(Token<Issue485WithCallbackLexer> token)
+    {
+        string value = token.Value;
+
+        value = value.Replace("\\n", "\n");
+        value = value.Replace("\\r", "\r");
+        value = value.Replace("\\t", "\t");
+        value = value.Replace("\\\\", "\\"); 
+        token.SpanValue = new ReadOnlyMemory<char>(value.ToCharArray());
+        return token;
+    }
+}

tests/ParserTests/Issue485/Issue485WithCallbackParser.cs

@@ -0,0 +1,14 @@
+using sly.lexer;
+using sly.parser.generator;
+
+namespace ParserTests.Issue485;
+
+[ParserRoot("root")]
+public class Issue485WithCallbackParser
+{
+    [Production("root : PROPERTY[d] COLON[d] STRING")]
+    public string Parse(Token<Issue485WithCallbackLexer> str)
+    {
+        return str.StringWithoutQuotes;
+    }
+}

tests/ParserTests/lexer/GenericLexerTests.cs

@@ -591,8 +591,9 @@ public void TestIssue348()
             Check.That(r.Tokens).CountIs(2);
             var tok = r.Tokens[0];
             Check.That(tok.TokenID).IsEqualTo(Issue348.STriNG);
-            Check.That(tok.Value).IsEqualTo("\"\"te\\\\st\"\"");
-            Check.That(tok.StringWithoutQuotes).IsEqualTo("\"te\\\\st\"");
+            Check.That(tok.StringWithoutQuotes).IsEqualTo("\"te\\\\\\\\st\"");
+            Check.That(tok.Value).IsEqualTo("\"\"te\\\\\\\\st\"\"");
+
         }
 
         [Fact]
@@ -809,6 +810,7 @@ public void TestTokenCallbacks()
             Check.That(tokens[0]).IsEqualTo(CallbackTokens.IDENTIFIER, "AAA");
             Check.That(tokens[1]).IsEqualTo(CallbackTokens.SKIP, "BBB");
         }
+
 
         [Fact]
         public void TestCharTokens()
@@ -1336,5 +1338,6 @@ public void TestI18nDuplicateLabel()
             Check.That(error.Message).Contains("left paranthesis").And.Contains("paranthèse ouvrante");
         }
 
+
     }
 }