Bugfix/#487

src/samples/HandMadeExpressions/Properties/launchSettings.json

@@ -0,0 +1,11 @@
+{
+  "$schema": "http://json.schemastore.org/launchsettings.json",
+  "profiles": {
+    "BenchGenGen": {
+      "commandName": "DebugRoslynComponent",
+      "targetProject": "../BenchGen/BenchGen.csproj"
+    }
+  }
+}
+
+

src/samples/HandMadeExpressions/sourceGenerator/SyntaxExtensions.cs

@@ -0,0 +1,40 @@
+using Microsoft.CodeAnalysis;
+using Microsoft.CodeAnalysis.CSharp.Syntax;
+
+namespace handExpressions.sourceGenerator;
+
+public static class SyntaxExtensions
+{
+    public static string GetNameSpace(this SyntaxNode declarationSyntax)
+    {
+        if (declarationSyntax.Parent is FileScopedNamespaceDeclarationSyntax fileScopedNamespace)
+        {
+            return fileScopedNamespace.Name.ToString();
+        }
+        else if (declarationSyntax.Parent is NamespaceDeclarationSyntax namespaceDeclaration)
+        {
+            return namespaceDeclaration.Name.ToString();
+        }
+
+        return string.Empty;
+    }
+
+    public static CompilationUnitSyntax GetCompilationUnit(this SyntaxNode syntaxNode)
+    {
+        if (syntaxNode is CompilationUnitSyntax compilationUnitSyntax)
+        {
+            return compilationUnitSyntax;
+        }
+
+        if (syntaxNode.Parent != null)
+        {
+            return syntaxNode.Parent.GetCompilationUnit();
+        }
+
+        return null;
+    }
+
+
+
+
+}

src/samples/ParserExample/Issue487Token.cs

@@ -0,0 +1,118 @@
+using sly.lexer;
+
+namespace ParserExample;
+
+public enum Issue487Token
+{
+    [Lexeme(GenericToken.UpTo, ">>", "??")]
+    [Mode("xlmode", "dmode", "calcmode", "lvar")]
+    [Pop]
+    OPS = 103,
+
+#region default
+
+    [Sugar(">>")]
+    [Mode("default")]
+    INSTALL = 15,
+
+    // PUSHERS
+
+    [Keyword("CC")]
+    [Push("calcmode")]
+    [Mode("default")]
+    START_C = 102,
+
+    [Keyword("XL")]
+    [Push("xlmode")]
+    [Mode("default")]
+    START_X = 100,           // I think rightmost operand need smaller priority than left operand.  Or it'll be extracted last from the leftovers.  (Moved from 102 to 53 and the right side finally pulled all of the necessary characters)
+
+    [Sugar("@")]
+    [Push("dmode")]
+    [Mode("default")]
+    START_D = 101,
+
+    [Sugar("$")]
+    [Push("lvar")]
+    [Mode("default")]
+    VARSTART = 60,
+
+    [Keyword("Int")]
+    [Keyword("String")]
+    [Keyword("Optset")]
+    [Keyword("Money")]
+    [Keyword("Decimal")]
+    [Keyword("EtnRef")]
+    [Keyword("toconvert")]
+    [Push("convert")]
+    [Mode("default")]
+    CONVERT = 62,
+
+    // END PUSHERS
+
+    [Sugar("??")]
+    [Mode("default")]
+    IFNULLTHEN = 49,
+
+    [Sugar("{")]
+    [Mode("default")]
+    BLOCK_START = 50,
+
+    [Sugar("}")]
+    [Mode("default")]
+    BLOCK_END = 48,
+
+    [Sugar("(")]
+    [Mode("default")]
+    LPAREN = 40,
+
+    [Sugar(")")]
+    [Mode("default")]
+    RPAREN = 41,
+
+
+
+
+
+
+    [String("\"", "\\")]
+    CONST_BLOCK = 31,
+
+    [AlphaId]
+    LOCALVAR = 61,
+
+    EOF = 0,
+
+#endregion
+
+#region dmode
+
+
+#endregion    
+
+
+
+#region convert
+
+
+
+    [Sugar(":")]
+    [Mode("convert")]
+    [Pop]
+    COLON = 43,
+
+
+
+    [Sugar("?")]
+    [Mode("convert")]
+    ISNULLABLE = 105,
+
+
+#endregion
+
+
+
+
+
+
+}

src/samples/ParserExample/Program.cs

@@ -1204,7 +1204,8 @@ print a
         }
         private static void Main(string[] args)
         {
-            BenchSimpleExpression();
+            TestIssue487();
+            //BenchSimpleExpression();
             // IndentRefactoring();
             //NodeNames();
             // BroadWindow();
@@ -1625,6 +1626,26 @@ private static void TestLexerPostProcessEBNF()
             Console.WriteLine("x x = "+(res.HasValue ? res.Value.ToString() : "?"));
 
         }
+
+          private static void TestIssue487() 
+          {
+              string source = "@calc1 >> toconvert: @calc2";
+              var lexBuild = LexerBuilder.BuildLexer<Issue487Token>();
+              Check.That(lexBuild).IsOk();
+              var lexer = lexBuild.Result;
+              var lexed = lexer.Tokenize(source);
+              if (lexed.IsOk)
+              {
+                  foreach (var token in lexed.Tokens)
+                  {
+                      Console.WriteLine(token);
+                  }
+              }
+              else
+              {
+                  Console.Write(lexed.Error);
+              }
+          }
     }
 
     public enum TestGrammarToken

src/sly/lexer/GenericLexer.cs

@@ -166,7 +166,6 @@ public LexerResult<IN> Tokenize(ReadOnlyMemory<char> source)
             string src = source.ToString();
 
             var r = LexerFsm.Run(source, new LexerPosition());
-            LexerFsm = SetLexerMode(r, lexersStack);
 
             var ignored = r.IgnoredTokens.Select(x =>
                 new Token<IN>(default(IN), x.SpanValue, x.Position, x.CommentType, x.Channel)).ToList();
@@ -199,6 +198,7 @@ public LexerResult<IN> Tokenize(ReadOnlyMemory<char> source)
             {
                 ComputePositionWhenIgnoringEOL(r, tokens, LexerFsm);
                 var transcoded = Transcode(r);
+                LexerFsm = SetLexerMode(transcoded, lexersStack);    
                 if (CallBacks.TryGetValue(transcoded.TokenID, out var callback))
                 {
                     transcoded = callback(transcoded);
@@ -228,7 +228,6 @@ public LexerResult<IN> Tokenize(ReadOnlyMemory<char> source)
                 {
                     position = r.NewPosition;
                 }
-                LexerFsm = SetLexerMode(r, lexersStack);
 
                 ignored = r.IgnoredTokens.Select(x =>
                     new Token<IN>(default(IN), x.SpanValue, x.Position, 
@@ -284,28 +283,28 @@ public LexerResult<IN> Tokenize(ReadOnlyMemory<char> source)
             return new LexerResult<IN>(tokens);
         }
 
-        private FSMLexer<GenericToken> SetLexerMode(FSMMatch<GenericToken> r, Stack<FSMLexer<GenericToken>> lexersStack)
+        private FSMLexer<GenericToken> SetLexerMode(Token<IN> token, Stack<FSMLexer<GenericToken>> lexersStack)
         {
             FSMLexer<GenericToken> LexerFsm = lexersStack.Peek();
 
-            if (!r.IsEOS)
+            if (!token.IsEOS)
             {
-                if (r.IsPop)
+                if (token.Position.IsPop)
                 {
                     lexersStack.Pop();
                     LexerFsm = lexersStack.Peek();
-                    r.NewPosition.Mode = LexerFsm.Mode;
                     return LexerFsm;
                 }
+
 
-                if (r.IsPush)
+                if (token.Position.IsPush && !string.IsNullOrEmpty(token.Position.Mode))
                 {
-                    LexerFsm = SubLexersFsm[r.NewPosition.Mode];
+                    LexerFsm = SubLexersFsm[token.Position.Mode ?? ModeAttribute.DefaultLexerMode];
                     lexersStack.Push(LexerFsm);
                 }
                 else
                 {
-                    LexerFsm = SubLexersFsm[r.NewPosition.Mode];
+                    LexerFsm = SubLexersFsm[token.Position.Mode ?? ModeAttribute.DefaultLexerMode];
                 }
             }
 
@@ -1223,11 +1222,22 @@ public Token<IN> Transcode(FSMMatch<GenericToken> match)
         {
             var tok = new Token<IN>();
             var inTok = match.Result;
+            var newMode = match.NewPosition.Mode;
+            if (inTok != null
+                && derivedTokens.TryGetValue(inTok.TokenID, out var derivations)
+                && derivations.TryGetValue(inTok.Value, out var derivation))
+            {
+                newMode = derivation.mode;
+            }
+
             tok.IsComment = inTok.IsComment;
             tok.IsEmpty = inTok.IsEmpty;
             tok.SpanValue = inTok.SpanValue;
             tok.CommentType = inTok.CommentType;
             tok.Position = inTok.Position;
+            tok.Position.IsPop = match.NewPosition.IsPop;
+            tok.Position.IsPush = match.NewPosition.IsPush;
+            tok.Position.Mode = newMode;
             tok.Discarded = inTok.Discarded;
             tok.StringDelimiter = match.StringDelimiterChar;
             tok.HexaPrefix = match.HexaPrefix;

src/sly/lexer/LexerPosition.cs

@@ -7,7 +7,7 @@
 namespace sly.lexer
 
 {
     public class LexerPosition : IComparable
     {
 
         public LexerPosition() : this (0,0,0)
@@ -37,9 +37,13 @@
         public int Column { get; set; }
         public int Index { get; set; }
         public int Line { get; set; }
-        
+
         public string Mode { get; set; }
 
+        public bool IsPop { get; set; }
+
+        public bool IsPush { get; set; }
+
         public override string ToString()
         {
             return $"line {Line}, column {Column}";
@@ -67,7 +71,9 @@
             return new LexerPosition(Index, Line, Column, CurrentIndentation)
             {
                 Indentation = this.Indentation.Clone(),
-                Mode = Mode
+                Mode = Mode,
+                IsPop = IsPop,
+                IsPush = IsPush,
             };
         }
 

src/sly/lexer/fsm/FSMLexer.cs

@@ -296,9 +296,7 @@ public FSMMatch<N> Run(ReadOnlyMemory<char> source, LexerPosition lexerPosition)
                 // Failed on last character, so need to backtrack
                 lexerPosition.Index -= 1;
                 lexerPosition.Column -= 1;
-
             }
-
             var errorChar = source.Slice(lexerPosition.Index, 1);
             var ko = new FSMMatch<N>(false, default(N), errorChar, lexerPosition, -1, lexerPosition, false);
             return ko;

src/sly/lexer/fsm/FSMMatch.cs

@@ -103,7 +103,9 @@ public FSMMatch(bool success, N result, ReadOnlyMemory<char> value, LexerPositio
             }
             IsLineEnding = isLineEnding;
             IsPop = isPop;
+            NewPosition.IsPop = isPop;
             IsPush = isPush;
+            NewPosition.IsPush = isPush;
             IgnoredTokens = new List<Token<N>>();
         }