Lexer execution tests OK

funkwerk · Mar 23, 2020 · 6e46840 · 6e46840
1 parent 6d2a006
commit 6e46840
Show file tree

Hide file tree

Showing 4 changed files with 21 additions and 16 deletions.
diff --git a/Makefile b/Makefile
@@ -1,7 +1,7 @@
 # Make for antlr-d
 
 EXPORT = /usr/local
-DMD_EXE = ldc2
+DMD_EXE = ldc2 -g
 
 UNAME_M := $(shell uname -m)
 
@@ -24,7 +24,7 @@ ifeq ($(UNAME_M),armv7l)
     TESTRUNNER_OPT =
 endif
 
-DMD = $(DMD_EXE) -link-defaultlib-shared -w -O
+DMD = $(DMD_EXE) -link-defaultlib-shared -w
 MVN = MAVEN_OPTS="-Xmx1G" mvn
 SHELL = bash
 MKDIR_P = mkdir -p

diff --git a/source/antlr/v4/runtime/CommonToken.d b/source/antlr/v4/runtime/CommonToken.d
@@ -190,7 +190,8 @@ class CommonToken : WritableToken
     public override Variant getText()
     {
         Variant Null;
-    if (text !is Null) {
+        if (text !is Null)
+        {
             return text;
         }
 
@@ -315,11 +316,13 @@ class CommonToken : WritableToken
      */
     public override string toString()
     {
+        import std.format : format;
+
         string channelStr = "";
         if (channel > 0) {
             channelStr=",channel=" ~ to!string(channel);
         }
-        string txt = to!string(getText);
+        auto txt = getText.get!(string);
         if (txt) {
             txt = txt.replace("\n","\\n");
             txt = txt.replace("\r","\\r");
@@ -328,10 +331,8 @@ class CommonToken : WritableToken
         else {
             txt = "<no text>";
         }
-        return "[@" ~ to!string(getTokenIndex) ~ "," ~ to!string(startIndex_) ~ ":"
-            ~ to!string(stopIndex_) ~ "='" ~ txt ~ "',<" ~ to!string(type)
-            ~ ">" ~ channelStr ~ "," ~ to!string(line) ~ ":"
-            ~ to!string(getCharPositionInLine) ~ "]";
+        return format!"[@%s,%s:%s='%s',<%s>%s,%s:%s]"(getTokenIndex, cast(int)startIndex_,
+            cast(int)stopIndex_, txt, type, channelStr, line, getCharPositionInLine);
     }
 
     public final size_t startIndex()

diff --git a/test/D.test.stg b/test/D.test.stg
@@ -124,26 +124,29 @@ public Token emit() {
 }
 
 private bool handleAcceptPositionForIdentifier() {
-    string tokenText = getText.get!(string);
+    import std.conv : to;
+    auto tokenText = getText.get!(string);
     int identifierLength = 0;
     while (identifierLength \< tokenText.length && isIdentifierChar(tokenText[identifierLength])) {
         identifierLength++;
     }
 
     if (getInputStream().index() > _tokenStartCharIndex + identifierLength) {
-        int offset = identifierLength - 1;
-        getInterpreter().resetAcceptPosition(getInputStream(), _tokenStartCharIndex + offset, _tokenStartLine, _tokenStartCharPositionInLine + offset);
+        auto offset = identifierLength - 1;
+        getInterpreter().resetAcceptPosition(getInputStream(), _tokenStartCharIndex + offset, _tokenStartLine,
+                                             to!int(_tokenStartCharPositionInLine + offset));
         return true;
     }
 
     return false;
 }
 
 private bool handleAcceptPositionForKeyword(string keyword) {
+    import std.conv : to;
     if (getInputStream().index() > _tokenStartCharIndex + keyword.length) {
-        import std.conv;
         auto offset = to!int(keyword.length) - 1;
-        getInterpreter.resetAcceptPosition(getInputStream(), _tokenStartCharIndex + offset, _tokenStartLine, _tokenStartCharPositionInLine + offset);
+        getInterpreter.resetAcceptPosition(getInputStream(), _tokenStartCharIndex + offset, _tokenStartLine,
+                                           to!int(_tokenStartCharPositionInLine + offset));
         return true;
     }
 
@@ -169,7 +172,8 @@ protected static class PositionAdjustingLexerATNSimulator : LexerATNSimulator {
         super(recog, atn, decisionToDFA, sharedContextCache);
     }
 
-    protected void resetAcceptPosition(CharStream input, int index, int line, int charPositionInLine) {
+    protected void resetAcceptPosition(CharStream input, size_t index, int line,
+                                       int charPositionInLine) {
         input.seek(index);
         this.line = line;
         this.charPositionInLine = charPositionInLine;

diff --git a/test/d/BaseDTest.java b/test/d/BaseDTest.java
@@ -892,7 +892,7 @@ protected void writeParserTestFile(String parserName, String lexerName,
 
     protected void writeLexerTestFile(String lexerName, boolean showDFA) {
         ST outputFileST = new ST(
-            "import std.stdio;\n"
+            "import std.stdio : File, writefln;\n"
                 + "\n"
                 + "import antlr.v4.runtime.Token;\n"
                 + "import antlr.v4.runtime.ANTLRInputStream;\n"
@@ -906,7 +906,7 @@ protected void writeLexerTestFile(String lexerName, boolean showDFA) {
                 + "  auto tokens = new CommonTokenStream(lexer);\n"
                 + "  tokens.fill();\n"
                 + "  foreach (token; tokens.getTokens)\n"
-                + "    writefln(\"%s\", token);\n"
+                + "    writefln!\"%s\"(token);\n"
                 + (showDFA ? "  std::cout \\<\\< lexer.getInterpreter\\<atn::LexerATNSimulator>()->getDFA(Lexer::DEFAULT_MODE).toLexerString();\n" : "\n")
                 + "  return 0;\n"
                 + "}\n");