Merge pull request #2393 from guwirth/pp-refactoring-9

preprocessor refactoring

cxx-sensors/src/main/java/org/sonar/cxx/sensors/coverage/ctc/TestwellCtcTxtResult.java

@@ -50,7 +50,7 @@ public enum TestwellCtcTxtResult {
                                                   Pattern.MULTILINE);
   public static final Pattern SECTION_SEP = Pattern.compile("^(-{77}|={77})$", Pattern.MULTILINE);
   public static final Pattern LINE_RESULT = Pattern.compile(
-    "^(?: {10}| *([0-9Ee]+)) (?: {10}| *([0-9Ee]+)) -? *([0-9Ee]+) *(?:}([+-]+))?(.*)$",
+    "^(?:[ ]{10}|[ ]{0,9}([0-9Ee]{1,9}))[ ](?:[ ]{10}|[ ]{0,9}([0-9Ee]{1,9}))[ ]-?[ ]{0,9}([0-9Ee]{1,9})[ ]{0,256}(?:}([+-]+))?(.{0,256})$",
     Pattern.MULTILINE);
   public static final Pattern FILE_RESULT = Pattern.compile(
     String.join("\\s+", FILE_COND.patternString, FILE_STMT.patternString),

cxx-squid/src/main/java/org/sonar/cxx/channels/PreprocessorChannel.java

@@ -46,7 +46,7 @@ public PreprocessorChannel(TokenType[]... keywordSets) {
       for (var keyword : keywords) {
         regexp.append("|");
         regexp.append(keyword.getValue());
-        regexp.append("\\s");
+        regexp.append("\\s++");
       }
     }
     matcher = Pattern.compile(regexp.toString()).matcher("");

cxx-squid/src/main/java/org/sonar/cxx/config/MsBuild.java

@@ -167,7 +167,7 @@ public void parse(File buildLog, String baseDir, String encodingName) {
         // match "bin\CL.exe", "bin\amd64\CL.exe", "bin\x86_amd64\CL.exe"
         if (PATH_TO_CL_PATTERN.matcher(line).matches()) {
           detectedPlatform = setPlatformToolsetFromLine(line);
-          String[] allElems = line.split("\\s+");
+          String[] allElems = line.split("\\s++");
           String data = allElems[allElems.length - 1];
           parseCLParameters(line, currentProjectPath, data);
           LOG.debug("build log parser cl.exe line='{}'", line);

cxx-squid/src/main/java/org/sonar/cxx/parser/CxxLexerPool.java

@@ -79,7 +79,7 @@ public static CxxLexerPool create(Charset charset, Preprocessor... preprocessors
     lexer.builder = Lexer.builder()
       .withCharset(charset)
       .withFailIfNoChannelToConsumeOneCharacter(true)
-      .withChannel(new BlackHoleChannel("\\s"))
+      .withChannel(new BlackHoleChannel("\\s++"))
       // C++ Standard, Section 2.8 "Comments"
       .withChannel(commentRegexp("//[^\\n\\r]*+"))
       .withChannel(commentRegexp("/\\*", ANY_CHAR + "*?", "\\*/"))

cxx-squid/src/main/java/org/sonar/cxx/preprocessor/CxxPreprocessor.java

@@ -32,7 +32,6 @@
 import java.nio.charset.Charset;
 import java.util.ArrayList;
 import java.util.Collections;
-import java.util.LinkedList;
 import java.util.List;
 import java.util.Optional;
 import javax.annotation.CheckForNull;
@@ -196,7 +195,10 @@ public PreprocessorAction process(List<Token> tokens) {
     } else if (state().skipTokens() && !GenericTokenType.EOF.equals(type)) {
       return oneConsumedToken(token);
     } else if (GenericTokenType.IDENTIFIER.equals(type) || (type instanceof CxxKeyword)) {
-      return handleMacroReplacement(tokens);
+      PPMacro macro = getMacro(token.getValue());
+      if (macro != null) {
+        return macroReplacement(macro, tokens);
+      }
     }
 
     return PreprocessorAction.NO_OPERATION;
@@ -557,70 +559,71 @@ private PreprocessorAction handleModuleLine(AstNode ast, Token token) {
    * Every identifier and every keyword can be a macro instance. Pipe the resulting string through a lexer to create
    * proper tokens and to expand recursively all macros which may be in there.
    */
-  private PreprocessorAction handleMacroReplacement(List<Token> tokens) {
+  private PreprocessorAction macroReplacement(PPMacro macro, List<Token> tokens) {
     var action = PreprocessorAction.NO_OPERATION;
     var firstToken = tokens.get(0);
-    var macro = getMacro(firstToken.getValue());
-    if (macro != null) {
-      var totalTokensConsumed = 0;
-      List<Token> result;
-
-      if (macro.parameterList == null) {
-        totalTokensConsumed = 1;
-        result = new LinkedList<>(replace.replaceObjectLikeMacro(
-          macro, TokenUtils.merge(PPConcatenation.concatenate(macro.replacementList)))
-        );
-      } else {
-        result = new LinkedList<>();
-        int argsTokensConsumed = replace.replaceFunctionLikeMacro(
-          macro, tokens.subList(1, tokens.size()), result);
-        if (argsTokensConsumed > 0) {
-          totalTokensConsumed = 1 + argsTokensConsumed;
-        }
+    var consumedTokens = 0;
+    List<Token> result;
+
+    if (macro.isFunctionLikeMacro()) {
+      result = new ArrayList<>();
+      int consumedArgTokens = replace.replaceFunctionLikeMacro(
+        macro, tokens.subList(1, tokens.size()), result);
+      if (consumedArgTokens > 0) {
+        consumedTokens = 1 + consumedArgTokens;
       }
+    } else {
+      consumedTokens = 1;
+      result = replace.replaceObjectLikeMacro(
+        macro, TokenUtils.merge(PPConcatenation.concatenate(macro.replacementList))
+      );
+    }
 
-      if (totalTokensConsumed > 0) {
-        // Rescanning to expand function like macros, in case it requires consuming more tokens
-        unitMacros.pushDisable(macro.identifier);
-        List<Token> rescanningResult = new LinkedList<>();
-        totalTokensConsumed = rescanning(result, tokens, totalTokensConsumed, rescanningResult);
-        unitMacros.popDisable();
-
-        result = TokenList.adjustPosition(rescanningResult, firstToken);
-        action = new PreprocessorAction(
-          totalTokensConsumed,
-          Collections.singletonList(Trivia.createSkippedText(tokens.subList(0, totalTokensConsumed))),
-          result);
-      }
+    if (consumedTokens > 0) {
+      // Rescanning to expand function like macros, in case it requires consuming more tokens
+      unitMacros.pushDisable(macro.identifier);
+      List<Token> rescanningResult = new ArrayList<>();
+      consumedTokens = macroRescanning(result, tokens, consumedTokens, rescanningResult);
+      unitMacros.popDisable();
+
+      result = TokenList.adjustPosition(rescanningResult, firstToken);
+      action = new PreprocessorAction(
+        consumedTokens,
+        Collections.singletonList(Trivia.createSkippedText(tokens.subList(0, consumedTokens))),
+        result);
     }
 
     return action;
   }
 
-  private int rescanning(List<Token> input, List<Token> tokens, int tokensConsumed, List<Token> output) {
-    while (!input.isEmpty()) {
-      var firstToken = input.get(0);
+  private int macroRescanning(List<Token> input, List<Token> tokens, int consumedTokens, List<Token> output) {
+    List<Token> view = input;
+    while (!view.isEmpty()) {
+      var firstToken = view.get(0);
       var action = PreprocessorAction.NO_OPERATION;
-      if (firstToken.getType().equals(GenericTokenType.IDENTIFIER)) {
-        List<Token> rest = new ArrayList<>(input);
-        rest.addAll(tokens.subList(tokensConsumed, tokens.size()));
-        action = handleMacroReplacement(rest);
+      if (GenericTokenType.IDENTIFIER.equals(firstToken.getType())) {
+        PPMacro macro = getMacro(firstToken.getValue());
+        if (macro != null) {
+          List<Token> rest = new ArrayList<>(view);
+          rest.addAll(tokens.subList(consumedTokens, tokens.size()));
+          action = macroReplacement(macro, rest);
+        }
       }
-      if (action.equals(PreprocessorAction.NO_OPERATION)) {
-        input.remove(0);
+      if (PreprocessorAction.NO_OPERATION.equals(action)) {
+        view = view.subList(1, view.size()); // next tokens
         output.add(firstToken);
       } else {
         output.addAll(action.getTokensToInject());
-        int tokensConsumedRescanning = action.getNumberOfConsumedTokens();
-        if (tokensConsumedRescanning >= input.size()) {
-          tokensConsumed += tokensConsumedRescanning - input.size();
-          input.clear();
+        int consumedRescanningTokens = action.getNumberOfConsumedTokens();
+        if (consumedRescanningTokens >= view.size()) {
+          consumedTokens += consumedRescanningTokens - view.size();
+          view = view.subList(view.size(), view.size()); // was last token
         } else {
-          input.subList(0, tokensConsumedRescanning).clear();
+          view = view.subList(consumedRescanningTokens, view.size()); // next tokens
         }
       }
     }
-    return tokensConsumed;
+    return consumedTokens;
   }
 
   private static PreprocessorAction oneConsumedToken(Token token) {

cxx-squid/src/main/java/org/sonar/cxx/preprocessor/IncludeDirectiveLexer.java → cxx-squid/src/main/java/org/sonar/cxx/preprocessor/IncludeFileLexer.java

@@ -27,9 +27,13 @@
 import org.sonar.cxx.channels.PreprocessorChannel;
 import org.sonar.cxx.config.CxxSquidConfiguration;
 
-final class IncludeDirectiveLexer {
+/**
+ * Included files have to be scanned with the (only) goal of gathering macros. Process include files using
+ * a special lexer, which calls back only if it finds relevant preprocessor directives (#...).
+ */
+final class IncludeFileLexer {
 
-  private IncludeDirectiveLexer() {
+  private IncludeFileLexer() {
   }
 
   static Lexer create(Preprocessor... preprocessors) {
@@ -40,10 +44,10 @@ static Lexer create(CxxSquidConfiguration squidConfig, Preprocessor... preproces
     var builder = Lexer.builder()
       .withCharset(squidConfig.getCharset())
       .withFailIfNoChannelToConsumeOneCharacter(true)
-      .withChannel(new BlackHoleChannel("\\s"))
+      .withChannel(new BlackHoleChannel("\\s++"))
       .withChannel(new PreprocessorChannel())
       .withChannel(commentRegexp("/\\*", ANY_CHAR + "*?", "\\*/"))
-      .withChannel(new BlackHoleChannel(".*"));
+      .withChannel(new BlackHoleChannel(".*+"));
 
     for (var preprocessor : preprocessors) {
       builder.withPreprocessor(preprocessor);

cxx-squid/src/main/java/org/sonar/cxx/preprocessor/PPInclude.java

@@ -66,7 +66,7 @@ public class PPInclude {
 
   public PPInclude(CxxPreprocessor pp, @Nonnull File contextFile) {
     this.pp = pp;
-    fileLexer = IncludeDirectiveLexer.create(pp);
+    fileLexer = IncludeFileLexer.create(pp);
     state = PPState.build(contextFile);
   }
 

cxx-squid/src/main/java/org/sonar/cxx/preprocessor/PPMacro.java

@@ -90,14 +90,18 @@ static PPMacro create(String source) {
    * ...). Otherwise the program is ill-formed.
    */
   boolean checkArgumentsCount(int count) {
-    if (parameterList != null) {
+    if (isFunctionLikeMacro()) {
       return isVariadic ? count >= parameterList.size() - 1 : count == parameterList.size();
     }
     return false;
   }
 
+  boolean isFunctionLikeMacro() {
+    return parameterList != null;
+  }
+
   int getParameterIndex(String parameterName) {
-    if (parameterList != null) {
+    if (isFunctionLikeMacro()) {
       for (int i = 0; i < parameterList.size(); i++) {
         if (parameterList.get(i).getValue().equals(parameterName)) {
           return i;
@@ -112,7 +116,7 @@ public String toString() {
     StringBuilder ab = new StringBuilder(64);
     ab.append("{");
     ab.append(identifier);
-    if (parameterList != null) {
+    if (isFunctionLikeMacro()) {
       ab.append("(");
       ab.append(parameterList.stream().map(Token::getValue).collect(Collectors.joining(", ")));
       if (isVariadic) {

cxx-squid/src/test/java/org/sonar/cxx/preprocessor/IncludeDirectiveLexerTest.java → cxx-squid/src/test/java/org/sonar/cxx/preprocessor/IncludeFileLexerTest.java

@@ -28,9 +28,9 @@
 import org.junit.jupiter.api.Test;
 import org.sonar.cxx.parser.CxxTokenType;
 
-class IncludeDirectiveLexerTest {
+class IncludeFileLexerTest {
 
-  private final static Lexer LEXER = IncludeDirectiveLexer.create();
+  private final static Lexer LEXER = IncludeFileLexer.create();
 
   @Test
   void proper_preprocessor_directives_are_created() {