preprocessor refactoring

cxx-squid/src/main/java/org/sonar/cxx/channels/KeywordChannel.java

@@ -27,7 +27,7 @@
 import java.util.Map;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
-import static org.sonar.cxx.preprocessor.CppPunctuator.HASH;
+import static org.sonar.cxx.preprocessor.PPPunctuator.HASH;
 import org.sonar.cxx.sslr.channel.Channel;
 import org.sonar.cxx.sslr.channel.CodeReader;
 

cxx-squid/src/main/java/org/sonar/cxx/config/CxxSquidConfiguration.java

@@ -55,12 +55,12 @@
  * information is not found on one level, the next higher level is searched. Additional information can be e.g. on file
  * level (translation unit), global or from sonar-project.properties.
  *
- * Pre-defined hierarchy (levels): PredefinedMacros, SonarProjectProperties, Global, Files
+ * Pre-defined hierarchy (levels): PredefinedMacros, SonarProjectProperties, Global, Units
  *
  * With {@code add} the key/value pairs are added to the database. The level parameter defines the level on which the
  * data should be inserted. For level a predefined name can be used or a new one can be defined. If level is an
  * identifier, the information is created in an element with the level-name directly under root. If level is a path, the
- * information is stored on Files level.
+ * information is stored on Units level.
  *
  * With {@code get} and {@code getValues} the information is read out again afterwards. {@code get} returns the first
  * found value for key, whereby the search starts on level. {@code getValues} collects all found values over all levels.
@@ -72,7 +72,7 @@ public class CxxSquidConfiguration extends SquidConfiguration {
   public static final String PREDEFINED_MACROS = "PredefinedMacros";
   public static final String SONAR_PROJECT_PROPERTIES = "SonarProjectProperties";
   public static final String GLOBAL = "Global";
-  public static final String FILES = "Files";
+  public static final String UNITS = "Units";
 
   // SonarProjectProperties
   public static final String ERROR_RECOVERY_ENABLED = "ErrorRecoveryEnabled";
@@ -129,19 +129,28 @@ public CxxSquidConfiguration(String baseDir, Charset encoding) {
     root.addContent(element);
     parentList.addFirst(element);
 
-    // <Files> must be first one in the list
-    element = new Element(FILES);
+    // UNITS must be first one in the list
+    element = new Element(UNITS);
     root.addContent(element);
     parentList.addFirst(element);
   }
 
+  /**
+   * Does items on Units level exist?
+   *
+   * @return false if empty
+   */
+  public boolean isUnitsEmpty() {
+    return parentList.getFirst().getContentSize() == 0;
+  }
+
   /**
    * Add a single key/value pair (property) to the database.
    *
    * @param level The level parameter defines the level on which the data should be inserted. For level a predefined
    * name can be used or a new one can be defined. <br>
    * - If level is an identifier, the information is created in an element with the level-name directly under root.<br>
-   * - If level is a path, the information is stored on Files level. In that case the level-string is normalized and
+   * - If level is a path, the information is stored on Units level. In that case the level-string is normalized and
    * converted to lower case letters to simplify the following search.
    * @param key the key to be placed into the database.
    * @param value the value corresponding to key. Several values can be assigned to one key. Internally a value-list for
@@ -239,7 +248,7 @@ public Optional<String> get(String level, String key) {
    * The method can return an empty list if the property is not set.
    *
    * @param level level to read
-   * @param property key that is searched for
+   * @param key key that is searched for
    * @return the values with the specified key value
    */
   public List<String> getLevelValues(String level, String key) {
@@ -264,7 +273,7 @@ public List<String> getLevelValues(String level, String key) {
    * are added to the end of the list. The method can return an empty list if the property is not set.
    *
    * @param level level at which the search is started
-   * @param property key that is searched for
+   * @param key key that is searched for
    * @return the values with the specified key value
    */
   public List<String> getValues(String level, String key) {
@@ -499,11 +508,11 @@ private Element findLevel(String level, @Nullable Element defaultElement) {
     if (Verifier.checkElementName(level) == null) {
       xpath = "/CompilationDatabase/" + level;
     } else {
-      // handle special case 'FILES empty' no need to search in tree
-      if (parentList.getFirst().getContentSize() == 0) {
+      // handle special case 'UNITS empty' no need to search in tree
+      if (isUnitsEmpty()) {
         return defaultElement;
       }
-      xpath = "//Files/File[@path='" + unifyPath(level) + "']";
+      xpath = "//" + UNITS + "/File[@path='" + unifyPath(level) + "']";
     }
     XPathExpression<Element> expr = xFactory.compile(xpath, Filters.element());
     List<Element> elements = expr.evaluate(document);

cxx-squid/src/main/java/org/sonar/cxx/parser/CxxLexer.java → cxx-squid/src/main/java/org/sonar/cxx/parser/CxxLexerPool.java

@@ -35,14 +35,16 @@
 import static com.sonar.cxx.sslr.impl.channel.RegexpChannelBuilder.regexp;
 import com.sonar.cxx.sslr.impl.channel.UnknownCharacterChannel;
 import java.nio.charset.Charset;
+import java.util.HashSet;
+import java.util.Set;
 import org.sonar.cxx.channels.BackslashChannel;
 import org.sonar.cxx.channels.CharacterLiteralsChannel;
 import org.sonar.cxx.channels.PreprocessorChannel;
 import org.sonar.cxx.channels.RightAngleBracketsChannel;
 import org.sonar.cxx.channels.StringLiteralsChannel;
-import org.sonar.cxx.preprocessor.CppSpecialIdentifier;
+import org.sonar.cxx.preprocessor.PPSpecialIdentifier;
 
-public final class CxxLexer {
+public final class CxxLexerPool {
 
   private static final String HEX_PREFIX = "0[xX]";
   private static final String BIN_PREFIX = "0[bB]";
@@ -57,19 +59,24 @@ public final class CxxLexer {
   private static final String BINDIGIT_SEQUENCE = "[01]([']?+[01]++)*+";
   private static final String POINT = "\\.";
 
-  private CxxLexer() {
+  private Lexer.Builder builder;
+  private final Set<Lexer> available = new HashSet<>();
+  private final Set<Lexer> inUse = new HashSet<>();
+
+  private CxxLexerPool() {
   }
 
-  public static Lexer create(Preprocessor... preprocessors) {
+  public static CxxLexerPool create(Preprocessor... preprocessors) {
     return create(Charset.defaultCharset(), preprocessors);
   }
 
-  public static Lexer create(Charset charset, Preprocessor... preprocessors) {
+  public static CxxLexerPool create(Charset charset, Preprocessor... preprocessors) {
+    var lexer = new CxxLexerPool();
 
     //
-    // changes here must be always aligned: CxxLexer.java <=> CppLexer.java
+    // changes here must be always aligned: CxxLexerPool.java <=> CppLexer.java
     //
-    var builder = Lexer.builder()
+    lexer.builder = Lexer.builder()
       .withCharset(charset)
       .withFailIfNoChannelToConsumeOneCharacter(true)
       .withChannel(new BlackHoleChannel("\\s"))
@@ -81,7 +88,7 @@ public static Lexer create(Charset charset, Preprocessor... preprocessors) {
       // detects preprocessor directives:
       // This channel detects source code lines which should be handled by the preprocessor.
       // If a line is not marked CxxTokenType.PREPROCESSOR it is not handled by CppLexer and CppGrammar.
-      .withChannel(new PreprocessorChannel(CppSpecialIdentifier.values()))
+      .withChannel(new PreprocessorChannel(PPSpecialIdentifier.values()))
       // C++ Standard, Section 2.14.3 "Character literals"
       .withChannel(new CharacterLiteralsChannel())
       // C++ Standard, Section 2.14.5 "String literals"
@@ -114,10 +121,29 @@ public static Lexer create(Charset charset, Preprocessor... preprocessors) {
       .withChannel(new UnknownCharacterChannel());
 
     for (var preprocessor : preprocessors) {
-      builder.withPreprocessor(preprocessor);
+      lexer.builder.withPreprocessor(preprocessor);
     }
 
+    return lexer;
+  }
+
+  public Lexer getLexer() {
     return builder.build();
   }
 
+  public synchronized Lexer borrowLexer() {
+    if (available.isEmpty()) {
+      available.add(getLexer());
+    }
+    var instance = available.iterator().next();
+    available.remove(instance);
+    inUse.add(instance);
+    return instance;
+  }
+
+  public synchronized void returnLexer(Lexer instance) {
+    inUse.remove(instance);
+    available.add(instance);
+  }
+
 }

cxx-squid/src/main/java/org/sonar/cxx/parser/CxxParser.java

@@ -54,7 +54,7 @@ public static Parser<Grammar> create(SquidAstVisitorContext<Grammar> context, Cx
     var cxxpp = new CxxPreprocessor(context, squidConfig);
     currentPreprocessorInstance = new WeakReference<>(cxxpp);
     return Parser.builder(CxxGrammarImpl.create(squidConfig))
-      .withLexer(CxxLexer.create(squidConfig.getCharset(), cxxpp, new JoinStringsPreprocessor()))
+      .withLexer(CxxLexerPool.create(squidConfig.getCharset(), cxxpp, new JoinStringsPreprocessor()).getLexer())
       .build();
   }