decoupling indices from lang.impl

GitOrigin-RevId: dcebbb96a489012754b342ab6ed0f266066cff7f
JetBrains · Jan 25, 2022 · 7aa4148 · 7aa4148
1 parent 352e14c
commit 7aa4148
Show file tree

Hide file tree

Showing 18 changed files with 725 additions and 566 deletions.
diff --git a/platform/indexing-impl/src/com/intellij/psi/stubs/StubIndexEx.java b/platform/indexing-impl/src/com/intellij/psi/stubs/StubIndexEx.java
diff --git a/...intellij/psi/stubs/StubIndexImplUtil.java → ...intellij/psi/stubs/StubIndexImplUtil.java b/...intellij/psi/stubs/StubIndexImplUtil.java → ...intellij/psi/stubs/StubIndexImplUtil.java
@@ -1,8 +1,9 @@
-// Copyright 2000-2021 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
+// Copyright 2000-2021 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
 package com.intellij.psi.stubs;
 
 import com.intellij.openapi.vfs.VirtualFile;
-import com.intellij.openapi.vfs.newvfs.persistent.PersistentFS;
+import com.intellij.util.indexing.FileBasedIndex;
+import com.intellij.util.indexing.FileBasedIndexEx;
 import it.unimi.dsi.fastutil.ints.IntIterator;
 import org.jetbrains.annotations.NotNull;
 
@@ -13,7 +14,7 @@
 class StubIndexImplUtil {
   @NotNull
   static Iterator<VirtualFile> mapIdIterator(@NotNull IntIterator idIterator, @NotNull IntPredicate filter) {
-    PersistentFS fs = PersistentFS.getInstance();
+    FileBasedIndexEx fileBasedIndex = (FileBasedIndexEx)FileBasedIndex.getInstance();
     return new Iterator<>() {
       VirtualFile next;
       boolean hasNext;
@@ -32,7 +33,7 @@ private void findNext() {
           if (!filter.test(id)) {
             continue;
           }
-          VirtualFile t = fs.findFileByIdIfCached(id);
+          VirtualFile t = fileBasedIndex.findFileById(id);
           if (t != null) {
             next = t;
             hasNext = true;

diff --git a/...ellij/psi/stubs/StubProcessingHelper.java → ...ellij/psi/stubs/StubProcessingHelper.java b/...ellij/psi/stubs/StubProcessingHelper.java → ...ellij/psi/stubs/StubProcessingHelper.java
@@ -1,4 +1,4 @@
-// Copyright 2000-2021 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
+// Copyright 2000-2021 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
 package com.intellij.psi.stubs;
 
 import com.intellij.openapi.application.AppUIExecutor;
@@ -35,7 +35,7 @@ public <Key, Psi extends PsiElement> StubIdList retrieveStubIdList(@NotNull Stub
                                                                      boolean failOnMissedKeys) {
     int id = ((VirtualFileWithId)file).getId();
     try {
-      Map<Integer, SerializedStubTree> data = StubIndexImpl.getStubUpdatingIndex().getIndexedFileData(id);
+      Map<Integer, SerializedStubTree> data = StubIndexEx.getStubUpdatingIndex().getIndexedFileData(id);
       if (data.size() != 1) {
         if (failOnMissedKeys) {
           LOG.error("Stub index points to a file (" + getFileTypeInfo(file, project) + ") without indexed stub tree; " +

diff --git a/platform/indexing-impl/src/com/intellij/psi/stubs/StubUpdatableIndexFactory.java b/platform/indexing-impl/src/com/intellij/psi/stubs/StubUpdatableIndexFactory.java
@@ -0,0 +1,24 @@
+// Copyright 2000-2022 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
+package com.intellij.psi.stubs;
+
+import com.intellij.openapi.application.ApplicationManager;
+import com.intellij.util.indexing.FileBasedIndexExtension;
+import com.intellij.util.indexing.storage.MapReduceIndexBase;
+import com.intellij.util.indexing.storage.VfsAwareIndexStorageLayout;
+import org.jetbrains.annotations.ApiStatus;
+import org.jetbrains.annotations.NotNull;
+
+import java.io.IOException;
+
+@ApiStatus.Internal
+public abstract class StubUpdatableIndexFactory {
+  static StubUpdatableIndexFactory getInstance() {
+    return ApplicationManager.getApplication().getService(StubUpdatableIndexFactory.class);
+  }
+
+  @NotNull
+  public abstract MapReduceIndexBase<Integer, SerializedStubTree> createIndex(@NotNull FileBasedIndexExtension<Integer, SerializedStubTree> extension,
+                                                                              @NotNull VfsAwareIndexStorageLayout<Integer, SerializedStubTree> layout,
+                                                                              @NotNull SerializationManagerEx serializationManager)
+    throws IOException;
+}
diff --git a/...intellij/psi/stubs/StubUpdatingIndex.java → ...intellij/psi/stubs/StubUpdatingIndex.java b/...intellij/psi/stubs/StubUpdatingIndex.java → ...intellij/psi/stubs/StubUpdatingIndex.java
@@ -16,47 +16,49 @@
 import com.intellij.openapi.progress.ProcessCanceledException;
 import com.intellij.openapi.progress.ProgressManager;
 import com.intellij.openapi.project.Project;
-import com.intellij.openapi.project.ProjectUtil;
+import com.intellij.openapi.project.ProjectLocator;
 import com.intellij.openapi.roots.impl.PushedFilePropertiesRetriever;
 import com.intellij.openapi.util.KeyedExtensionCollector;
 import com.intellij.openapi.vfs.VirtualFile;
-import com.intellij.openapi.vfs.newvfs.FileAttribute;
-import com.intellij.openapi.vfs.newvfs.persistent.FSRecords;
 import com.intellij.psi.impl.DebugUtil;
 import com.intellij.psi.tree.IFileElementType;
 import com.intellij.psi.tree.IStubFileElementType;
-import com.intellij.util.*;
+import com.intellij.util.ExceptionUtil;
+import com.intellij.util.KeyedLazyInstance;
+import com.intellij.util.ObjectUtils;
+import com.intellij.util.SystemProperties;
 import com.intellij.util.indexing.*;
 import com.intellij.util.indexing.impl.IndexDebugProperties;
 import com.intellij.util.indexing.impl.IndexStorage;
 import com.intellij.util.indexing.impl.MapReduceIndexMappingException;
 import com.intellij.util.indexing.impl.forward.ForwardIndex;
 import com.intellij.util.indexing.impl.forward.ForwardIndexAccessor;
 import com.intellij.util.indexing.impl.storage.TransientChangesIndexStorage;
+import com.intellij.util.indexing.storage.MapReduceIndexBase;
 import com.intellij.util.indexing.storage.VfsAwareIndexStorageLayout;
-import com.intellij.util.io.*;
+import com.intellij.util.io.DataExternalizer;
+import com.intellij.util.io.EnumeratorStringDescriptor;
+import com.intellij.util.io.KeyDescriptor;
+import com.intellij.util.io.PersistentHashMapValueStorage;
+import org.jetbrains.annotations.ApiStatus;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
 
-import java.io.DataInputStream;
-import java.io.DataOutputStream;
 import java.io.IOException;
 import java.util.List;
 import java.util.Objects;
 import java.util.function.Consumer;
 
 public final class StubUpdatingIndex extends SingleEntryFileBasedIndexExtension<SerializedStubTree>
   implements CustomImplementationFileBasedIndexExtension<Integer, SerializedStubTree> {
-  private static final Logger LOG = Logger.getInstance(StubUpdatingIndex.class);
+  @ApiStatus.Internal
+  static final Logger LOG = Logger.getInstance(StubUpdatingIndex.class);
   private static final boolean DEBUG_PREBUILT_INDICES = SystemProperties.getBooleanProperty("debug.prebuilt.indices", false);
 
   public static final boolean USE_SNAPSHOT_MAPPINGS = false; //TODO
 
   private static final int VERSION = 45 + (PersistentHashMapValueStorage.COMPRESSION_ENABLED ? 1 : 0);
 
-  // todo remove once we don't need this for stub-ast mismatch debug info
-  private static final FileAttribute INDEXED_STAMP = new FileAttribute("stubIndexStamp", 3, true);
-
   public static final ID<Integer, SerializedStubTree> INDEX_ID = ID.create("Stubs");
 
   @NotNull
@@ -76,7 +78,7 @@ public StubUpdatingIndex(@NotNull StubForwardIndexExternalizer<?> stubIndexesExt
   }
 
   public static boolean canHaveStub(@NotNull VirtualFile file) {
-    Project project = ProjectUtil.guessProjectForFile(file);
+    Project project = ProjectLocator.getInstance().guessProjectForFile(file);
     FileType fileType = SubstitutedFileType.substituteFileType(file, file.getFileType(), project);
     return canHaveStub(file, fileType);
   }
@@ -85,26 +87,26 @@ private static boolean canHaveStub(@NotNull VirtualFile file, @NotNull FileType
     if (fileType instanceof LanguageFileType) {
       Language l = ((LanguageFileType)fileType).getLanguage();
       ParserDefinition parserDefinition = LanguageParserDefinitions.INSTANCE.forLanguage(l);
-      FileBasedIndexImpl fileBasedIndex = ObjectUtils.tryCast(FileBasedIndex.getInstance(), FileBasedIndexImpl.class);
+      FileBasedIndexEx fileBasedIndex = ObjectUtils.tryCast(FileBasedIndex.getInstance(), FileBasedIndexEx.class);
 
       if (parserDefinition == null) {
         if (fileBasedIndex != null && fileBasedIndex.doTraceStubUpdates(StubUpdatingIndex.INDEX_ID)) {
-          FileBasedIndexImpl.LOG.info("No parser definition for " + file.getName());
+          fileBasedIndex.getLogger().info("No parser definition for " + file.getName());
         }
         return false;
       }
 
       final IFileElementType elementType = parserDefinition.getFileNodeType();
       if (elementType instanceof IStubFileElementType && ((IStubFileElementType<?>)elementType).shouldBuildStubFor(file)) {
         if (fileBasedIndex != null && fileBasedIndex.doTraceStubUpdates(StubUpdatingIndex.INDEX_ID)) {
-          FileBasedIndexImpl.LOG.info("Should build stub for " + file.getName());
+          fileBasedIndex.getLogger().info("Should build stub for " + file.getName());
         }
         return true;
       }
 
       if (fileBasedIndex != null && fileBasedIndex.doTraceStubUpdates(StubUpdatingIndex.INDEX_ID)) {
-        FileBasedIndexImpl.LOG.info("Can't build stub using stub file element type " + file.getName() +
-                                    ", properties: " + PushedFilePropertiesRetriever.getInstance().dumpSortedPushedProperties(file));
+        fileBasedIndex.getLogger().info("Can't build stub using stub file element type " + file.getName() +
+                                        ", properties: " + PushedFilePropertiesRetriever.getInstance().dumpSortedPushedProperties(file));
       }
     }
     final BinaryFileStubBuilder builder = BinaryFileStubBuilders.INSTANCE.forFileType(fileType);
@@ -278,8 +280,6 @@ private void assertPrebuiltStubTreeMatchesActualTree(@NotNull SerializedStubTree
     }
   }
 
-  private static final byte IS_BINARY_MASK = 1;
-  private static final byte BYTE_AND_CHAR_LENGTHS_ARE_THE_SAME_MASK = 1 << 1;
   @NotNull
   static IndexingStampInfo calculateIndexingStamp(@NotNull FileContent content) {
     VirtualFile file = content.getFile();
@@ -290,63 +290,11 @@ static IndexingStampInfo calculateIndexingStamp(@NotNull FileContent content) {
     return new IndexingStampInfo(file.getTimeStamp(), byteLength, contentLength, isBinary);
   }
 
-  static void saveIndexingStampInfo(@Nullable IndexingStampInfo indexingStampInfo, int fileId) {
-    try (DataOutputStream stream = FSRecords.writeAttribute(fileId, INDEXED_STAMP)) {
-      if (indexingStampInfo == null) return;
-      DataInputOutputUtil.writeTIME(stream, indexingStampInfo.indexingFileStamp);
-      DataInputOutputUtil.writeLONG(stream, indexingStampInfo.indexingByteLength);
-
-      boolean lengthsAreTheSame = indexingStampInfo.indexingCharLength == indexingStampInfo.indexingByteLength;
-      byte flags = 0;
-      flags = BitUtil.set(flags, IS_BINARY_MASK, indexingStampInfo.isBinary);
-      flags = BitUtil.set(flags, BYTE_AND_CHAR_LENGTHS_ARE_THE_SAME_MASK, lengthsAreTheSame);
-      stream.writeByte(flags);
-
-      if (!lengthsAreTheSame && !indexingStampInfo.isBinary) {
-        DataInputOutputUtil.writeINT(stream, indexingStampInfo.indexingCharLength);
-      }
-    }
-    catch (IOException e) {
-      LOG.error(e);
-    }
-  }
-
   @Override
   public int getCacheSize() {
     return super.getCacheSize() * Runtime.getRuntime().availableProcessors();
   }
 
-  @Nullable
-  static IndexingStampInfo readSavedIndexingStampInfo(@NotNull VirtualFile file) {
-    try (DataInputStream stream = INDEXED_STAMP.readAttribute(file)) {
-      if (stream == null || stream.available() <= 0) {
-        return null;
-      }
-      long stamp = DataInputOutputUtil.readTIME(stream);
-      long byteLength = DataInputOutputUtil.readLONG(stream);
-
-      byte flags = stream.readByte();
-      boolean isBinary = BitUtil.isSet(flags, IS_BINARY_MASK);
-      boolean readOnlyOneLength = BitUtil.isSet(flags, BYTE_AND_CHAR_LENGTHS_ARE_THE_SAME_MASK);
-
-      int charLength;
-      if (isBinary) {
-        charLength = -1;
-      }
-      else if (readOnlyOneLength) {
-        charLength = (int)byteLength;
-      }
-      else {
-        charLength = DataInputOutputUtil.readINT(stream);
-      }
-      return new IndexingStampInfo(stamp, byteLength, charLength, isBinary);
-    }
-    catch (IOException e) {
-      LOG.error(e);
-      return null;
-    }
-  }
-
   @NotNull
   @Override
   public DataExternalizer<SerializedStubTree> getValueExternalizer() {
@@ -377,19 +325,19 @@ public boolean enableWal() {
 
   @Override
   public void handleInitializationError(@NotNull Throwable e) {
-    ((StubIndexImpl)StubIndex.getInstance()).initializationFailed(e);
+    ((StubIndexEx)StubIndex.getInstance()).initializationFailed(e);
   }
 
   @NotNull
   @Override
   public UpdatableIndex<Integer, SerializedStubTree, FileContent> createIndexImplementation(@NotNull final FileBasedIndexExtension<Integer, SerializedStubTree> extension,
                                                                                             @NotNull VfsAwareIndexStorageLayout<Integer, SerializedStubTree> layout)
     throws StorageException, IOException {
-    ((StubIndexImpl)StubIndex.getInstance()).initializeStubIndexes();
+    ((StubIndexEx)StubIndex.getInstance()).initializeStubIndexes();
     checkNameStorage();
     mySerializationManager.initialize();
 
-    StubUpdatingIndexStorage index = new StubUpdatingIndexStorage(extension, new VfsAwareIndexStorageLayout<>() {
+    MapReduceIndexBase<Integer, SerializedStubTree> index = StubUpdatableIndexFactory.getInstance().createIndex(extension, new VfsAwareIndexStorageLayout<>() {
       @Override
       public void clearIndexData() {
         layout.clearIndexData();
@@ -416,12 +364,12 @@ public void clearIndexData() {
       memStorage.addBufferingStateListener(new TransientChangesIndexStorage.BufferingStateListener() {
         @Override
         public void bufferingStateChanged(final boolean newState) {
-          ((StubIndexImpl)StubIndex.getInstance()).setDataBufferingEnabled(newState);
+          ((StubIndexEx)StubIndex.getInstance()).setDataBufferingEnabled(newState);
         }
 
         @Override
         public void memoryStorageCleared() {
-          ((StubIndexImpl)StubIndex.getInstance()).cleanupMemoryStorage();
+          ((StubIndexEx)StubIndex.getInstance()).cleanupMemoryStorage();
         }
       });
     }

diff --git a/platform/indexing-impl/src/com/intellij/util/indexing/FileBasedIndexEx.java b/platform/indexing-impl/src/com/intellij/util/indexing/FileBasedIndexEx.java
@@ -1,4 +1,4 @@
-// Copyright 2000-2021 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
+// Copyright 2000-2021 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
 package com.intellij.util.indexing;
 
 import com.intellij.ide.lightEdit.LightEditCompatible;
@@ -26,6 +26,7 @@
 import com.intellij.psi.search.EverythingGlobalScope;
 import com.intellij.psi.search.GlobalSearchScope;
 import com.intellij.psi.search.impl.VirtualFileEnumeration;
+import com.intellij.psi.stubs.StubUpdatingIndex;
 import com.intellij.psi.util.PsiModificationTracker;
 import com.intellij.util.*;
 import com.intellij.util.containers.ContainerUtil;
@@ -51,11 +52,37 @@
 
 @ApiStatus.Internal
 public abstract class FileBasedIndexEx extends FileBasedIndex {
+  public static final boolean DO_TRACE_STUB_INDEX_UPDATE = Boolean.getBoolean("idea.trace.stub.index.update");
   @SuppressWarnings("SSBasedInspection")
   private static final ThreadLocal<Stack<DumbModeAccessType>> ourDumbModeAccessTypeStack =
     ThreadLocal.withInitial(() -> new com.intellij.util.containers.Stack<>());
   private static final RecursionGuard<Object> ourIgnoranceGuard = RecursionManager.createGuard("ignoreDumbMode");
   private final IndexAccessValidator myAccessValidator = new IndexAccessValidator();
+  private volatile boolean myTraceIndexUpdates;
+  private volatile boolean myTraceStubIndexUpdates;
+  private volatile boolean myTraceSharedIndexUpdates;
+
+  @ApiStatus.Internal
+  boolean doTraceIndexUpdates() {
+    return myTraceIndexUpdates;
+  }
+
+  @ApiStatus.Internal
+  public boolean doTraceStubUpdates(@NotNull ID<?, ?> indexId) {
+    return myTraceStubIndexUpdates && indexId.equals(StubUpdatingIndex.INDEX_ID);
+  }
+
+  @ApiStatus.Internal
+  boolean doTraceSharedIndexUpdates() {
+    return myTraceSharedIndexUpdates;
+  }
+
+  @ApiStatus.Internal
+  public void loadIndexes() {
+    myTraceIndexUpdates = SystemProperties.getBooleanProperty("trace.file.based.index.update", false);
+    myTraceStubIndexUpdates = SystemProperties.getBooleanProperty("trace.stub.index.update", false);
+    myTraceSharedIndexUpdates = SystemProperties.getBooleanProperty("trace.shared.index.update", false);
+  }
 
   @ApiStatus.Internal
   @NotNull
@@ -709,8 +736,6 @@ public static boolean isTooLarge(@NotNull VirtualFile file,
     return false;
   }
 
-  public static final boolean DO_TRACE_STUB_INDEX_UPDATE = Boolean.getBoolean("idea.trace.stub.index.update");
-
   public static boolean acceptsInput(@NotNull InputFilter filter, @NotNull IndexedFile indexedFile) {
     if (filter instanceof ProjectSpecificInputFilter) {
       if (indexedFile.getProject() == null) {
@@ -733,4 +758,13 @@ public boolean acceptInput(@NotNull IndexedFile file) {
     }
            : file -> filter.acceptInput(file) && condition.test(file);
   }
+
+  @ApiStatus.Internal
+  public void runCleanupAction(@NotNull Runnable cleanupAction) {
+  }
+
+  @ApiStatus.Internal
+  public static <T,E extends Throwable> T disableUpToDateCheckIn(@NotNull ThrowableComputable<T, E> runnable) throws E {
+    return IndexUpToDateCheckIn.disableUpToDateCheckIn(runnable);
+  }
 }
diff --git a/...ij/util/indexing/IndexInfrastructure.java → ...ij/util/indexing/IndexInfrastructure.java b/...ij/util/indexing/IndexInfrastructure.java → ...ij/util/indexing/IndexInfrastructure.java
diff --git a/...lij/util/indexing/IndexUpToDateCheckIn.kt → ...lij/util/indexing/IndexUpToDateCheckIn.kt b/...lij/util/indexing/IndexUpToDateCheckIn.kt → ...lij/util/indexing/IndexUpToDateCheckIn.kt
@@ -2,8 +2,10 @@
 package com.intellij.util.indexing
 
 import com.intellij.openapi.util.ThrowableComputable
+import org.jetbrains.annotations.ApiStatus
 
-internal object IndexUpToDateCheckIn {
+@ApiStatus.Internal
+object IndexUpToDateCheckIn {
   private val upToDateCheckState = ThreadLocal<Int>()
 
   @JvmStatic

diff --git a/...ransientChangeTrackingValueContainer.java → ...ransientChangeTrackingValueContainer.java b/...ransientChangeTrackingValueContainer.java → ...ransientChangeTrackingValueContainer.java
diff --git a/...storage/TransientChangesIndexStorage.java → ...storage/TransientChangesIndexStorage.java b/...storage/TransientChangesIndexStorage.java → ...storage/TransientChangesIndexStorage.java
@@ -1,4 +1,4 @@
-// Copyright 2000-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
+// Copyright 2000-2022 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
 
 package com.intellij.util.indexing.impl.storage;
 
@@ -90,7 +90,7 @@ public void clearCaches() {
 
       if (IndexDebugProperties.DEBUG) {
         String message = "Dropping caches for " + myIndexId + ", number of items:" + myMap.size();
-        FileBasedIndexImpl.LOG.info(message);
+        ((FileBasedIndexEx)FileBasedIndex.getInstance()).getLogger().info(message);
       }
 
       for (ChangeTrackingValueContainer<Value> v : myMap.values()) {