From 7aa4148d5e746e99fe42cb229f33928d7549e606 Mon Sep 17 00:00:00 2001 From: Petr Golubev Date: Mon, 24 Jan 2022 23:54:33 +0300 Subject: [PATCH] decoupling indices from lang.impl GitOrigin-RevId: dcebbb96a489012754b342ab6ed0f266066cff7f --- .../com/intellij/psi/stubs/StubIndexEx.java | 377 +++++++++++++++++- .../intellij/psi/stubs/StubIndexImplUtil.java | 9 +- .../psi/stubs/StubProcessingHelper.java | 4 +- .../psi/stubs/StubUpdatableIndexFactory.java | 24 ++ .../intellij/psi/stubs/StubUpdatingIndex.java | 100 ++--- .../util/indexing/FileBasedIndexEx.java | 40 +- .../util/indexing/IndexInfrastructure.java | 0 .../util/indexing/IndexUpToDateCheckIn.kt | 4 +- ...TransientChangeTrackingValueContainer.java | 0 .../storage/TransientChangesIndexStorage.java | 4 +- .../indexing/storage/MapReduceIndexBase.java | 146 +++++++ .../com/intellij/psi/stubs/StubIndexImpl.java | 339 +--------------- .../psi/stubs/StubTreeLoaderImpl.java | 64 ++- .../stubs/StubUpdatableIndexFactoryImpl.java | 22 + .../psi/stubs/StubUpdatingIndexStorage.java | 2 +- .../util/indexing/FileBasedIndexImpl.java | 35 +- .../impl/storage/VfsAwareMapReduceIndex.java | 118 +----- .../src/META-INF/LangExtensions.xml | 3 + 18 files changed, 725 insertions(+), 566 deletions(-) rename platform/{lang-impl => indexing-impl}/src/com/intellij/psi/stubs/StubIndexImplUtil.java (77%) rename platform/{lang-impl => indexing-impl}/src/com/intellij/psi/stubs/StubProcessingHelper.java (95%) create mode 100644 platform/indexing-impl/src/com/intellij/psi/stubs/StubUpdatableIndexFactory.java rename platform/{lang-impl => indexing-impl}/src/com/intellij/psi/stubs/StubUpdatingIndex.java (81%) rename platform/{lang-impl => indexing-impl}/src/com/intellij/util/indexing/IndexInfrastructure.java (100%) rename platform/{lang-impl => indexing-impl}/src/com/intellij/util/indexing/IndexUpToDateCheckIn.kt (92%) rename platform/{lang-impl => indexing-impl}/src/com/intellij/util/indexing/impl/storage/TransientChangeTrackingValueContainer.java (100%) rename platform/{lang-impl => indexing-impl}/src/com/intellij/util/indexing/impl/storage/TransientChangesIndexStorage.java (96%) create mode 100644 platform/indexing-impl/src/com/intellij/util/indexing/storage/MapReduceIndexBase.java create mode 100644 platform/lang-impl/src/com/intellij/psi/stubs/StubUpdatableIndexFactoryImpl.java diff --git a/platform/indexing-impl/src/com/intellij/psi/stubs/StubIndexEx.java b/platform/indexing-impl/src/com/intellij/psi/stubs/StubIndexEx.java index b8f02f3975bac..874dc0be33838 100644 --- a/platform/indexing-impl/src/com/intellij/psi/stubs/StubIndexEx.java +++ b/platform/indexing-impl/src/com/intellij/psi/stubs/StubIndexEx.java @@ -1,20 +1,44 @@ // Copyright 2000-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file. package com.intellij.psi.stubs; +import com.intellij.ide.lightEdit.LightEditCompatible; +import com.intellij.model.ModelBranchImpl; import com.intellij.openapi.diagnostic.Logger; import com.intellij.openapi.progress.ProgressManager; +import com.intellij.openapi.project.DumbService; +import com.intellij.openapi.project.Project; +import com.intellij.openapi.util.ModificationTracker; +import com.intellij.openapi.vfs.CompactVirtualFileSet; +import com.intellij.openapi.vfs.VirtualFile; +import com.intellij.openapi.vfs.VirtualFileWithId; +import com.intellij.psi.PsiElement; +import com.intellij.psi.search.GlobalSearchScope; +import com.intellij.psi.util.CachedValue; +import com.intellij.psi.util.CachedValueProvider; +import com.intellij.util.*; +import com.intellij.util.containers.FactoryMap; import com.intellij.util.indexing.*; +import com.intellij.util.indexing.diagnostic.IndexAccessValidator; import com.intellij.util.indexing.impl.AbstractUpdateData; import com.intellij.util.indexing.impl.KeyValueUpdateProcessor; import com.intellij.util.indexing.impl.RemovedKeyProcessor; import com.intellij.util.io.DataExternalizer; import com.intellij.util.io.KeyDescriptor; import com.intellij.util.io.VoidDataExternalizer; +import it.unimi.dsi.fastutil.ints.IntIterator; +import it.unimi.dsi.fastutil.ints.IntLinkedOpenHashSet; +import it.unimi.dsi.fastutil.ints.IntSet; +import it.unimi.dsi.fastutil.objects.ObjectIterators; import org.jetbrains.annotations.ApiStatus; import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.jetbrains.annotations.TestOnly; -import java.util.Arrays; -import java.util.Set; +import java.io.IOException; +import java.util.*; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.locks.Lock; +import java.util.function.IntPredicate; @ApiStatus.Internal public abstract class StubIndexEx extends StubIndex { @@ -25,6 +49,21 @@ static void initExtensions() { } } + private final Map, CachedValue, StubIdList>>> myCachedStubIds = FactoryMap.createMap(k -> { + UpdatableIndex index = getStubUpdatingIndex(); + ModificationTracker tracker = index::getModificationStamp; + return new CachedValueImpl<>(() -> new CachedValueProvider.Result<>(new ConcurrentHashMap<>(), tracker)); + }, ConcurrentHashMap::new); + + private final StubProcessingHelper myStubProcessingHelper = new StubProcessingHelper(); + private final IndexAccessValidator myAccessValidator = new IndexAccessValidator(); + + @ApiStatus.Internal + abstract void initializeStubIndexes(); + + @ApiStatus.Internal + public abstract void initializationFailed(@NotNull Throwable error); + public void updateIndex(@NotNull StubIndexKey stubIndexKey, int fileId, @NotNull Set oldKeys, @@ -80,10 +119,292 @@ protected boolean iterateKeys(@NotNull KeyValueUpdateProcessor boolean processElements(@NotNull StubIndexKey indexKey, + @NotNull Key key, + @NotNull Project project, + @Nullable GlobalSearchScope scope, + @Nullable IdFilter idFilter, + @NotNull Class requiredClass, + @NotNull Processor processor) { + boolean dumb = DumbService.isDumb(project); + if (dumb) { + if (project instanceof LightEditCompatible) return false; + DumbModeAccessType accessType = FileBasedIndex.getInstance().getCurrentDumbModeAccessType(); + if (accessType == DumbModeAccessType.RAW_INDEX_DATA_ACCEPTABLE) { + throw new AssertionError("raw index data access is not available for StubIndex"); + } + } + + PairProcessor stubProcessor = (file, list) -> + myStubProcessingHelper.processStubsInFile(project, file, list, processor, scope, requiredClass); + + if (!ModelBranchImpl.processModifiedFilesInScope(scope != null ? scope : GlobalSearchScope.everythingScope(project), + file -> processInMemoryStubs(indexKey, key, project, stubProcessor, file))) { + return false; + } + + Iterator singleFileInScope = extractSingleFile(scope); + Iterator fileStream; + boolean shouldHaveKeys; + + if (singleFileInScope != null) { + if (!(singleFileInScope.hasNext())) return true; + FileBasedIndex.getInstance().ensureUpToDate(StubUpdatingIndex.INDEX_ID, project, scope); + fileStream = singleFileInScope; + shouldHaveKeys = false; + } + else { + IntSet ids = getContainingIds(indexKey, key, project, idFilter, scope); + if (ids == null) return true; + IntPredicate accessibleFileFilter = ((FileBasedIndexEx)FileBasedIndex.getInstance()).getAccessibleFileIdFilter(project); + // already ensured up-to-date in getContainingIds() method + IntIterator idIterator = ids.iterator(); + fileStream = StubIndexImplUtil.mapIdIterator(idIterator, accessibleFileFilter); + shouldHaveKeys = true; + } + + try { + while (fileStream.hasNext()) { + VirtualFile file = fileStream.next(); + assert file != null; + + List filesInScope = scope != null ? FileBasedIndexEx.filesInScopeWithBranches(scope, file) : Collections.singletonList(file); + if (filesInScope.isEmpty()) { + continue; + } + + int id = ((VirtualFileWithId)file).getId(); + StubIdList list = myCachedStubIds.get(indexKey).getValue().computeIfAbsent(new KeyAndFileId<>(key, id), __ -> + myStubProcessingHelper.retrieveStubIdList(indexKey, key, file, project, shouldHaveKeys) + ); + if (list == null) { + // stub index inconsistency + continue; + } + for (VirtualFile eachFile : filesInScope) { + if (!stubProcessor.process(eachFile, list)) { + return false; + } + } + } + } + catch (RuntimeException e) { + final Throwable cause = FileBasedIndexEx.getCauseToRebuildIndex(e); + if (cause != null) { + forceRebuild(cause); + } + else { + throw e; + } + } finally { + wipeProblematicFileIdsForParticularKeyAndStubIndex(indexKey, key); + } + return true; + } + + private static boolean processInMemoryStubs(StubIndexKey indexKey, + Key key, + Project project, + PairProcessor stubProcessor, + VirtualFile file) { + Map data = FileBasedIndex.getInstance().getFileData(StubUpdatingIndex.INDEX_ID, file, project); + if (data.size() == 1) { + try { + StubIdList list = data.values().iterator().next().restoreIndexedStubs(indexKey, key); + if (list != null) { + return stubProcessor.process(file, list); + } + } + catch (IOException e) { + throw new RuntimeException(e); + } + } + return true; + } + @ApiStatus.Experimental @ApiStatus.Internal protected abstract UpdatableIndex getIndex(@NotNull StubIndexKey indexKey); + // Self repair for IDEA-181227, caused by (yet) unknown file event processing problem in indices + // FileBasedIndex.requestReindex doesn't handle the situation properly because update requires old data that was lost + private void wipeProblematicFileIdsForParticularKeyAndStubIndex(@NotNull StubIndexKey indexKey, + @NotNull Key key) { + Set filesWithProblems = myStubProcessingHelper.takeAccumulatedFilesWithIndexProblems(); + + if (filesWithProblems != null) { + getLogger().info("data for " + indexKey.getName() + " will be wiped for a some files because of internal stub processing error"); + ((FileBasedIndexEx)FileBasedIndex.getInstance()).runCleanupAction(() -> { + Lock writeLock = getIndex(indexKey).getLock().writeLock(); + boolean locked = writeLock.tryLock(); + if (!locked) return; // nested indices invocation, can not cleanup without deadlock + try { + for (VirtualFile file : filesWithProblems) { + updateIndex(indexKey, + FileBasedIndex.getFileId(file), + Collections.singleton(key), + Collections.emptySet()); + } + } + finally { + writeLock.unlock(); + } + }); + } + } + + @Override + public @NotNull Collection getAllKeys(@SuppressWarnings("BoundedWildcard") @NotNull StubIndexKey indexKey, @NotNull Project project) { + Set allKeys = new HashSet<>(); + processAllKeys(indexKey, project, Processors.cancelableCollectProcessor(allKeys)); + return allKeys; + } + + @Override + public boolean processAllKeys(@NotNull StubIndexKey indexKey, + @NotNull Processor processor, + @NotNull GlobalSearchScope scope, + @Nullable IdFilter idFilter) { + final UpdatableIndex index = getIndex(indexKey); // wait for initialization to finish + FileBasedIndexEx fileBasedIndexEx = (FileBasedIndexEx)FileBasedIndex.getInstance(); + if (index == null || + !fileBasedIndexEx.ensureUpToDate(StubUpdatingIndex.INDEX_ID, scope.getProject(), scope, null)) { + return true; + } + + if (idFilter == null) { + idFilter = fileBasedIndexEx.extractIdFilter(scope, scope.getProject()); + } + + try { + @Nullable IdFilter finalIdFilter = idFilter; + return myAccessValidator.validate(StubUpdatingIndex.INDEX_ID, ()->FileBasedIndexEx.disableUpToDateCheckIn(()-> + index.processAllKeys(processor, scope, finalIdFilter))); + } + catch (StorageException e) { + forceRebuild(e); + } + catch (RuntimeException e) { + final Throwable cause = e.getCause(); + if (cause instanceof IOException || cause instanceof StorageException) { + forceRebuild(e); + } + throw e; + } + return true; + } + + @Override + public @NotNull IdIterator getContainingIds(@NotNull StubIndexKey indexKey, + @NotNull Key dataKey, + final @NotNull Project project, + final @Nullable GlobalSearchScope scope) { + IntSet result = getContainingIds(indexKey, dataKey, project, null, scope); + if (result == null) return IdIterator.EMPTY; + return new IdIterator() { + final IntIterator iterator = result.iterator(); + + @Override + public boolean hasNext() { + return iterator.hasNext(); + } + + @Override + public int next() { + return iterator.nextInt(); + } + + @Override + public int size() { + return result.size(); + } + }; + } + + @Override + public @NotNull Iterator getContainingFiles(@NotNull StubIndexKey indexKey, + @NotNull Key dataKey, + @NotNull Project project, + @NotNull GlobalSearchScope scope) { + IntSet result = getContainingIds(indexKey, dataKey, project, null, scope); + Set fileSet = new CompactVirtualFileSet(result == null ? ArrayUtil.EMPTY_INT_ARRAY : result.toIntArray()).freeze(); + return fileSet.stream().filter(scope::contains).iterator(); + } + + @Override + public int getMaxContainingFileCount(@NotNull StubIndexKey indexKey, + @NotNull Key dataKey, + @NotNull Project project, + @NotNull GlobalSearchScope scope) { + IntSet result = getContainingIds(indexKey, dataKey, project, null, scope); + return result == null ? 0 : result.size(); + } + + private @Nullable IntSet getContainingIds(@NotNull StubIndexKey indexKey, + @NotNull Key dataKey, + final @NotNull Project project, + @Nullable IdFilter idFilter, + final @Nullable GlobalSearchScope scope) { + final FileBasedIndexEx fileBasedIndex = (FileBasedIndexEx)FileBasedIndex.getInstance(); + ID stubUpdatingIndexId = StubUpdatingIndex.INDEX_ID; + final UpdatableIndex index = getIndex(indexKey); // wait for initialization to finish + if (index == null || !fileBasedIndex.ensureUpToDate(stubUpdatingIndexId, project, scope, null)) return null; + + IdFilter finalIdFilter = idFilter != null + ? idFilter + : ((FileBasedIndexEx)FileBasedIndex.getInstance()).extractIdFilter(scope, project); + + UpdatableIndex stubUpdatingIndex = fileBasedIndex.getIndex(stubUpdatingIndexId); + + try { + IntSet result = new IntLinkedOpenHashSet(); // workaround duplicates keys + myAccessValidator.validate(stubUpdatingIndexId, ()-> { + // disable up-to-date check to avoid locks on attempt to acquire index write lock while holding at the same time the readLock for this index + //noinspection Convert2Lambda (workaround for JBR crash, JBR-2349),Convert2Diamond + return FileBasedIndexEx.disableUpToDateCheckIn(() -> ConcurrencyUtil.withLock(stubUpdatingIndex.getLock().readLock(), () -> + index.getData(dataKey).forEach(new ValueContainer.ContainerAction<>() { + @Override + public boolean perform(int id, Void value) { + if (finalIdFilter == null || finalIdFilter.containsFileId(id)) { + result.add(id); + } + return true; + } + }) + )); + }); + return result; + } + catch (StorageException e) { + forceRebuild(e); + } + catch (RuntimeException e) { + final Throwable cause = FileBasedIndexEx.getCauseToRebuildIndex(e); + if (cause != null) { + forceRebuild(cause); + } + else { + throw e; + } + } + + return null; + } + + @ApiStatus.Internal + protected void clearState() { + StubIndexKeyDescriptorCache.INSTANCE.clear(); + ((SerializationManagerImpl)SerializationManagerEx.getInstanceEx()).dropSerializerData(); + myCachedStubIds.clear(); + } + + @ApiStatus.Internal + void setDataBufferingEnabled(final boolean enabled) { } + + @ApiStatus.Internal + void cleanupMemoryStorage() { } + @ApiStatus.Internal public static @NotNull FileBasedIndexExtension wrapStubIndexExtension(StubIndexExtension extension) { return new FileBasedIndexExtension<>() { @@ -138,4 +459,56 @@ public boolean traceKeyHashToVirtualFileMapping() { } }; } + + @ApiStatus.Internal + static UpdatableIndex getStubUpdatingIndex() { + return ((FileBasedIndexEx)FileBasedIndex.getInstance()).getIndex(StubUpdatingIndex.INDEX_ID); + } + + @SuppressWarnings("unchecked") + private static @Nullable Iterator extractSingleFile(@Nullable GlobalSearchScope scope) { + if (!(scope instanceof Iterable)) { + return null; + } + Iterable scopeAsFileIterable = (Iterable)scope; + Iterator result = null; + for (VirtualFile file : scopeAsFileIterable) { + if (result == null) { + result = file != null ? ObjectIterators.singleton(file) : ObjectIterators.emptyIterator(); + } + else { + return null; + } + } + return result; + } + + private static final class KeyAndFileId { + @NotNull + private final K key; + private final int fileId; + + private KeyAndFileId(@NotNull K key, int fileId) { + this.key = key; + this.fileId = fileId; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + KeyAndFileId key1 = (KeyAndFileId)o; + return fileId == key1.fileId && Objects.equals(key, key1.key); + } + + @Override + public int hashCode() { + return Objects.hash(key, fileId); + } + } + + @TestOnly + public boolean areAllProblemsProcessedInTheCurrentThread() { + return myStubProcessingHelper.areAllProblemsProcessedInTheCurrentThread(); + } } diff --git a/platform/lang-impl/src/com/intellij/psi/stubs/StubIndexImplUtil.java b/platform/indexing-impl/src/com/intellij/psi/stubs/StubIndexImplUtil.java similarity index 77% rename from platform/lang-impl/src/com/intellij/psi/stubs/StubIndexImplUtil.java rename to platform/indexing-impl/src/com/intellij/psi/stubs/StubIndexImplUtil.java index 4157a8358ef68..3cec33d516e01 100644 --- a/platform/lang-impl/src/com/intellij/psi/stubs/StubIndexImplUtil.java +++ b/platform/indexing-impl/src/com/intellij/psi/stubs/StubIndexImplUtil.java @@ -1,8 +1,9 @@ -// Copyright 2000-2021 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file. +// Copyright 2000-2021 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. package com.intellij.psi.stubs; import com.intellij.openapi.vfs.VirtualFile; -import com.intellij.openapi.vfs.newvfs.persistent.PersistentFS; +import com.intellij.util.indexing.FileBasedIndex; +import com.intellij.util.indexing.FileBasedIndexEx; import it.unimi.dsi.fastutil.ints.IntIterator; import org.jetbrains.annotations.NotNull; @@ -13,7 +14,7 @@ class StubIndexImplUtil { @NotNull static Iterator mapIdIterator(@NotNull IntIterator idIterator, @NotNull IntPredicate filter) { - PersistentFS fs = PersistentFS.getInstance(); + FileBasedIndexEx fileBasedIndex = (FileBasedIndexEx)FileBasedIndex.getInstance(); return new Iterator<>() { VirtualFile next; boolean hasNext; @@ -32,7 +33,7 @@ private void findNext() { if (!filter.test(id)) { continue; } - VirtualFile t = fs.findFileByIdIfCached(id); + VirtualFile t = fileBasedIndex.findFileById(id); if (t != null) { next = t; hasNext = true; diff --git a/platform/lang-impl/src/com/intellij/psi/stubs/StubProcessingHelper.java b/platform/indexing-impl/src/com/intellij/psi/stubs/StubProcessingHelper.java similarity index 95% rename from platform/lang-impl/src/com/intellij/psi/stubs/StubProcessingHelper.java rename to platform/indexing-impl/src/com/intellij/psi/stubs/StubProcessingHelper.java index 1b6dc7b0ac2ca..50571ad38c013 100644 --- a/platform/lang-impl/src/com/intellij/psi/stubs/StubProcessingHelper.java +++ b/platform/indexing-impl/src/com/intellij/psi/stubs/StubProcessingHelper.java @@ -1,4 +1,4 @@ -// Copyright 2000-2021 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file. +// Copyright 2000-2021 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. package com.intellij.psi.stubs; import com.intellij.openapi.application.AppUIExecutor; @@ -35,7 +35,7 @@ public StubIdList retrieveStubIdList(@NotNull Stub boolean failOnMissedKeys) { int id = ((VirtualFileWithId)file).getId(); try { - Map data = StubIndexImpl.getStubUpdatingIndex().getIndexedFileData(id); + Map data = StubIndexEx.getStubUpdatingIndex().getIndexedFileData(id); if (data.size() != 1) { if (failOnMissedKeys) { LOG.error("Stub index points to a file (" + getFileTypeInfo(file, project) + ") without indexed stub tree; " + diff --git a/platform/indexing-impl/src/com/intellij/psi/stubs/StubUpdatableIndexFactory.java b/platform/indexing-impl/src/com/intellij/psi/stubs/StubUpdatableIndexFactory.java new file mode 100644 index 0000000000000..76d2e7510bd4c --- /dev/null +++ b/platform/indexing-impl/src/com/intellij/psi/stubs/StubUpdatableIndexFactory.java @@ -0,0 +1,24 @@ +// Copyright 2000-2022 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. +package com.intellij.psi.stubs; + +import com.intellij.openapi.application.ApplicationManager; +import com.intellij.util.indexing.FileBasedIndexExtension; +import com.intellij.util.indexing.storage.MapReduceIndexBase; +import com.intellij.util.indexing.storage.VfsAwareIndexStorageLayout; +import org.jetbrains.annotations.ApiStatus; +import org.jetbrains.annotations.NotNull; + +import java.io.IOException; + +@ApiStatus.Internal +public abstract class StubUpdatableIndexFactory { + static StubUpdatableIndexFactory getInstance() { + return ApplicationManager.getApplication().getService(StubUpdatableIndexFactory.class); + } + + @NotNull + public abstract MapReduceIndexBase createIndex(@NotNull FileBasedIndexExtension extension, + @NotNull VfsAwareIndexStorageLayout layout, + @NotNull SerializationManagerEx serializationManager) + throws IOException; +} diff --git a/platform/lang-impl/src/com/intellij/psi/stubs/StubUpdatingIndex.java b/platform/indexing-impl/src/com/intellij/psi/stubs/StubUpdatingIndex.java similarity index 81% rename from platform/lang-impl/src/com/intellij/psi/stubs/StubUpdatingIndex.java rename to platform/indexing-impl/src/com/intellij/psi/stubs/StubUpdatingIndex.java index 7ae61402eac54..7ab6529822d3e 100644 --- a/platform/lang-impl/src/com/intellij/psi/stubs/StubUpdatingIndex.java +++ b/platform/indexing-impl/src/com/intellij/psi/stubs/StubUpdatingIndex.java @@ -16,16 +16,17 @@ import com.intellij.openapi.progress.ProcessCanceledException; import com.intellij.openapi.progress.ProgressManager; import com.intellij.openapi.project.Project; -import com.intellij.openapi.project.ProjectUtil; +import com.intellij.openapi.project.ProjectLocator; import com.intellij.openapi.roots.impl.PushedFilePropertiesRetriever; import com.intellij.openapi.util.KeyedExtensionCollector; import com.intellij.openapi.vfs.VirtualFile; -import com.intellij.openapi.vfs.newvfs.FileAttribute; -import com.intellij.openapi.vfs.newvfs.persistent.FSRecords; import com.intellij.psi.impl.DebugUtil; import com.intellij.psi.tree.IFileElementType; import com.intellij.psi.tree.IStubFileElementType; -import com.intellij.util.*; +import com.intellij.util.ExceptionUtil; +import com.intellij.util.KeyedLazyInstance; +import com.intellij.util.ObjectUtils; +import com.intellij.util.SystemProperties; import com.intellij.util.indexing.*; import com.intellij.util.indexing.impl.IndexDebugProperties; import com.intellij.util.indexing.impl.IndexStorage; @@ -33,13 +34,16 @@ import com.intellij.util.indexing.impl.forward.ForwardIndex; import com.intellij.util.indexing.impl.forward.ForwardIndexAccessor; import com.intellij.util.indexing.impl.storage.TransientChangesIndexStorage; +import com.intellij.util.indexing.storage.MapReduceIndexBase; import com.intellij.util.indexing.storage.VfsAwareIndexStorageLayout; -import com.intellij.util.io.*; +import com.intellij.util.io.DataExternalizer; +import com.intellij.util.io.EnumeratorStringDescriptor; +import com.intellij.util.io.KeyDescriptor; +import com.intellij.util.io.PersistentHashMapValueStorage; +import org.jetbrains.annotations.ApiStatus; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; -import java.io.DataInputStream; -import java.io.DataOutputStream; import java.io.IOException; import java.util.List; import java.util.Objects; @@ -47,16 +51,14 @@ public final class StubUpdatingIndex extends SingleEntryFileBasedIndexExtension implements CustomImplementationFileBasedIndexExtension { - private static final Logger LOG = Logger.getInstance(StubUpdatingIndex.class); + @ApiStatus.Internal + static final Logger LOG = Logger.getInstance(StubUpdatingIndex.class); private static final boolean DEBUG_PREBUILT_INDICES = SystemProperties.getBooleanProperty("debug.prebuilt.indices", false); public static final boolean USE_SNAPSHOT_MAPPINGS = false; //TODO private static final int VERSION = 45 + (PersistentHashMapValueStorage.COMPRESSION_ENABLED ? 1 : 0); - // todo remove once we don't need this for stub-ast mismatch debug info - private static final FileAttribute INDEXED_STAMP = new FileAttribute("stubIndexStamp", 3, true); - public static final ID INDEX_ID = ID.create("Stubs"); @NotNull @@ -76,7 +78,7 @@ public StubUpdatingIndex(@NotNull StubForwardIndexExternalizer stubIndexesExt } public static boolean canHaveStub(@NotNull VirtualFile file) { - Project project = ProjectUtil.guessProjectForFile(file); + Project project = ProjectLocator.getInstance().guessProjectForFile(file); FileType fileType = SubstitutedFileType.substituteFileType(file, file.getFileType(), project); return canHaveStub(file, fileType); } @@ -85,11 +87,11 @@ private static boolean canHaveStub(@NotNull VirtualFile file, @NotNull FileType if (fileType instanceof LanguageFileType) { Language l = ((LanguageFileType)fileType).getLanguage(); ParserDefinition parserDefinition = LanguageParserDefinitions.INSTANCE.forLanguage(l); - FileBasedIndexImpl fileBasedIndex = ObjectUtils.tryCast(FileBasedIndex.getInstance(), FileBasedIndexImpl.class); + FileBasedIndexEx fileBasedIndex = ObjectUtils.tryCast(FileBasedIndex.getInstance(), FileBasedIndexEx.class); if (parserDefinition == null) { if (fileBasedIndex != null && fileBasedIndex.doTraceStubUpdates(StubUpdatingIndex.INDEX_ID)) { - FileBasedIndexImpl.LOG.info("No parser definition for " + file.getName()); + fileBasedIndex.getLogger().info("No parser definition for " + file.getName()); } return false; } @@ -97,14 +99,14 @@ private static boolean canHaveStub(@NotNull VirtualFile file, @NotNull FileType final IFileElementType elementType = parserDefinition.getFileNodeType(); if (elementType instanceof IStubFileElementType && ((IStubFileElementType)elementType).shouldBuildStubFor(file)) { if (fileBasedIndex != null && fileBasedIndex.doTraceStubUpdates(StubUpdatingIndex.INDEX_ID)) { - FileBasedIndexImpl.LOG.info("Should build stub for " + file.getName()); + fileBasedIndex.getLogger().info("Should build stub for " + file.getName()); } return true; } if (fileBasedIndex != null && fileBasedIndex.doTraceStubUpdates(StubUpdatingIndex.INDEX_ID)) { - FileBasedIndexImpl.LOG.info("Can't build stub using stub file element type " + file.getName() + - ", properties: " + PushedFilePropertiesRetriever.getInstance().dumpSortedPushedProperties(file)); + fileBasedIndex.getLogger().info("Can't build stub using stub file element type " + file.getName() + + ", properties: " + PushedFilePropertiesRetriever.getInstance().dumpSortedPushedProperties(file)); } } final BinaryFileStubBuilder builder = BinaryFileStubBuilders.INSTANCE.forFileType(fileType); @@ -278,8 +280,6 @@ private void assertPrebuiltStubTreeMatchesActualTree(@NotNull SerializedStubTree } } - private static final byte IS_BINARY_MASK = 1; - private static final byte BYTE_AND_CHAR_LENGTHS_ARE_THE_SAME_MASK = 1 << 1; @NotNull static IndexingStampInfo calculateIndexingStamp(@NotNull FileContent content) { VirtualFile file = content.getFile(); @@ -290,63 +290,11 @@ static IndexingStampInfo calculateIndexingStamp(@NotNull FileContent content) { return new IndexingStampInfo(file.getTimeStamp(), byteLength, contentLength, isBinary); } - static void saveIndexingStampInfo(@Nullable IndexingStampInfo indexingStampInfo, int fileId) { - try (DataOutputStream stream = FSRecords.writeAttribute(fileId, INDEXED_STAMP)) { - if (indexingStampInfo == null) return; - DataInputOutputUtil.writeTIME(stream, indexingStampInfo.indexingFileStamp); - DataInputOutputUtil.writeLONG(stream, indexingStampInfo.indexingByteLength); - - boolean lengthsAreTheSame = indexingStampInfo.indexingCharLength == indexingStampInfo.indexingByteLength; - byte flags = 0; - flags = BitUtil.set(flags, IS_BINARY_MASK, indexingStampInfo.isBinary); - flags = BitUtil.set(flags, BYTE_AND_CHAR_LENGTHS_ARE_THE_SAME_MASK, lengthsAreTheSame); - stream.writeByte(flags); - - if (!lengthsAreTheSame && !indexingStampInfo.isBinary) { - DataInputOutputUtil.writeINT(stream, indexingStampInfo.indexingCharLength); - } - } - catch (IOException e) { - LOG.error(e); - } - } - @Override public int getCacheSize() { return super.getCacheSize() * Runtime.getRuntime().availableProcessors(); } - @Nullable - static IndexingStampInfo readSavedIndexingStampInfo(@NotNull VirtualFile file) { - try (DataInputStream stream = INDEXED_STAMP.readAttribute(file)) { - if (stream == null || stream.available() <= 0) { - return null; - } - long stamp = DataInputOutputUtil.readTIME(stream); - long byteLength = DataInputOutputUtil.readLONG(stream); - - byte flags = stream.readByte(); - boolean isBinary = BitUtil.isSet(flags, IS_BINARY_MASK); - boolean readOnlyOneLength = BitUtil.isSet(flags, BYTE_AND_CHAR_LENGTHS_ARE_THE_SAME_MASK); - - int charLength; - if (isBinary) { - charLength = -1; - } - else if (readOnlyOneLength) { - charLength = (int)byteLength; - } - else { - charLength = DataInputOutputUtil.readINT(stream); - } - return new IndexingStampInfo(stamp, byteLength, charLength, isBinary); - } - catch (IOException e) { - LOG.error(e); - return null; - } - } - @NotNull @Override public DataExternalizer getValueExternalizer() { @@ -377,7 +325,7 @@ public boolean enableWal() { @Override public void handleInitializationError(@NotNull Throwable e) { - ((StubIndexImpl)StubIndex.getInstance()).initializationFailed(e); + ((StubIndexEx)StubIndex.getInstance()).initializationFailed(e); } @NotNull @@ -385,11 +333,11 @@ public void handleInitializationError(@NotNull Throwable e) { public UpdatableIndex createIndexImplementation(@NotNull final FileBasedIndexExtension extension, @NotNull VfsAwareIndexStorageLayout layout) throws StorageException, IOException { - ((StubIndexImpl)StubIndex.getInstance()).initializeStubIndexes(); + ((StubIndexEx)StubIndex.getInstance()).initializeStubIndexes(); checkNameStorage(); mySerializationManager.initialize(); - StubUpdatingIndexStorage index = new StubUpdatingIndexStorage(extension, new VfsAwareIndexStorageLayout<>() { + MapReduceIndexBase index = StubUpdatableIndexFactory.getInstance().createIndex(extension, new VfsAwareIndexStorageLayout<>() { @Override public void clearIndexData() { layout.clearIndexData(); @@ -416,12 +364,12 @@ public void clearIndexData() { memStorage.addBufferingStateListener(new TransientChangesIndexStorage.BufferingStateListener() { @Override public void bufferingStateChanged(final boolean newState) { - ((StubIndexImpl)StubIndex.getInstance()).setDataBufferingEnabled(newState); + ((StubIndexEx)StubIndex.getInstance()).setDataBufferingEnabled(newState); } @Override public void memoryStorageCleared() { - ((StubIndexImpl)StubIndex.getInstance()).cleanupMemoryStorage(); + ((StubIndexEx)StubIndex.getInstance()).cleanupMemoryStorage(); } }); } diff --git a/platform/indexing-impl/src/com/intellij/util/indexing/FileBasedIndexEx.java b/platform/indexing-impl/src/com/intellij/util/indexing/FileBasedIndexEx.java index 6870b6ef02958..90844291cf012 100644 --- a/platform/indexing-impl/src/com/intellij/util/indexing/FileBasedIndexEx.java +++ b/platform/indexing-impl/src/com/intellij/util/indexing/FileBasedIndexEx.java @@ -1,4 +1,4 @@ -// Copyright 2000-2021 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file. +// Copyright 2000-2021 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file. package com.intellij.util.indexing; import com.intellij.ide.lightEdit.LightEditCompatible; @@ -26,6 +26,7 @@ import com.intellij.psi.search.EverythingGlobalScope; import com.intellij.psi.search.GlobalSearchScope; import com.intellij.psi.search.impl.VirtualFileEnumeration; +import com.intellij.psi.stubs.StubUpdatingIndex; import com.intellij.psi.util.PsiModificationTracker; import com.intellij.util.*; import com.intellij.util.containers.ContainerUtil; @@ -51,11 +52,37 @@ @ApiStatus.Internal public abstract class FileBasedIndexEx extends FileBasedIndex { + public static final boolean DO_TRACE_STUB_INDEX_UPDATE = Boolean.getBoolean("idea.trace.stub.index.update"); @SuppressWarnings("SSBasedInspection") private static final ThreadLocal> ourDumbModeAccessTypeStack = ThreadLocal.withInitial(() -> new com.intellij.util.containers.Stack<>()); private static final RecursionGuard ourIgnoranceGuard = RecursionManager.createGuard("ignoreDumbMode"); private final IndexAccessValidator myAccessValidator = new IndexAccessValidator(); + private volatile boolean myTraceIndexUpdates; + private volatile boolean myTraceStubIndexUpdates; + private volatile boolean myTraceSharedIndexUpdates; + + @ApiStatus.Internal + boolean doTraceIndexUpdates() { + return myTraceIndexUpdates; + } + + @ApiStatus.Internal + public boolean doTraceStubUpdates(@NotNull ID indexId) { + return myTraceStubIndexUpdates && indexId.equals(StubUpdatingIndex.INDEX_ID); + } + + @ApiStatus.Internal + boolean doTraceSharedIndexUpdates() { + return myTraceSharedIndexUpdates; + } + + @ApiStatus.Internal + public void loadIndexes() { + myTraceIndexUpdates = SystemProperties.getBooleanProperty("trace.file.based.index.update", false); + myTraceStubIndexUpdates = SystemProperties.getBooleanProperty("trace.stub.index.update", false); + myTraceSharedIndexUpdates = SystemProperties.getBooleanProperty("trace.shared.index.update", false); + } @ApiStatus.Internal @NotNull @@ -709,8 +736,6 @@ public static boolean isTooLarge(@NotNull VirtualFile file, return false; } - public static final boolean DO_TRACE_STUB_INDEX_UPDATE = Boolean.getBoolean("idea.trace.stub.index.update"); - public static boolean acceptsInput(@NotNull InputFilter filter, @NotNull IndexedFile indexedFile) { if (filter instanceof ProjectSpecificInputFilter) { if (indexedFile.getProject() == null) { @@ -733,4 +758,13 @@ public boolean acceptInput(@NotNull IndexedFile file) { } : file -> filter.acceptInput(file) && condition.test(file); } + + @ApiStatus.Internal + public void runCleanupAction(@NotNull Runnable cleanupAction) { + } + + @ApiStatus.Internal + public static T disableUpToDateCheckIn(@NotNull ThrowableComputable runnable) throws E { + return IndexUpToDateCheckIn.disableUpToDateCheckIn(runnable); + } } diff --git a/platform/lang-impl/src/com/intellij/util/indexing/IndexInfrastructure.java b/platform/indexing-impl/src/com/intellij/util/indexing/IndexInfrastructure.java similarity index 100% rename from platform/lang-impl/src/com/intellij/util/indexing/IndexInfrastructure.java rename to platform/indexing-impl/src/com/intellij/util/indexing/IndexInfrastructure.java diff --git a/platform/lang-impl/src/com/intellij/util/indexing/IndexUpToDateCheckIn.kt b/platform/indexing-impl/src/com/intellij/util/indexing/IndexUpToDateCheckIn.kt similarity index 92% rename from platform/lang-impl/src/com/intellij/util/indexing/IndexUpToDateCheckIn.kt rename to platform/indexing-impl/src/com/intellij/util/indexing/IndexUpToDateCheckIn.kt index 0c20893cf0dd7..c326fd223b7c6 100644 --- a/platform/lang-impl/src/com/intellij/util/indexing/IndexUpToDateCheckIn.kt +++ b/platform/indexing-impl/src/com/intellij/util/indexing/IndexUpToDateCheckIn.kt @@ -2,8 +2,10 @@ package com.intellij.util.indexing import com.intellij.openapi.util.ThrowableComputable +import org.jetbrains.annotations.ApiStatus -internal object IndexUpToDateCheckIn { +@ApiStatus.Internal +object IndexUpToDateCheckIn { private val upToDateCheckState = ThreadLocal() @JvmStatic diff --git a/platform/lang-impl/src/com/intellij/util/indexing/impl/storage/TransientChangeTrackingValueContainer.java b/platform/indexing-impl/src/com/intellij/util/indexing/impl/storage/TransientChangeTrackingValueContainer.java similarity index 100% rename from platform/lang-impl/src/com/intellij/util/indexing/impl/storage/TransientChangeTrackingValueContainer.java rename to platform/indexing-impl/src/com/intellij/util/indexing/impl/storage/TransientChangeTrackingValueContainer.java diff --git a/platform/lang-impl/src/com/intellij/util/indexing/impl/storage/TransientChangesIndexStorage.java b/platform/indexing-impl/src/com/intellij/util/indexing/impl/storage/TransientChangesIndexStorage.java similarity index 96% rename from platform/lang-impl/src/com/intellij/util/indexing/impl/storage/TransientChangesIndexStorage.java rename to platform/indexing-impl/src/com/intellij/util/indexing/impl/storage/TransientChangesIndexStorage.java index c0ab7abda8da0..5a14f685455ca 100644 --- a/platform/lang-impl/src/com/intellij/util/indexing/impl/storage/TransientChangesIndexStorage.java +++ b/platform/indexing-impl/src/com/intellij/util/indexing/impl/storage/TransientChangesIndexStorage.java @@ -1,4 +1,4 @@ -// Copyright 2000-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file. +// Copyright 2000-2022 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. package com.intellij.util.indexing.impl.storage; @@ -90,7 +90,7 @@ public void clearCaches() { if (IndexDebugProperties.DEBUG) { String message = "Dropping caches for " + myIndexId + ", number of items:" + myMap.size(); - FileBasedIndexImpl.LOG.info(message); + ((FileBasedIndexEx)FileBasedIndex.getInstance()).getLogger().info(message); } for (ChangeTrackingValueContainer v : myMap.values()) { diff --git a/platform/indexing-impl/src/com/intellij/util/indexing/storage/MapReduceIndexBase.java b/platform/indexing-impl/src/com/intellij/util/indexing/storage/MapReduceIndexBase.java new file mode 100644 index 0000000000000..29e57a7d2da3b --- /dev/null +++ b/platform/indexing-impl/src/com/intellij/util/indexing/storage/MapReduceIndexBase.java @@ -0,0 +1,146 @@ +// Copyright 2000-2022 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. +package com.intellij.util.indexing.storage; + +import com.intellij.openapi.diagnostic.Logger; +import com.intellij.openapi.progress.ProcessCanceledException; +import com.intellij.openapi.progress.ProgressManager; +import com.intellij.openapi.util.Ref; +import com.intellij.openapi.util.ThrowableComputable; +import com.intellij.openapi.util.io.ByteArraySequence; +import com.intellij.psi.search.GlobalSearchScope; +import com.intellij.util.ConcurrencyUtil; +import com.intellij.util.Processor; +import com.intellij.util.containers.ContainerUtil; +import com.intellij.util.indexing.*; +import com.intellij.util.indexing.impl.AbstractUpdateData; +import com.intellij.util.indexing.impl.IndexStorage; +import com.intellij.util.indexing.impl.InputDataDiffBuilder; +import com.intellij.util.indexing.impl.MapReduceIndex; +import com.intellij.util.indexing.impl.forward.AbstractMapForwardIndexAccessor; +import com.intellij.util.indexing.impl.forward.ForwardIndex; +import com.intellij.util.indexing.impl.forward.ForwardIndexAccessor; +import org.jetbrains.annotations.ApiStatus; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +import java.io.IOException; +import java.util.Collections; +import java.util.Map; +import java.util.concurrent.locks.ReadWriteLock; + +@ApiStatus.Experimental +@ApiStatus.Internal +public abstract class MapReduceIndexBase extends MapReduceIndex + implements UpdatableIndex { + private final boolean mySingleEntryIndex; + + protected MapReduceIndexBase(@NotNull IndexExtension extension, + @NotNull ThrowableComputable, ? extends IOException> storage, + @Nullable ThrowableComputable forwardIndex, + @Nullable ForwardIndexAccessor forwardIndexAccessor, + @Nullable ReadWriteLock lock) throws IOException { + super(extension, storage, forwardIndex, forwardIndexAccessor, lock); + if (!(myIndexId instanceof ID)) { + throw new IllegalArgumentException("myIndexId should be instance of com.intellij.util.indexing.ID"); + } + mySingleEntryIndex = extension instanceof SingleEntryFileBasedIndexExtension; + } + + @Override + public boolean processAllKeys(@NotNull Processor processor, @NotNull GlobalSearchScope scope, @Nullable IdFilter idFilter) throws StorageException { + return ConcurrencyUtil.withLock(getLock().readLock(), () -> + ((VfsAwareIndexStorage)myStorage).processKeys(processor, scope, idFilter) + ); + } + + @NotNull + @Override + public Map getIndexedFileData(int fileId) throws StorageException { + return ConcurrencyUtil.withLock(getLock().readLock(), () -> { + try { + return Collections.unmodifiableMap(ContainerUtil.notNullize(getNullableIndexedData(fileId))); + } + catch (IOException e) { + throw new StorageException(e); + } + }); + } + + @Nullable + protected Map getNullableIndexedData(int fileId) throws IOException, StorageException { + if (isDisposed()) { + return null; + } + // in future we will get rid of forward index for SingleEntryFileBasedIndexExtension + if (mySingleEntryIndex) { + @SuppressWarnings("unchecked") + Key key = (Key)(Object)fileId; + Ref> result = new Ref<>(Collections.emptyMap()); + ValueContainer container = getData(key); + container.forEach((id, value) -> { + boolean acceptNullValues = ((SingleEntryIndexer)myIndexer).isAcceptNullValues(); + if (value != null || acceptNullValues) { + result.set(Collections.singletonMap(key, value)); + } + return false; + }); + return result.get(); + } + if (getForwardIndexAccessor() instanceof AbstractMapForwardIndexAccessor) { + ByteArraySequence serializedInputData = getForwardIndex().get(fileId); + AbstractMapForwardIndexAccessor forwardIndexAccessor = (AbstractMapForwardIndexAccessor)getForwardIndexAccessor(); + return forwardIndexAccessor.convertToInputDataMap(fileId, serializedInputData); + } + getLogger().error("Can't fetch indexed data for index " + myIndexId.getName()); + return null; + } + + @Override + public void checkCanceled() { + ProgressManager.checkCanceled(); + } + + @Override + public void updateWithMap(@NotNull AbstractUpdateData updateData) throws StorageException { + try { + super.updateWithMap(updateData); + } + catch (ProcessCanceledException e) { + getLogger().error("ProcessCancelledException is not expected here!", e); + throw e; + } + } + + @Override + public void setBufferingEnabled(boolean enabled) { + // TODO to be removed + throw new UnsupportedOperationException(); + } + + @Override + public void cleanupMemoryStorage() { + // TODO to be removed + throw new UnsupportedOperationException(); + } + + @Override + public void cleanupForNextTest() { + // TODO to be removed + throw new UnsupportedOperationException(); + } + + @Override + public void removeTransientDataForFile(int inputId) { + // TODO to be removed + throw new UnsupportedOperationException(); + } + + @Override + public void removeTransientDataForKeys(int inputId, + @NotNull InputDataDiffBuilder diffBuilder) { + // TODO to be removed + throw new UnsupportedOperationException(); + } + + protected abstract Logger getLogger(); +} diff --git a/platform/lang-impl/src/com/intellij/psi/stubs/StubIndexImpl.java b/platform/lang-impl/src/com/intellij/psi/stubs/StubIndexImpl.java index 19f349fbbe947..4ed84c19f190c 100644 --- a/platform/lang-impl/src/com/intellij/psi/stubs/StubIndexImpl.java +++ b/platform/lang-impl/src/com/intellij/psi/stubs/StubIndexImpl.java @@ -63,15 +63,6 @@ private static final class AsyncState { private final Map, UpdatableIndex> myIndices = CollectionFactory.createSmallMemoryFootprintMap(); } - private final Map, CachedValue, StubIdList>>> myCachedStubIds = FactoryMap.createMap(k -> { - UpdatableIndex index = getStubUpdatingIndex(); - ModificationTracker tracker = index::getModificationStamp; - return new CachedValueImpl<>(() -> new CachedValueProvider.Result<>(new ConcurrentHashMap<>(), tracker)); - }, ConcurrentHashMap::new); - - private final StubProcessingHelper myStubProcessingHelper = new StubProcessingHelper(); - private final IndexAccessValidator myAccessValidator = new IndexAccessValidator(); - private final AtomicBoolean myForcedClean = new AtomicBoolean(); private volatile CompletableFuture myStateFuture; private volatile AsyncState myState; @@ -114,6 +105,7 @@ public void waitUntilStubIndexedInitialized() { getAsyncState(); } + @Override public void initializationFailed(@NotNull Throwable error) { myStateFuture = new CompletableFuture<>(); myStateFuture.completeExceptionally(error); @@ -220,111 +212,6 @@ public void flush() throws StorageException { } } - - @Override - public boolean processElements(@NotNull StubIndexKey indexKey, - @NotNull Key key, - @NotNull Project project, - @Nullable GlobalSearchScope scope, - @Nullable IdFilter idFilter, - @NotNull Class requiredClass, - @NotNull Processor processor) { - boolean dumb = DumbService.isDumb(project); - if (dumb) { - if (LightEdit.owns(project)) return false; - DumbModeAccessType accessType = FileBasedIndex.getInstance().getCurrentDumbModeAccessType(); - if (accessType == DumbModeAccessType.RAW_INDEX_DATA_ACCEPTABLE) { - throw new AssertionError("raw index data access is not available for StubIndex"); - } - } - - PairProcessor stubProcessor = (file, list) -> - myStubProcessingHelper.processStubsInFile(project, file, list, processor, scope, requiredClass); - - if (!ModelBranchImpl.processModifiedFilesInScope(scope != null ? scope : GlobalSearchScope.everythingScope(project), - file -> processInMemoryStubs(indexKey, key, project, stubProcessor, file))) { - return false; - } - - VirtualFile singleFileInScope = extractSingleFile(scope); - Iterator fileStream; - boolean shouldHaveKeys; - - if (singleFileInScope != null) { - if (!(singleFileInScope instanceof VirtualFileWithId)) return true; - FileBasedIndex.getInstance().ensureUpToDate(StubUpdatingIndex.INDEX_ID, project, scope); - fileStream = ObjectIterators.singleton(singleFileInScope); - shouldHaveKeys = false; - } - else { - IntSet ids = getContainingIds(indexKey, key, project, idFilter, scope); - if (ids == null) return true; - IntPredicate accessibleFileFilter = ((FileBasedIndexEx)FileBasedIndex.getInstance()).getAccessibleFileIdFilter(project); - // already ensured up-to-date in getContainingIds() method - IntIterator idIterator = ids.iterator(); - fileStream = StubIndexImplUtil.mapIdIterator(idIterator, accessibleFileFilter); - shouldHaveKeys = true; - } - - try { - while (fileStream.hasNext()) { - VirtualFile file = fileStream.next(); - assert file != null; - - List filesInScope = scope != null ? FileBasedIndexEx.filesInScopeWithBranches(scope, file) : Collections.singletonList(file); - if (filesInScope.isEmpty()) { - continue; - } - - int id = ((VirtualFileWithId)file).getId(); - StubIdList list = myCachedStubIds.get(indexKey).getValue().computeIfAbsent(new KeyAndFileId<>(key, id), __ -> - myStubProcessingHelper.retrieveStubIdList(indexKey, key, file, project, shouldHaveKeys) - ); - if (list == null) { - // stub index inconsistency - continue; - } - for (VirtualFile eachFile : filesInScope) { - if (!stubProcessor.process(eachFile, list)) { - return false; - } - } - } - } - catch (RuntimeException e) { - final Throwable cause = FileBasedIndexEx.getCauseToRebuildIndex(e); - if (cause != null) { - forceRebuild(cause); - } - else { - throw e; - } - } finally { - wipeProblematicFileIdsForParticularKeyAndStubIndex(indexKey, key); - } - return true; - } - - private static boolean processInMemoryStubs(StubIndexKey indexKey, - Key key, - Project project, - PairProcessor stubProcessor, - VirtualFile file) { - Map data = FileBasedIndex.getInstance().getFileData(StubUpdatingIndex.INDEX_ID, file, project); - if (data.size() == 1) { - try { - StubIdList list = data.values().iterator().next().restoreIndexedStubs(indexKey, key); - if (list != null) { - return stubProcessor.process(file, list); - } - } - catch (IOException e) { - throw new RuntimeException(e); - } - } - return true; - } - @ApiStatus.Internal @Override @SuppressWarnings("unchecked") @@ -332,176 +219,12 @@ protected UpdatableIndex getIndex(@NotNull StubInd return (UpdatableIndex)getAsyncState().myIndices.get(indexKey); } - // Self repair for IDEA-181227, caused by (yet) unknown file event processing problem in indices - // FileBasedIndex.requestReindex doesn't handle the situation properly because update requires old data that was lost - private void wipeProblematicFileIdsForParticularKeyAndStubIndex(@NotNull StubIndexKey indexKey, - @NotNull Key key) { - Set filesWithProblems = myStubProcessingHelper.takeAccumulatedFilesWithIndexProblems(); - - if (filesWithProblems != null) { - LOG.info("data for " + indexKey.getName() + " will be wiped for a some files because of internal stub processing error"); - ((FileBasedIndexImpl)FileBasedIndex.getInstance()).runCleanupAction(() -> { - Lock writeLock = getIndex(indexKey).getLock().writeLock(); - boolean locked = writeLock.tryLock(); - if (!locked) return; // nested indices invocation, can not cleanup without deadlock - try { - for (VirtualFile file : filesWithProblems) { - updateIndex(indexKey, - FileBasedIndex.getFileId(file), - Collections.singleton(key), - Collections.emptySet()); - } - } - finally { - writeLock.unlock(); - } - }); - } - } - @Override public void forceRebuild(@NotNull Throwable e) { FileBasedIndex.getInstance().scheduleRebuild(StubUpdatingIndex.INDEX_ID, e); } @Override - public @NotNull Collection getAllKeys(@SuppressWarnings("BoundedWildcard") @NotNull StubIndexKey indexKey, @NotNull Project project) { - Set allKeys = new HashSet<>(); - processAllKeys(indexKey, project, Processors.cancelableCollectProcessor(allKeys)); - return allKeys; - } - - @Override - public boolean processAllKeys(@NotNull StubIndexKey indexKey, - @NotNull Processor processor, - @NotNull GlobalSearchScope scope, - @Nullable IdFilter idFilter) { - final UpdatableIndex index = getIndex(indexKey); // wait for initialization to finish - FileBasedIndexEx fileBasedIndexEx = (FileBasedIndexEx)FileBasedIndex.getInstance(); - if (index == null || - !fileBasedIndexEx.ensureUpToDate(StubUpdatingIndex.INDEX_ID, scope.getProject(), scope, null)) { - return true; - } - - if (idFilter == null) { - idFilter = fileBasedIndexEx.extractIdFilter(scope, scope.getProject()); - } - - try { - @Nullable IdFilter finalIdFilter = idFilter; - return myAccessValidator.validate(StubUpdatingIndex.INDEX_ID, ()->FileBasedIndexImpl.disableUpToDateCheckIn(()-> - index.processAllKeys(processor, scope, finalIdFilter))); - } - catch (StorageException e) { - forceRebuild(e); - } - catch (RuntimeException e) { - final Throwable cause = e.getCause(); - if (cause instanceof IOException || cause instanceof StorageException) { - forceRebuild(e); - } - throw e; - } - return true; - } - - @Override - public @NotNull IdIterator getContainingIds(@NotNull StubIndexKey indexKey, - @NotNull Key dataKey, - final @NotNull Project project, - final @Nullable GlobalSearchScope scope) { - IntSet result = getContainingIds(indexKey, dataKey, project, null, scope); - if (result == null) return IdIterator.EMPTY; - return new IdIterator() { - final IntIterator iterator = result.iterator(); - - @Override - public boolean hasNext() { - return iterator.hasNext(); - } - - @Override - public int next() { - return iterator.nextInt(); - } - - @Override - public int size() { - return result.size(); - } - }; - } - - @Override - public @NotNull Iterator getContainingFiles(@NotNull StubIndexKey indexKey, - @NotNull Key dataKey, - @NotNull Project project, - @NotNull GlobalSearchScope scope) { - IntSet result = getContainingIds(indexKey, dataKey, project, null, scope); - Set fileSet = new CompactVirtualFileSet(result == null ? ArrayUtil.EMPTY_INT_ARRAY : result.toIntArray()).freeze(); - return Iterators.filter(fileSet.iterator(), scope::contains); - } - - @Override - public int getMaxContainingFileCount(@NotNull StubIndexKey indexKey, - @NotNull Key dataKey, - @NotNull Project project, - @NotNull GlobalSearchScope scope) { - IntSet result = getContainingIds(indexKey, dataKey, project, null, scope); - return result == null ? 0 : result.size(); - } - - private @Nullable IntSet getContainingIds(@NotNull StubIndexKey indexKey, - @NotNull Key dataKey, - final @NotNull Project project, - @Nullable IdFilter idFilter, - final @Nullable GlobalSearchScope scope) { - final FileBasedIndexEx fileBasedIndex = (FileBasedIndexEx)FileBasedIndex.getInstance(); - ID stubUpdatingIndexId = StubUpdatingIndex.INDEX_ID; - final UpdatableIndex index = getIndex(indexKey); // wait for initialization to finish - if (index == null || !fileBasedIndex.ensureUpToDate(stubUpdatingIndexId, project, scope, null)) return null; - - IdFilter finalIdFilter = idFilter != null - ? idFilter - : ((FileBasedIndexEx)FileBasedIndex.getInstance()).extractIdFilter(scope, project); - - UpdatableIndex stubUpdatingIndex = fileBasedIndex.getIndex(stubUpdatingIndexId); - - try { - IntSet result = new IntLinkedOpenHashSet(); // workaround duplicates keys - myAccessValidator.validate(stubUpdatingIndexId, ()-> { - // disable up-to-date check to avoid locks on attempt to acquire index write lock while holding at the same time the readLock for this index - //noinspection Convert2Lambda (workaround for JBR crash, JBR-2349),Convert2Diamond - return FileBasedIndexImpl.disableUpToDateCheckIn(() -> ConcurrencyUtil.withLock(stubUpdatingIndex.getLock().readLock(), () -> - index.getData(dataKey).forEach(new ValueContainer.ContainerAction<>() { - @Override - public boolean perform(int id, Void value) { - if (finalIdFilter == null || finalIdFilter.containsFileId(id)) { - result.add(id); - } - return true; - } - }) - )); - }); - return result; - } - catch (StorageException e) { - forceRebuild(e); - } - catch (RuntimeException e) { - final Throwable cause = FileBasedIndexEx.getCauseToRebuildIndex(e); - if (cause != null) { - forceRebuild(cause); - } - else { - throw e; - } - } - - return null; - } - void initializeStubIndexes() { assert !myInitialized; @@ -534,16 +257,16 @@ public void dispose() { } } - private void clearState() { - StubIndexKeyDescriptorCache.INSTANCE.clear(); - ((SerializationManagerImpl)SerializationManagerEx.getInstanceEx()).dropSerializerData(); - myCachedStubIds.clear(); + @Override + protected void clearState() { + super.clearState(); myStateFuture = null; myState = null; myInitialized = false; LOG.info("StubIndexExtension-s were unloaded"); } + @Override void setDataBufferingEnabled(final boolean enabled) { AsyncState state = ProgressManager.getInstance().computeInNonCancelableSection(this::getAsyncState); for (UpdatableIndex index : state.myIndices.values()) { @@ -551,6 +274,7 @@ void setDataBufferingEnabled(final boolean enabled) { } } + @Override void cleanupMemoryStorage() { UpdatableIndex stubUpdatingIndex = getStubUpdatingIndex(); stubUpdatingIndex.getLock().writeLock().lock(); @@ -683,55 +407,4 @@ protected String getInitializationFinishedMessage(AsyncState initializationResul return "Initialized stub indexes: " + initializationResult.myIndices.keySet() + "."; } } - - static UpdatableIndex getStubUpdatingIndex() { - return ((FileBasedIndexEx)FileBasedIndex.getInstance()).getIndex(StubUpdatingIndex.INDEX_ID); - } - - @SuppressWarnings("unchecked") - private static @Nullable VirtualFile extractSingleFile(@Nullable GlobalSearchScope scope) { - if (!(scope instanceof Iterable)) { - return null; - } - Iterable scopeAsFileIterable = (Iterable)scope; - VirtualFile result = null; - for (VirtualFile file : scopeAsFileIterable) { - if (result == null) { - result = ObjectUtils.notNull(file, NullVirtualFile.INSTANCE); - } - else { - return null; - } - } - return result; - } - - private static final class KeyAndFileId { - @NotNull - private final K key; - private final int fileId; - - private KeyAndFileId(@NotNull K key, int fileId) { - this.key = key; - this.fileId = fileId; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - KeyAndFileId key1 = (KeyAndFileId)o; - return fileId == key1.fileId && Objects.equals(key, key1.key); - } - - @Override - public int hashCode() { - return Objects.hash(key, fileId); - } - } - - @TestOnly - public boolean areAllProblemsProcessedInTheCurrentThread() { - return myStubProcessingHelper.areAllProblemsProcessedInTheCurrentThread(); - } } \ No newline at end of file diff --git a/platform/lang-impl/src/com/intellij/psi/stubs/StubTreeLoaderImpl.java b/platform/lang-impl/src/com/intellij/psi/stubs/StubTreeLoaderImpl.java index a113caff3ece1..07831c503f24a 100644 --- a/platform/lang-impl/src/com/intellij/psi/stubs/StubTreeLoaderImpl.java +++ b/platform/lang-impl/src/com/intellij/psi/stubs/StubTreeLoaderImpl.java @@ -13,16 +13,22 @@ import com.intellij.openapi.util.RecursionManager; import com.intellij.openapi.vfs.VirtualFile; import com.intellij.openapi.vfs.VirtualFileWithId; +import com.intellij.openapi.vfs.newvfs.FileAttribute; +import com.intellij.openapi.vfs.newvfs.persistent.FSRecords; import com.intellij.psi.PsiDocumentManager; import com.intellij.psi.PsiElement; import com.intellij.psi.PsiFile; import com.intellij.psi.impl.PsiManagerEx; import com.intellij.psi.impl.source.PsiFileImpl; +import com.intellij.util.BitUtil; import com.intellij.util.containers.ContainerUtil; import com.intellij.util.indexing.*; +import com.intellij.util.io.DataInputOutputUtil; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; +import java.io.DataInputStream; +import java.io.DataOutputStream; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; @@ -31,6 +37,10 @@ final class StubTreeLoaderImpl extends StubTreeLoader { private static final Logger LOG = Logger.getInstance(StubTreeLoaderImpl.class); + // todo remove once we don't need this for stub-ast mismatch debug info + private static final FileAttribute INDEXED_STAMP = new FileAttribute("stubIndexStamp", 3, true); + private static final byte IS_BINARY_MASK = 1; + private static final byte BYTE_AND_CHAR_LENGTHS_ARE_THE_SAME_MASK = 1 << 1; private static volatile boolean ourStubReloadingProhibited; @Override @@ -264,11 +274,63 @@ protected boolean hasPsiInManyProjects(@NotNull final VirtualFile virtualFile) { @Override protected IndexingStampInfo getIndexingStampInfo(@NotNull VirtualFile file) { - return StubUpdatingIndex.readSavedIndexingStampInfo(file); + return readSavedIndexingStampInfo(file); } @Override protected boolean isTooLarge(@NotNull VirtualFile file) { return ((FileBasedIndexImpl)FileBasedIndex.getInstance()).isTooLarge(file); } + + static void saveIndexingStampInfo(@Nullable IndexingStampInfo indexingStampInfo, int fileId) { + try (DataOutputStream stream = FSRecords.writeAttribute(fileId, INDEXED_STAMP)) { + if (indexingStampInfo == null) return; + DataInputOutputUtil.writeTIME(stream, indexingStampInfo.indexingFileStamp); + DataInputOutputUtil.writeLONG(stream, indexingStampInfo.indexingByteLength); + + boolean lengthsAreTheSame = indexingStampInfo.indexingCharLength == indexingStampInfo.indexingByteLength; + byte flags = 0; + flags = BitUtil.set(flags, IS_BINARY_MASK, indexingStampInfo.isBinary); + flags = BitUtil.set(flags, BYTE_AND_CHAR_LENGTHS_ARE_THE_SAME_MASK, lengthsAreTheSame); + stream.writeByte(flags); + + if (!lengthsAreTheSame && !indexingStampInfo.isBinary) { + DataInputOutputUtil.writeINT(stream, indexingStampInfo.indexingCharLength); + } + } + catch (IOException e) { + StubUpdatingIndex.LOG.error(e); + } + } + + @Nullable + static IndexingStampInfo readSavedIndexingStampInfo(@NotNull VirtualFile file) { + try (DataInputStream stream = INDEXED_STAMP.readAttribute(file)) { + if (stream == null || stream.available() <= 0) { + return null; + } + long stamp = DataInputOutputUtil.readTIME(stream); + long byteLength = DataInputOutputUtil.readLONG(stream); + + byte flags = stream.readByte(); + boolean isBinary = BitUtil.isSet(flags, IS_BINARY_MASK); + boolean readOnlyOneLength = BitUtil.isSet(flags, BYTE_AND_CHAR_LENGTHS_ARE_THE_SAME_MASK); + + int charLength; + if (isBinary) { + charLength = -1; + } + else if (readOnlyOneLength) { + charLength = (int)byteLength; + } + else { + charLength = DataInputOutputUtil.readINT(stream); + } + return new IndexingStampInfo(stamp, byteLength, charLength, isBinary); + } + catch (IOException e) { + StubUpdatingIndex.LOG.error(e); + return null; + } + } } diff --git a/platform/lang-impl/src/com/intellij/psi/stubs/StubUpdatableIndexFactoryImpl.java b/platform/lang-impl/src/com/intellij/psi/stubs/StubUpdatableIndexFactoryImpl.java new file mode 100644 index 0000000000000..76cc28f798711 --- /dev/null +++ b/platform/lang-impl/src/com/intellij/psi/stubs/StubUpdatableIndexFactoryImpl.java @@ -0,0 +1,22 @@ +// Copyright 2000-2022 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. +package com.intellij.psi.stubs; + +import com.intellij.util.indexing.FileBasedIndexExtension; +import com.intellij.util.indexing.storage.MapReduceIndexBase; +import com.intellij.util.indexing.storage.VfsAwareIndexStorageLayout; +import org.jetbrains.annotations.ApiStatus; +import org.jetbrains.annotations.NotNull; + +import java.io.IOException; + +@ApiStatus.Internal +public class StubUpdatableIndexFactoryImpl extends StubUpdatableIndexFactory { + @NotNull + @Override + public MapReduceIndexBase createIndex(@NotNull FileBasedIndexExtension extension, + @NotNull VfsAwareIndexStorageLayout layout, + @NotNull SerializationManagerEx serializationManager) + throws IOException { + return new StubUpdatingIndexStorage(extension, layout, serializationManager); + } +} diff --git a/platform/lang-impl/src/com/intellij/psi/stubs/StubUpdatingIndexStorage.java b/platform/lang-impl/src/com/intellij/psi/stubs/StubUpdatingIndexStorage.java index 4fa6ea658d71c..0bc8c5076d7c1 100644 --- a/platform/lang-impl/src/com/intellij/psi/stubs/StubUpdatingIndexStorage.java +++ b/platform/lang-impl/src/com/intellij/psi/stubs/StubUpdatingIndexStorage.java @@ -64,7 +64,7 @@ private StubIndexImpl getStubIndex() { try { Boolean result = indexUpdateComputable.compute(); if (Boolean.TRUE.equals(result) && !StaleIndexesChecker.isStaleIdDeletion()) { - StubUpdatingIndex.saveIndexingStampInfo(indexingStampInfo, inputId); + StubTreeLoaderImpl.saveIndexingStampInfo(indexingStampInfo, inputId); } return result; } diff --git a/platform/lang-impl/src/com/intellij/util/indexing/FileBasedIndexImpl.java b/platform/lang-impl/src/com/intellij/util/indexing/FileBasedIndexImpl.java index 1eeae8fa93cf9..c625ef809ce6d 100644 --- a/platform/lang-impl/src/com/intellij/util/indexing/FileBasedIndexImpl.java +++ b/platform/lang-impl/src/com/intellij/util/indexing/FileBasedIndexImpl.java @@ -37,10 +37,7 @@ import com.intellij.openapi.vfs.newvfs.NewVirtualFile; import com.intellij.openapi.vfs.newvfs.impl.VirtualFileSystemEntry; import com.intellij.openapi.vfs.newvfs.persistent.PersistentFS; -import com.intellij.psi.PsiBinaryFile; -import com.intellij.psi.PsiDocumentManager; -import com.intellij.psi.PsiFile; -import com.intellij.psi.PsiManager; +import com.intellij.psi.*; import com.intellij.psi.impl.PsiDocumentTransactionListener; import com.intellij.psi.impl.cache.impl.id.PlatformIdTableBuilding; import com.intellij.psi.impl.source.PsiFileImpl; @@ -82,8 +79,7 @@ import com.intellij.util.io.storage.HeavyProcessLatch; import com.intellij.util.messages.MessageBus; import com.intellij.util.messages.SimpleMessageBusConnection; -import it.unimi.dsi.fastutil.ints.IntOpenHashSet; -import it.unimi.dsi.fastutil.ints.IntSet; +import it.unimi.dsi.fastutil.ints.*; import org.jetbrains.annotations.ApiStatus; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; @@ -114,9 +110,6 @@ public final class FileBasedIndexImpl extends FileBasedIndexEx { @ApiStatus.Internal public static final Logger LOG = Logger.getInstance(FileBasedIndexImpl.class); - private volatile boolean myTraceIndexUpdates; - private volatile boolean myTraceStubIndexUpdates; - private volatile boolean myTraceSharedIndexUpdates; private volatile RegisteredIndexes myRegisteredIndexes; private volatile @Nullable String myShutdownReason; @@ -230,20 +223,6 @@ public void extensionRemoved(@NotNull FileBasedIndexExtension extension, @ myIndexableFilesFilterHolder = new IncrementalProjectIndexableFilesFilterHolder(); } - boolean doTraceIndexUpdates() { - return myTraceIndexUpdates; - } - - @ApiStatus.Internal - public boolean doTraceStubUpdates(@NotNull ID indexId) { - return myTraceStubIndexUpdates && indexId.equals(StubUpdatingIndex.INDEX_ID); - } - - @ApiStatus.Internal - boolean doTraceSharedIndexUpdates() { - return myTraceSharedIndexUpdates; - } - void scheduleFullIndexesRescan(@NotNull Collection> indexesToRebuild, @NotNull String reason) { cleanupProcessedFlag(); doClearIndices(id -> indexesToRebuild.contains(id)); @@ -419,11 +398,10 @@ public void requestReindex(@NotNull VirtualFile file, boolean forceRebuild) { } } + @Override public synchronized void loadIndexes() { if (myRegisteredIndexes == null) { - myTraceIndexUpdates = SystemProperties.getBooleanProperty("trace.file.based.index.update", false); - myTraceStubIndexUpdates = SystemProperties.getBooleanProperty("trace.stub.index.update", false); - myTraceSharedIndexUpdates = SystemProperties.getBooleanProperty("trace.shared.index.update", false); + super.loadIndexes(); LOG.assertTrue(myRegisteredIndexes == null); myStorageBufferingHandler.resetState(); @@ -773,10 +751,6 @@ private void flushAllIndices(final long modCount) { SnapshotHashEnumeratorService.getInstance().flush(); } - public static T disableUpToDateCheckIn(@NotNull ThrowableComputable runnable) throws E { - return IndexUpToDateCheckIn.disableUpToDateCheckIn(runnable); - } - private final ThreadLocal myReentrancyGuard = ThreadLocal.withInitial(() -> Boolean.FALSE); @Override @@ -1214,6 +1188,7 @@ private Map getInMemoryData(@NotNull ID id, @NotNull VirtualF }; @ApiStatus.Internal + @Override public void runCleanupAction(@NotNull Runnable cleanupAction) { Computable updateComputable = () -> { ProgressManager.getInstance().executeNonCancelableSection(cleanupAction); diff --git a/platform/lang-impl/src/com/intellij/util/indexing/impl/storage/VfsAwareMapReduceIndex.java b/platform/lang-impl/src/com/intellij/util/indexing/impl/storage/VfsAwareMapReduceIndex.java index 538a37db364ac..689360dec38d3 100644 --- a/platform/lang-impl/src/com/intellij/util/indexing/impl/storage/VfsAwareMapReduceIndex.java +++ b/platform/lang-impl/src/com/intellij/util/indexing/impl/storage/VfsAwareMapReduceIndex.java @@ -8,15 +8,8 @@ import com.intellij.openapi.application.ex.ApplicationManagerEx; import com.intellij.openapi.diagnostic.Logger; import com.intellij.openapi.extensions.PluginId; -import com.intellij.openapi.progress.ProcessCanceledException; import com.intellij.openapi.progress.ProgressManager; -import com.intellij.openapi.util.Ref; import com.intellij.openapi.util.ThrowableComputable; -import com.intellij.openapi.util.io.ByteArraySequence; -import com.intellij.psi.search.GlobalSearchScope; -import com.intellij.util.ConcurrencyUtil; -import com.intellij.util.Processor; -import com.intellij.util.containers.ContainerUtil; import com.intellij.util.indexing.*; import com.intellij.util.indexing.impl.*; import com.intellij.util.indexing.impl.forward.AbstractMapForwardIndexAccessor; @@ -28,6 +21,7 @@ import com.intellij.util.indexing.snapshot.SnapshotInputMappings; import com.intellij.util.indexing.snapshot.SnapshotInputMappingsStatistics; import com.intellij.util.indexing.snapshot.SnapshotSingleValueIndexStorage; +import com.intellij.util.indexing.storage.MapReduceIndexBase; import com.intellij.util.indexing.storage.SnapshotInputMappingIndex; import com.intellij.util.indexing.storage.UpdatableSnapshotInputMappingIndex; import com.intellij.util.indexing.storage.VfsAwareIndexStorageLayout; @@ -37,14 +31,13 @@ import org.jetbrains.annotations.Nullable; import java.io.IOException; -import java.util.Collections; import java.util.Map; import java.util.concurrent.locks.ReadWriteLock; /** * @author Eugene Zhuravlev */ -public class VfsAwareMapReduceIndex extends MapReduceIndex implements UpdatableIndex{ +public class VfsAwareMapReduceIndex extends MapReduceIndexBase { private static final Logger LOG = Logger.getInstance(VfsAwareMapReduceIndex.class); @ApiStatus.Internal @@ -66,7 +59,6 @@ public class VfsAwareMapReduceIndex extends MapReduceIndex mySnapshotInputMappings; private final boolean myUpdateMappings; - private final boolean mySingleEntryIndex; public VfsAwareMapReduceIndex(@NotNull FileBasedIndexExtension extension, @NotNull VfsAwareIndexStorageLayout indexStorageLayout, @@ -86,9 +78,6 @@ protected VfsAwareMapReduceIndex(@NotNull FileBasedIndexExtension ex @Nullable ThrowableComputable, ? extends IOException> snapshotInputMappings, @Nullable ReadWriteLock lock) throws IOException { super(extension, storage, forwardIndexMap, forwardIndexAccessor, lock); - if (!(myIndexId instanceof ID)) { - throw new IllegalArgumentException("myIndexId should be instance of com.intellij.util.indexing.ID"); - } SnapshotInputMappingIndex inputMappings; try { inputMappings = snapshotInputMappings == null ? null : snapshotInputMappings.compute(); @@ -127,7 +116,6 @@ protected VfsAwareMapReduceIndex(@NotNull FileBasedIndexExtension ex } else { mySubIndexerRetriever = null; } - mySingleEntryIndex = extension instanceof SingleEntryFileBasedIndexExtension; } public void resetSnapshotInputMappingsStatistics() { @@ -143,6 +131,11 @@ public void resetSnapshotInputMappingsStatistics() { return null; } + @Override + protected Logger getLogger() { + return LOG; + } + public static boolean isCompositeIndexer(@NotNull DataIndexer indexer) { return indexer instanceof CompositeDataIndexer && !FileBasedIndex.USE_IN_MEMORY_INDEX; } @@ -267,61 +260,6 @@ protected FileIndexingState isIndexConfigurationUpToDate(int fileId, @NotNull In protected void setIndexConfigurationUpToDate(int fileId, @NotNull IndexedFile file) { } - - @Override - public boolean processAllKeys(@NotNull Processor processor, @NotNull GlobalSearchScope scope, @Nullable IdFilter idFilter) throws StorageException { - return ConcurrencyUtil.withLock(getLock().readLock(), () -> - ((VfsAwareIndexStorage)myStorage).processKeys(processor, scope, idFilter) - ); - } - - @NotNull - @Override - public Map getIndexedFileData(int fileId) throws StorageException { - return ConcurrencyUtil.withLock(getLock().readLock(), () -> { - try { - return Collections.unmodifiableMap(ContainerUtil.notNullize(getNullableIndexedData(fileId))); - } - catch (IOException e) { - throw new StorageException(e); - } - }); - } - - @Nullable - protected Map getNullableIndexedData(int fileId) throws IOException, StorageException { - if (isDisposed()) { - return null; - } - // in future we will get rid of forward index for SingleEntryFileBasedIndexExtension - if (mySingleEntryIndex) { - @SuppressWarnings("unchecked") - Key key = (Key)(Object)fileId; - Ref> result = new Ref<>(Collections.emptyMap()); - ValueContainer container = getData(key); - container.forEach((id, value) -> { - boolean acceptNullValues = ((SingleEntryIndexer)myIndexer).isAcceptNullValues(); - if (value != null || acceptNullValues) { - result.set(Collections.singletonMap(key, value)); - } - return false; - }); - return result.get(); - } - if (getForwardIndexAccessor() instanceof AbstractMapForwardIndexAccessor) { - ByteArraySequence serializedInputData = getForwardIndex().get(fileId); - AbstractMapForwardIndexAccessor forwardIndexAccessor = (AbstractMapForwardIndexAccessor)getForwardIndexAccessor(); - return forwardIndexAccessor.convertToInputDataMap(fileId, serializedInputData); - } - LOG.error("Can't fetch indexed data for index " + myIndexId.getName()); - return null; - } - - @Override - public void checkCanceled() { - ProgressManager.checkCanceled(); - } - @Override protected void requestRebuild(@NotNull Throwable ex) { Runnable action = () -> FileBasedIndex.getInstance().requestRebuild((ID)myIndexId, ex); @@ -341,48 +279,6 @@ else if (app.isReadAccessAllowed()) { } } - @Override - public void updateWithMap(@NotNull AbstractUpdateData updateData) throws StorageException { - try { - super.updateWithMap(updateData); - } - catch (ProcessCanceledException e) { - LOG.error("ProcessCancelledException is not expected here!", e); - throw e; - } - } - - @Override - public void setBufferingEnabled(boolean enabled) { - // TODO to be removed - throw new UnsupportedOperationException(); - } - - @Override - public void cleanupMemoryStorage() { - // TODO to be removed - throw new UnsupportedOperationException(); - } - - @Override - public void cleanupForNextTest() { - // TODO to be removed - throw new UnsupportedOperationException(); - } - - @Override - public void removeTransientDataForFile(int inputId) { - // TODO to be removed - throw new UnsupportedOperationException(); - } - - @Override - public void removeTransientDataForKeys(int inputId, - @NotNull InputDataDiffBuilder diffBuilder) { - // TODO to be removed - throw new UnsupportedOperationException(); - } - @Override protected @NotNull ValueSerializationProblemReporter getSerializationProblemReporter() { return problem -> { diff --git a/platform/platform-resources/src/META-INF/LangExtensions.xml b/platform/platform-resources/src/META-INF/LangExtensions.xml index 74f5177582ad7..ed6281294808b 100644 --- a/platform/platform-resources/src/META-INF/LangExtensions.xml +++ b/platform/platform-resources/src/META-INF/LangExtensions.xml @@ -59,6 +59,9 @@ + +