From c871b8b6ed2dd2996c455b05ef3f07fd7119d209 Mon Sep 17 00:00:00 2001 From: Andrew Lamb <38555688+andrewL-avlq@users.noreply.github.com> Date: Fri, 27 Oct 2023 11:13:09 +0100 Subject: [PATCH] Refactor IncrementalBuilder and Indexer. Refactor of the incremental builder and indexer to make it easier to write custom builders that calculate the affected resources separately from indexing the changed and deleted sources in the build request. Signed-off-by: Andrew Lamb --- .../xtext/build/IncrementalBuilder.java | 148 ++++++++++-------- .../src/org/eclipse/xtext/build/Indexer.java | 53 ++++++- 2 files changed, 131 insertions(+), 70 deletions(-) diff --git a/org.eclipse.xtext/src/org/eclipse/xtext/build/IncrementalBuilder.java b/org.eclipse.xtext/src/org/eclipse/xtext/build/IncrementalBuilder.java index 808c6e812b1..b3e995b711b 100644 --- a/org.eclipse.xtext/src/org/eclipse/xtext/build/IncrementalBuilder.java +++ b/org.eclipse.xtext/src/org/eclipse/xtext/build/IncrementalBuilder.java @@ -17,6 +17,7 @@ import java.util.Set; import java.util.function.Predicate; import java.util.stream.Collectors; +import java.util.stream.Stream; import org.eclipse.emf.common.util.URI; import org.eclipse.emf.ecore.resource.Resource; @@ -44,6 +45,7 @@ import org.eclipse.xtext.resource.XtextResourceSet; import org.eclipse.xtext.resource.clustering.DisabledClusteringPolicy; import org.eclipse.xtext.resource.clustering.IResourceClusteringPolicy; +import org.eclipse.xtext.resource.impl.ResourceDescriptionsData; import org.eclipse.xtext.resource.persistence.IResourceStorageFacade; import org.eclipse.xtext.resource.persistence.SerializableResourceDescription; import org.eclipse.xtext.resource.persistence.SourceLevelURIsAdapter; @@ -227,80 +229,43 @@ protected void unloadResource(URI uri, Predicate condition) { } } + /** + * Unload the given uris that are not already in the unloaded set. + * + * @param uriStream + * the uris to unload. + * @param unloaded + * the set of unloaded uris, to which the given uris to unload will be added. + * @since 2.33 + */ + protected void unloadResources(final Stream uriStream, final Set unloaded) { + uriStream.forEach(uri -> { + if (unloaded.add(uri)) { + unloadResource(uri); + } + }); + } + public IncrementalBuilder.Result launch() { - Source2GeneratedMapping newSource2GeneratedMapping = request.getState().getFileMappings(); Set unloaded = new HashSet<>(); - for (URI deleted : request.getDeletedFiles()) { - if (unloaded.add(deleted)) { - unloadResource(deleted); - } - } - for (URI dirty : request.getDirtyFiles()) { - if (unloaded.add(dirty)) { - unloadResource(dirty); - } - } - for (URI source : request.getDeletedFiles()) { - request.getAfterValidate().afterValidate(source, Collections.emptyList()); - Map outputConfigs = newSource2GeneratedMapping.deleteSourceAndGetOutputConfigs(source); - for (URI generated : outputConfigs.keySet()) { - IResourceServiceProvider serviceProvider = context.getResourceServiceProvider(source); - XtextResourceSet resourceSet = request.getResourceSet(); - Set configs = serviceProvider - .get(IContextualOutputConfigurationProvider2.class).getOutputConfigurations(resourceSet); - String configName = outputConfigs.get(generated); - OutputConfiguration config = FluentIterable.from(configs) - .firstMatch(it -> it.getName().equals(configName)).orNull(); - if (config != null && config.isCleanUpDerivedResources()) { - try { - resourceSet.getURIConverter().delete(generated, Collections.emptyMap()); - request.getAfterDeleteFile().apply(generated); - } catch (IOException e) { - throw new RuntimeIOException(e); - } - } - } - } + unloadResources(request.getDeletedFiles().stream(), unloaded); + unloadResources(request.getDirtyFiles().stream(), unloaded); + + Source2GeneratedMapping newSource2GeneratedMapping = request.getState().getFileMappings(); + deleteGeneratedSources(getRequest().getDeletedFiles(), newSource2GeneratedMapping); Indexer.IndexResult result = indexer.computeAndIndexAffected(request, context); operationCanceledManager.checkCanceled(request.getCancelIndicator()); List resolvedDeltas = new ArrayList<>(); - for (IResourceDescription.Delta delta : result.getResourceDeltas()) { - URI uri = delta.getUri(); - if (delta.getOld() != null && unloaded.add(uri)) { - unloadResource(uri); - } - if (delta.getNew() == null) { - resolvedDeltas.add(delta); - } - } + result.getResourceDeltas().stream().filter(delta -> delta.getNew() == null).forEach(resolvedDeltas::add); + unloadResources(result.getResourceDeltas().stream().filter(it -> it.getOld() != null).map(Delta::getUri), + unloaded); List toBeBuilt = result.getResourceDeltas().stream().filter((it) -> it.getNew() != null) .map(Delta::getUri).collect(Collectors.toList()); - + installSourceLevelURIs(toBeBuilt); Iterable deltas = context.executeClustered(toBeBuilt, - (resource) -> { - CancelIndicator cancelIndicator = request.getCancelIndicator(); - operationCanceledManager.checkCanceled(cancelIndicator); - // trigger init - resource.getContents(); - EcoreUtil2.resolveLazyCrossReferences(resource, CancelIndicator.NullImpl); - operationCanceledManager.checkCanceled(cancelIndicator); - IResourceServiceProvider serviceProvider = getResourceServiceProvider(resource); - IResourceDescription.Manager manager = serviceProvider.getResourceDescriptionManager(); - IResourceDescription description = manager.getResourceDescription(resource); - IResourceDescription copiedDescription = getSerializableResourceDescription(description); - result.getNewIndex().addDescription(resource.getURI(), copiedDescription); - operationCanceledManager.checkCanceled(cancelIndicator); - if (!request.isIndexOnly() && validate(resource) && serviceProvider.get(IShouldGenerate.class) - .shouldGenerate(resource, CancelIndicator.NullImpl)) { - operationCanceledManager.checkCanceled(cancelIndicator); - generate(resource, request, newSource2GeneratedMapping); - } - IResourceDescription old = context.getOldState().getResourceDescriptions() - .getResourceDescription(resource.getURI()); - return manager.createDelta(old, copiedDescription); - }); + resource -> buildResource(resource, result.getNewIndex(), newSource2GeneratedMapping)); Iterables.addAll(resolvedDeltas, deltas); return new IncrementalBuilder.Result(request.getState(), resolvedDeltas); @@ -332,6 +297,61 @@ protected Indexer getIndexer() { return indexer; } + /** + * @since 2.33 + */ + protected void deleteGeneratedSources(final List deletedSources, + final Source2GeneratedMapping newSource2GeneratedMapping) { + for (URI source : deletedSources) { + request.getAfterValidate().afterValidate(source, Collections.emptyList()); + Map outputConfigs = newSource2GeneratedMapping.deleteSourceAndGetOutputConfigs(source); + for (URI generated : outputConfigs.keySet()) { + IResourceServiceProvider serviceProvider = context.getResourceServiceProvider(source); + XtextResourceSet resourceSet = request.getResourceSet(); + Set configs = serviceProvider + .get(IContextualOutputConfigurationProvider2.class).getOutputConfigurations(resourceSet); + String configName = outputConfigs.get(generated); + OutputConfiguration config = FluentIterable.from(configs) + .firstMatch(it -> it.getName().equals(configName)).orNull(); + if (config != null && config.isCleanUpDerivedResources()) { + try { + resourceSet.getURIConverter().delete(generated, Collections.emptyMap()); + request.getAfterDeleteFile().apply(generated); + } catch (IOException e) { + throw new RuntimeIOException(e); + } + } + } + } + } + + /** + * @since 2.33 + */ + protected IResourceDescription.Delta buildResource(final Resource resource, + final ResourceDescriptionsData newIndex, final Source2GeneratedMapping newSource2GeneratedMapping) { + CancelIndicator cancelIndicator = request.getCancelIndicator(); + operationCanceledManager.checkCanceled(cancelIndicator); + // trigger init + resource.getContents(); + EcoreUtil2.resolveLazyCrossReferences(resource, CancelIndicator.NullImpl); + operationCanceledManager.checkCanceled(cancelIndicator); + IResourceServiceProvider serviceProvider = getResourceServiceProvider(resource); + IResourceDescription.Manager manager = serviceProvider.getResourceDescriptionManager(); + IResourceDescription description = manager.getResourceDescription(resource); + IResourceDescription copiedDescription = getSerializableResourceDescription(description); + newIndex.addDescription(resource.getURI(), copiedDescription); + operationCanceledManager.checkCanceled(cancelIndicator); + if (!request.isIndexOnly() && validate(resource) + && serviceProvider.get(IShouldGenerate.class).shouldGenerate(resource, CancelIndicator.NullImpl)) { + operationCanceledManager.checkCanceled(cancelIndicator); + generate(resource, request, newSource2GeneratedMapping); + } + IResourceDescription old = context.getOldState().getResourceDescriptions() + .getResourceDescription(resource.getURI()); + return manager.createDelta(old, copiedDescription); + } + /** * @since 2.28 */ diff --git a/org.eclipse.xtext/src/org/eclipse/xtext/build/Indexer.java b/org.eclipse.xtext/src/org/eclipse/xtext/build/Indexer.java index c6d79daf2dd..e3015902443 100644 --- a/org.eclipse.xtext/src/org/eclipse/xtext/build/Indexer.java +++ b/org.eclipse.xtext/src/org/eclipse/xtext/build/Indexer.java @@ -108,9 +108,12 @@ public String toString() { private OperationCanceledManager operationCanceledManager; /** - * Compute an updated index. + * Compute an initial index. Explicit calls to {@link #computeAndIndexAffected(Collection, IndexResult, BuildRequest, BuildContext)} + * are needed to complete indexing. + * + * @since 2.33 */ - public Indexer.IndexResult computeAndIndexAffected(BuildRequest request, BuildContext context) { + public Indexer.IndexResult initialIndex(final BuildRequest request, final BuildContext context) { ResourceDescriptionsData previousIndex = context.getOldState().getResourceDescriptions(); ResourceDescriptionsData newIndex = request.getState().getResourceDescriptions(); List deltas = new ArrayList<>(); @@ -119,16 +122,30 @@ public Indexer.IndexResult computeAndIndexAffected(BuildRequest request, BuildCo for (IResourceDescription.Delta delta : deltas) { newIndex.register(delta); } + return new Indexer.IndexResult(deltas, newIndex); + } + + /** + * Compute an updated index based on new deltas and a previously calculated {@link #initialIndex(BuildRequest, BuildContext)}. + * + * @since 2.33 + */ + public List computeAndIndexAffected( + final Collection newDeltas, final Indexer.IndexResult result, + final BuildRequest request, final BuildContext context) { + List deltas = result.getResourceDeltas(); Set allDeltas = new HashSet<>(deltas); allDeltas.addAll(request.getExternalDeltas()); Set deltaSet = FluentIterable.from(deltas).transform(Delta::getUri).toSet(); + ResourceDescriptionsData previousIndex = context.getOldState().getResourceDescriptions(); List allAffected = FluentIterable.from(previousIndex.getAllResourceDescriptions()) .transform(IResourceDescription::getURI).filter(it -> !deltaSet.contains(it)).filter(it -> { IResourceServiceProvider resourceServiceProvider = context.getResourceServiceProvider(it); if (resourceServiceProvider != null) { IResourceDescription.Manager manager = resourceServiceProvider.getResourceDescriptionManager(); IResourceDescription resourceDescription = previousIndex.getResourceDescription(it); - return isAffected(resourceDescription, manager, allDeltas, allDeltas, newIndex); + return isAffected(resourceDescription, manager, newDeltas, allDeltas, + result.getNewIndex()); } else { IResourceDescription.Delta delta = getDeltaForDeletedResource(it, previousIndex); if (delta != null) { @@ -137,8 +154,19 @@ public Indexer.IndexResult computeAndIndexAffected(BuildRequest request, BuildCo return false; } }).toList(); - deltas.addAll(getDeltasForChangedResources(allAffected, previousIndex, context)); - return new Indexer.IndexResult(deltas, newIndex); + List affectedDeltas = getDeltasForChangedResources(allAffected, previousIndex, + context); + deltas.addAll(affectedDeltas); + return affectedDeltas; + } + + /** + * Compute an updated index. + */ + public Indexer.IndexResult computeAndIndexAffected(BuildRequest request, BuildContext context) { + Indexer.IndexResult result = initialIndex(request, context); + computeAndIndexAffected(result.getResourceDeltas(), result, request, context); + return result; } /** @@ -152,7 +180,7 @@ protected List getDeltasForDeletedResources(BuildReq IResourceServiceProvider resourceServiceProvider = context.getResourceServiceProvider(deleted); if (resourceServiceProvider != null) { operationCanceledManager.checkCanceled(context.getCancelIndicator()); - IResourceDescription.Delta delta = getDeltaForDeletedResource(deleted, oldIndex); + IResourceDescription.Delta delta = getDeltaForDeletedResource(deleted, oldIndex, resourceServiceProvider.getResourceDescriptionManager()); if (delta != null) { deltas.add(delta); } @@ -169,8 +197,21 @@ protected List getDeltasForDeletedResources(BuildReq * */ protected IResourceDescription.Delta getDeltaForDeletedResource(URI uri, ResourceDescriptionsData oldIndex) { + return getDeltaForDeletedResource(uri, oldIndex, null); + } + + /** + * Gets a delta for a resource that shall be deleted using the resource description manager if provided. + * + * @since 2.33 + * + */ + protected IResourceDescription.Delta getDeltaForDeletedResource(URI uri, ResourceDescriptionsData oldIndex, IResourceDescription.Manager manager) { IResourceDescription oldDescription = oldIndex.getResourceDescription(uri); if (oldDescription != null) { + if (manager != null) { + return manager.createDelta(oldDescription, null); + } return new DefaultResourceDescriptionDelta(oldDescription, null); } return null;