Fix slow sync test clustres artifacts task (#42012)

* Fix slow sync test clustres artifacts task The task was mistakenly adding a combinational explosion of task actions all doing the same thing. With this PR this is fixed and each version - distribution pair is only extracted once. I appologieze for the SSD wear. * Look for configurations on the root project * Add dependency on configurations * This should be a `copy` so we don't blow away all the other distros * Don't copy example plugin build directory in integration tests
elastic · May 10, 2019 · 044af2a · 044af2a
1 parent a3cc03e
commit 044af2a
Show file tree

Hide file tree

Showing 2 changed files with 36 additions and 28 deletions.
diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/TestClustersPlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/TestClustersPlugin.java
@@ -182,8 +182,9 @@ private void configureClaimClustersHook(Project project) {
                         claimsInventory.put(elasticsearchCluster, claimsInventory.getOrDefault(elasticsearchCluster, 0) + 1);
                     }
                 }));
-
-            logger.info("Claims inventory: {}", claimsInventory);
+            if (claimsInventory.isEmpty() == false) {
+                logger.info("Claims inventory: {}", claimsInventory);
+            }
         });
     }
 
@@ -279,8 +280,14 @@ private static void autoConfigureClusterDependencies(
         // the clusters will look for artifacts there based on the naming conventions.
         // Tasks that use a cluster will add this as a dependency automatically so it's guaranteed to run early in
         // the build.
-        Task sync = Boilerplate.maybeCreate(rootProject.getTasks(), SYNC_ARTIFACTS_TASK_NAME, onCreate -> {
+        Boilerplate.maybeCreate(rootProject.getTasks(), SYNC_ARTIFACTS_TASK_NAME, onCreate -> {
             onCreate.getOutputs().dir(getExtractDir(rootProject));
+            onCreate.getInputs().files(
+                project.getRootProject().getConfigurations().matching(conf -> conf.getName().startsWith(HELPER_CONFIGURATION_PREFIX))
+            );
+            onCreate.dependsOn(project.getRootProject().getConfigurations()
+                .matching(conf -> conf.getName().startsWith(HELPER_CONFIGURATION_PREFIX))
+            );
             // NOTE: Gradle doesn't allow a lambda here ( fails at runtime )
             onCreate.doFirst(new Action<Task>() {
                 @Override
@@ -290,6 +297,31 @@ public void execute(Task task) {
                     project.delete(getExtractDir(rootProject));
                 }
             });
+            onCreate.doLast(new Action<Task>() {
+                    @Override
+                    public void execute(Task task) {
+                        project.getRootProject().getConfigurations()
+                            .matching(config -> config.getName().startsWith(HELPER_CONFIGURATION_PREFIX))
+                            .forEach(config -> project.copy(spec ->
+                                config.getResolvedConfiguration()
+                                    .getResolvedArtifacts()
+                                    .forEach(resolvedArtifact -> {
+                                        final FileTree files;
+                                        File file = resolvedArtifact.getFile();
+                                        if (file.getName().endsWith(".zip")) {
+                                            files = project.zipTree(file);
+                                        } else if (file.getName().endsWith("tar.gz")) {
+                                            files = project.tarTree(file);
+                                        } else {
+                                            throw new IllegalArgumentException("Can't extract " + file + " unknown file extension");
+                                        }
+                                        logger.info("Extracting {}@{}", resolvedArtifact, config);
+                                        spec.from(files, s -> s.into(resolvedArtifact.getModuleVersion().getId().getGroup()));
+                                        spec.into(getExtractDir(project));
+                                    }))
+                            );
+                    }
+            });
         });
 
         // When the project evaluated we know of all tasks that use clusters.
@@ -347,29 +379,6 @@ public void execute(Task task) {
                             distribution.getFileExtension());
 
                 }
-
-                sync.getInputs().files(helperConfiguration);
-                // NOTE: Gradle doesn't allow a lambda here ( fails at runtime )
-                sync.doLast(new Action<Task>() {
-                    @Override
-                    public void execute(Task task) {
-                        project.copy(spec ->
-                            helperConfiguration.getResolvedConfiguration().getResolvedArtifacts().forEach(resolvedArtifact -> {
-                                final FileTree files;
-                                File file = resolvedArtifact.getFile();
-                                if (file.getName().endsWith(".zip")) {
-                                    files = project.zipTree(file);
-                                } else if (file.getName().endsWith("tar.gz")) {
-                                    files = project.tarTree(file);
-                                } else {
-                                    throw new IllegalArgumentException("Can't extract " + file + " unknown file extension");
-                                }
-
-                                spec.from(files, s -> s.into(resolvedArtifact.getModuleVersion().getId().getGroup()));
-                                spec.into(getExtractDir(project));
-                            }));
-                    }
-                });
             })));
     }
 

diff --git a/buildSrc/src/test/java/org/elasticsearch/gradle/BuildExamplePluginsIT.java b/buildSrc/src/test/java/org/elasticsearch/gradle/BuildExamplePluginsIT.java
@@ -75,7 +75,7 @@ public static Iterable<Object[]> parameters() {
     }
 
     public void testCurrentExamplePlugin() throws IOException {
-        FileUtils.copyDirectory(examplePlugin, tmpDir.getRoot());
+        FileUtils.copyDirectory(examplePlugin, tmpDir.getRoot(), pathname -> pathname.getPath().contains("/build/") == false);
 
         adaptBuildScriptForTest();
 
@@ -156,5 +156,4 @@ private Path writeBuildScript(String script) {
             throw new RuntimeException(e);
         }
     }
-
 }