From 044af2a4c4712bff4538b5eeee7b0f54dc2aba82 Mon Sep 17 00:00:00 2001 From: Alpar Torok Date: Fri, 10 May 2019 14:24:42 +0300 Subject: [PATCH] Fix slow sync test clustres artifacts task (#42012) * Fix slow sync test clustres artifacts task The task was mistakenly adding a combinational explosion of task actions all doing the same thing. With this PR this is fixed and each version - distribution pair is only extracted once. I appologieze for the SSD wear. * Look for configurations on the root project * Add dependency on configurations * This should be a `copy` so we don't blow away all the other distros * Don't copy example plugin build directory in integration tests --- .../testclusters/TestClustersPlugin.java | 61 +++++++++++-------- .../gradle/BuildExamplePluginsIT.java | 3 +- 2 files changed, 36 insertions(+), 28 deletions(-) diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/TestClustersPlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/TestClustersPlugin.java index c1ed6b770f04c..daca1f5ebb191 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/TestClustersPlugin.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/TestClustersPlugin.java @@ -182,8 +182,9 @@ private void configureClaimClustersHook(Project project) { claimsInventory.put(elasticsearchCluster, claimsInventory.getOrDefault(elasticsearchCluster, 0) + 1); } })); - - logger.info("Claims inventory: {}", claimsInventory); + if (claimsInventory.isEmpty() == false) { + logger.info("Claims inventory: {}", claimsInventory); + } }); } @@ -279,8 +280,14 @@ private static void autoConfigureClusterDependencies( // the clusters will look for artifacts there based on the naming conventions. // Tasks that use a cluster will add this as a dependency automatically so it's guaranteed to run early in // the build. - Task sync = Boilerplate.maybeCreate(rootProject.getTasks(), SYNC_ARTIFACTS_TASK_NAME, onCreate -> { + Boilerplate.maybeCreate(rootProject.getTasks(), SYNC_ARTIFACTS_TASK_NAME, onCreate -> { onCreate.getOutputs().dir(getExtractDir(rootProject)); + onCreate.getInputs().files( + project.getRootProject().getConfigurations().matching(conf -> conf.getName().startsWith(HELPER_CONFIGURATION_PREFIX)) + ); + onCreate.dependsOn(project.getRootProject().getConfigurations() + .matching(conf -> conf.getName().startsWith(HELPER_CONFIGURATION_PREFIX)) + ); // NOTE: Gradle doesn't allow a lambda here ( fails at runtime ) onCreate.doFirst(new Action() { @Override @@ -290,6 +297,31 @@ public void execute(Task task) { project.delete(getExtractDir(rootProject)); } }); + onCreate.doLast(new Action() { + @Override + public void execute(Task task) { + project.getRootProject().getConfigurations() + .matching(config -> config.getName().startsWith(HELPER_CONFIGURATION_PREFIX)) + .forEach(config -> project.copy(spec -> + config.getResolvedConfiguration() + .getResolvedArtifacts() + .forEach(resolvedArtifact -> { + final FileTree files; + File file = resolvedArtifact.getFile(); + if (file.getName().endsWith(".zip")) { + files = project.zipTree(file); + } else if (file.getName().endsWith("tar.gz")) { + files = project.tarTree(file); + } else { + throw new IllegalArgumentException("Can't extract " + file + " unknown file extension"); + } + logger.info("Extracting {}@{}", resolvedArtifact, config); + spec.from(files, s -> s.into(resolvedArtifact.getModuleVersion().getId().getGroup())); + spec.into(getExtractDir(project)); + })) + ); + } + }); }); // When the project evaluated we know of all tasks that use clusters. @@ -347,29 +379,6 @@ public void execute(Task task) { distribution.getFileExtension()); } - - sync.getInputs().files(helperConfiguration); - // NOTE: Gradle doesn't allow a lambda here ( fails at runtime ) - sync.doLast(new Action() { - @Override - public void execute(Task task) { - project.copy(spec -> - helperConfiguration.getResolvedConfiguration().getResolvedArtifacts().forEach(resolvedArtifact -> { - final FileTree files; - File file = resolvedArtifact.getFile(); - if (file.getName().endsWith(".zip")) { - files = project.zipTree(file); - } else if (file.getName().endsWith("tar.gz")) { - files = project.tarTree(file); - } else { - throw new IllegalArgumentException("Can't extract " + file + " unknown file extension"); - } - - spec.from(files, s -> s.into(resolvedArtifact.getModuleVersion().getId().getGroup())); - spec.into(getExtractDir(project)); - })); - } - }); }))); } diff --git a/buildSrc/src/test/java/org/elasticsearch/gradle/BuildExamplePluginsIT.java b/buildSrc/src/test/java/org/elasticsearch/gradle/BuildExamplePluginsIT.java index 762bcc5ff9b31..bf982fa3aa2d2 100644 --- a/buildSrc/src/test/java/org/elasticsearch/gradle/BuildExamplePluginsIT.java +++ b/buildSrc/src/test/java/org/elasticsearch/gradle/BuildExamplePluginsIT.java @@ -75,7 +75,7 @@ public static Iterable parameters() { } public void testCurrentExamplePlugin() throws IOException { - FileUtils.copyDirectory(examplePlugin, tmpDir.getRoot()); + FileUtils.copyDirectory(examplePlugin, tmpDir.getRoot(), pathname -> pathname.getPath().contains("/build/") == false); adaptBuildScriptForTest(); @@ -156,5 +156,4 @@ private Path writeBuildScript(String script) { throw new RuntimeException(e); } } - }