Merge remote-tracking branch 'origin/master' into handle-relaystate

.ci/bwcVersions

@@ -8,6 +8,7 @@ BWC_VERSION:
   - "7.3.0"
   - "7.3.1"
   - "7.3.2"
+  - "7.3.3"
   - "7.4.0"
   - "7.5.0"
   - "8.0.0"

.ci/init.gradle

@@ -86,6 +86,10 @@ if (buildCacheUrl) {
             .getData();
     gradle.settingsEvaluated { settings ->
         settings.buildCache {
+            local {
+                // Disable the local build cache in CI since we use ephemeral workers and it incurs an IO penalty
+                enabled = false
+            }
             remote(HttpBuildCache) {
                 url = buildCacheUrl
                 push = buildCachePush

buildSrc/build.gradle

@@ -107,6 +107,7 @@ dependencies {
   compile localGroovy()
 
   compile 'commons-codec:commons-codec:1.12'
+  compile 'org.apache.commons:commons-compress:1.19'
 
   compile 'com.netflix.nebula:gradle-extra-configurations-plugin:3.0.3'
   compile 'com.netflix.nebula:nebula-publishing-plugin:4.4.4'

buildSrc/settings.gradle

@@ -1 +1,2 @@
 include 'reaper'
+include 'symbolic-link-preserving-tar'

buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/DocsTestPlugin.groovy

@@ -21,19 +21,21 @@ package org.elasticsearch.gradle.doc
 import org.elasticsearch.gradle.OS
 import org.elasticsearch.gradle.Version
 import org.elasticsearch.gradle.VersionProperties
-import org.elasticsearch.gradle.test.RestTestPlugin
+import org.gradle.api.Plugin
 import org.gradle.api.Project
 import org.gradle.api.Task
+
 /**
  * Sets up tests for documentation.
  */
-public class DocsTestPlugin extends RestTestPlugin {
+class DocsTestPlugin implements Plugin<Project> {
 
     @Override
-    public void apply(Project project) {
+    void apply(Project project) {
         project.pluginManager.apply('elasticsearch.testclusters')
         project.pluginManager.apply('elasticsearch.standalone-rest-test')
-        super.apply(project)
+        project.pluginManager.apply('elasticsearch.rest-test')
+
         String distribution = System.getProperty('tests.distribution', 'default')
         // The distribution can be configured with -Dtests.distribution on the command line
         project.testClusters.integTest.testDistribution = distribution.toUpperCase()

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/DistroTestPlugin.java

@@ -66,7 +66,9 @@
 public class DistroTestPlugin implements Plugin<Project> {
 
     private static final String SYSTEM_JDK_VERSION = "11.0.2+9";
+    private static final String SYSTEM_JDK_VENDOR = "openjdk";
     private static final String GRADLE_JDK_VERSION = "12.0.1+12@69cfe15208a647278a19ef0990eea691";
+    private static final String GRADLE_JDK_VENDOR = "openjdk";
 
     // all distributions used by distro tests. this is temporary until tests are per distribution
     private static final String DISTRIBUTIONS_CONFIGURATION = "distributions";
@@ -139,8 +141,10 @@ public void apply(Project project) {
         });
     }
 
-    private static Jdk createJdk(NamedDomainObjectContainer<Jdk> jdksContainer, String name, String version, String platform) {
+    private static Jdk createJdk(
+        NamedDomainObjectContainer<Jdk> jdksContainer, String name, String vendor, String version, String platform) {
         Jdk jdk = jdksContainer.create(name);
+        jdk.setVendor(vendor);
         jdk.setVersion(version);
         jdk.setPlatform(platform);
         return jdk;
@@ -172,11 +176,11 @@ private static List<Object> configureVM(Project project) {
         String box = project.getName();
 
         // setup jdks used by the distro tests, and by gradle executing
-        
+
         NamedDomainObjectContainer<Jdk> jdksContainer = JdkDownloadPlugin.getContainer(project);
         String platform = box.contains("windows") ? "windows" : "linux";
-        Jdk systemJdk = createJdk(jdksContainer, "system", SYSTEM_JDK_VERSION, platform);
-        Jdk gradleJdk = createJdk(jdksContainer, "gradle", GRADLE_JDK_VERSION, platform);
+        Jdk systemJdk = createJdk(jdksContainer, "system", SYSTEM_JDK_VENDOR, SYSTEM_JDK_VERSION, platform);
+        Jdk gradleJdk = createJdk(jdksContainer, "gradle", GRADLE_JDK_VENDOR, GRADLE_JDK_VERSION, platform);
 
         // setup VM used by these tests
         VagrantExtension vagrant = project.getExtensions().getByType(VagrantExtension.class);
@@ -309,7 +313,7 @@ private static TaskProvider<BatsTestTask> configureBatsTest(Project project, Str
                 }
             });
     }
-    
+
     private List<ElasticsearchDistribution> configureDistributions(Project project, Version upgradeVersion) {
         NamedDomainObjectContainer<ElasticsearchDistribution> distributions = DistributionDownloadPlugin.getContainer(project);
         List<ElasticsearchDistribution> currentDistros = new ArrayList<>();

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestIntegTestTask.groovy

@@ -22,10 +22,12 @@ import org.elasticsearch.gradle.VersionProperties
 import org.elasticsearch.gradle.testclusters.ElasticsearchCluster
 import org.elasticsearch.gradle.testclusters.RestTestRunnerTask
 import org.elasticsearch.gradle.testclusters.TestClustersPlugin
+import org.elasticsearch.gradle.tool.Boilerplate
 import org.elasticsearch.gradle.tool.ClasspathUtils
 import org.gradle.api.DefaultTask
 import org.gradle.api.Task
 import org.gradle.api.execution.TaskExecutionAdapter
+import org.gradle.api.file.FileCopyDetails
 import org.gradle.api.logging.Logger
 import org.gradle.api.logging.Logging
 import org.gradle.api.tasks.Copy
@@ -38,6 +40,7 @@ import org.gradle.plugins.ide.idea.IdeaPlugin
 import java.nio.charset.StandardCharsets
 import java.nio.file.Files
 import java.util.stream.Stream
+
 /**
  * A wrapper task around setting up a cluster and running rest tests.
  */
@@ -81,9 +84,9 @@ class RestIntegTestTask extends DefaultTask {
             }
             if (usesTestclusters == true) {
                 ElasticsearchCluster cluster = project.testClusters."${name}"
-                runner.nonInputProperties.systemProperty('tests.rest.cluster', "${-> cluster.allHttpSocketURI.join(",") }")
-                runner.nonInputProperties.systemProperty('tests.cluster', "${-> cluster.transportPortURI }")
-                runner.nonInputProperties.systemProperty('tests.clustername', "${-> cluster.getName() }")
+                runner.nonInputProperties.systemProperty('tests.rest.cluster', "${-> cluster.allHttpSocketURI.join(",")}")
+                runner.nonInputProperties.systemProperty('tests.cluster', "${-> cluster.transportPortURI}")
+                runner.nonInputProperties.systemProperty('tests.clustername', "${-> cluster.getName()}")
             } else {
                 // we pass all nodes to the rest cluster to allow the clients to round-robin between them
                 // this is more realistic than just talking to a single node
@@ -123,10 +126,10 @@ class RestIntegTestTask extends DefaultTask {
             runner.systemProperty('test.clustername', System.getProperty("tests.clustername"))
         }
 
-        // copy the rest spec/tests into the test resources
-        Task copyRestSpec = createCopyRestSpecTask()
-        runner.dependsOn(copyRestSpec)
-        
+        // copy the rest spec/tests onto the test classpath
+        Copy copyRestSpec = createCopyRestSpecTask()
+        project.sourceSets.test.output.builtBy(copyRestSpec)
+
         // this must run after all projects have been configured, so we know any project
         // references can be accessed as a fully configured
         project.gradle.projectsEvaluated {
@@ -150,8 +153,8 @@ class RestIntegTestTask extends DefaultTask {
     }
 
     @Option(
-        option = "debug-jvm",
-        description = "Enable debugging configuration, to allow attaching a debugger to elasticsearch."
+            option = "debug-jvm",
+            description = "Enable debugging configuration, to allow attaching a debugger to elasticsearch."
     )
     public void setDebug(boolean enabled) {
         clusterConfig.debug = enabled;
@@ -166,7 +169,7 @@ class RestIntegTestTask extends DefaultTask {
         runner.dependsOn(dependencies)
         for (Object dependency : dependencies) {
             if (dependency instanceof Fixture) {
-                runner.finalizedBy(((Fixture)dependency).getStopTask())
+                runner.finalizedBy(((Fixture) dependency).getStopTask())
             }
         }
         return this
@@ -177,7 +180,7 @@ class RestIntegTestTask extends DefaultTask {
         runner.setDependsOn(dependencies)
         for (Object dependency : dependencies) {
             if (dependency instanceof Fixture) {
-                runner.finalizedBy(((Fixture)dependency).getStopTask())
+                runner.finalizedBy(((Fixture) dependency).getStopTask())
             }
         }
     }
@@ -224,50 +227,37 @@ class RestIntegTestTask extends DefaultTask {
 
     }
 
-    /**
-     * Creates a task (if necessary) to copy the rest spec files.
-     *
-     * @param project The project to add the copy task to
-     * @param includePackagedTests true if the packaged tests should be copied, false otherwise
-     */
-    Task createCopyRestSpecTask() {
-        project.configurations {
-            restSpec
-        }
-        project.dependencies {
-            restSpec ClasspathUtils.isElasticsearchProject() ? project.project(':rest-api-spec') :
-                    "org.elasticsearch:rest-api-spec:${VersionProperties.elasticsearch}"
+    Copy createCopyRestSpecTask() {
+        Boilerplate.maybeCreate(project.configurations, 'restSpec') {
+            project.dependencies.add(
+                    'restSpec',
+                    ClasspathUtils.isElasticsearchProject() ? project.project(':rest-api-spec') :
+                            "org.elasticsearch:rest-api-spec:${VersionProperties.elasticsearch}"
+            )
         }
-        Task copyRestSpec = project.tasks.findByName('copyRestSpec')
-        if (copyRestSpec != null) {
-            return copyRestSpec
-        }
-        Map copyRestSpecProps = [
-                name     : 'copyRestSpec',
-                type     : Copy,
-                dependsOn: [project.configurations.restSpec, 'processTestResources']
-        ]
-        copyRestSpec = project.tasks.create(copyRestSpecProps) {
-            into project.sourceSets.test.output.resourcesDir
-        }
-        project.afterEvaluate {
-            copyRestSpec.from({ project.zipTree(project.configurations.restSpec.singleFile) }) {
-                include 'rest-api-spec/api/**'
-                if (includePackaged) {
-                    include 'rest-api-spec/test/**'
+
+        return Boilerplate.maybeCreate(project.tasks, 'copyRestSpec', Copy) { Copy copy ->
+            copy.dependsOn project.configurations.restSpec
+            copy.into(project.sourceSets.test.output.resourcesDir)
+            copy.from({ project.zipTree(project.configurations.restSpec.singleFile) }) {
+                includeEmptyDirs = false
+                include 'rest-api-spec/**'
+                filesMatching('rest-api-spec/test/**') { FileCopyDetails details ->
+                    if (includePackaged == false) {
+                        details.exclude()
+                    }
                 }
             }
-        }
-        if (project.plugins.hasPlugin(IdeaPlugin)) {
-            project.idea {
-                module {
-                    if (scopes.TEST != null) {
-                        scopes.TEST.plus.add(project.configurations.restSpec)
+
+            if (project.plugins.hasPlugin(IdeaPlugin)) {
+                project.idea {
+                    module {
+                        if (scopes.TEST != null) {
+                            scopes.TEST.plus.add(project.configurations.restSpec)
+                        }
                     }
                 }
             }
         }
-        return copyRestSpec
     }
-
 }

buildSrc/src/main/java/org/elasticsearch/gradle/EmptyDirTask.java

@@ -25,7 +25,7 @@
 import org.gradle.api.DefaultTask;
 import org.gradle.api.tasks.Input;
 import org.gradle.api.tasks.TaskAction;
-import org.gradle.internal.nativeintegration.filesystem.Chmod;
+import org.gradle.internal.file.Chmod;
 
 /**
  * Creates an empty directory.

buildSrc/src/main/java/org/elasticsearch/gradle/Jdk.java

@@ -34,19 +34,22 @@
 
 public class Jdk implements Buildable, Iterable<File> {
 
-    static final Pattern VERSION_PATTERN = Pattern.compile("(\\d+)(\\.\\d+\\.\\d+)?\\+(\\d+)(@([a-f0-9]{32}))?");
-    private static final List<String> ALLOWED_PLATFORMS = Collections.unmodifiableList(Arrays.asList("linux", "windows", "darwin"));
+    private static final List<String> ALLOWED_VENDORS = List.of("adoptopenjdk", "openjdk");
+    static final Pattern VERSION_PATTERN =
+        Pattern.compile("(\\d+)(\\.\\d+\\.\\d+)?\\+(\\d+(?:\\.\\d+)?)(@([a-f0-9]{32}))?");
+    private static final List<String> ALLOWED_PLATFORMS = Collections.unmodifiableList(Arrays.asList("darwin", "linux", "windows"));
 
     private final String name;
     private final Configuration configuration;
 
+    private final Property<String> vendor;
     private final Property<String> version;
     private final Property<String> platform;
 
-
     Jdk(String name, Project project) {
         this.name = name;
         this.configuration = project.getConfigurations().create("jdk_" + name);
+        this.vendor = project.getObjects().property(String.class);
         this.version = project.getObjects().property(String.class);
         this.platform = project.getObjects().property(String.class);
     }
@@ -55,6 +58,17 @@ public String getName() {
         return name;
     }
 
+    public String getVendor() {
+        return vendor.get();
+    }
+
+    public void setVendor(final String vendor) {
+        if (ALLOWED_VENDORS.contains(vendor) == false) {
+            throw new IllegalArgumentException("unknown vendor [" + vendor + "] for jdk [" + name + "], must be one of " + ALLOWED_VENDORS);
+        }
+        this.vendor.set(vendor);
+    }
+
     public String getVersion() {
         return version.get();
     }
@@ -105,12 +119,17 @@ void finalizeValues() {
         if (platform.isPresent() == false) {
             throw new IllegalArgumentException("platform not specified for jdk [" + name + "]");
         }
+        if (vendor.isPresent() == false) {
+            throw new IllegalArgumentException("vendor not specified for jdk [" + name + "]");
+        }
         version.finalizeValue();
         platform.finalizeValue();
+        vendor.finalizeValue();;
     }
 
     @Override
     public Iterator<File> iterator() {
         return configuration.iterator();
     }
+
 }