Merge branch 'master' into security_authz_engine

CONTRIBUTING.md

@@ -100,8 +100,8 @@ JDK 11 and testing on a JDK 8 runtime; to do this, set `RUNTIME_JAVA_HOME`
 pointing to the Java home of a JDK 8 installation. Note that this mechanism can
 be used to test against other JDKs as well, this is not only limited to JDK 8.
 
-> Note: It is also required to have `JAVA8_HOME`, `JAVA9_HOME`, and
-`JAVA10_HOME` are available so that the tests can pass.
+> Note: It is also required to have `JAVA8_HOME`, `JAVA9_HOME`, `JAVA10_HOME`
+and `JAVA11_HOME` available so that the tests can pass.
 
 > Warning: do not use `sdkman` for Java installations which do not have proper
 `jrunscript` for jdk distributions.

TESTING.asciidoc

@@ -42,7 +42,7 @@ In order to start with a different distribution use the `-Drun.distribution` arg
 To for example start the open source distribution:
 
 -------------------------------------
-./gradlew run -Drun.distribution=oss-zip
+./gradlew run -Drun.distribution=oss
 -------------------------------------
 
 ==== License type

build.gradle

@@ -221,14 +221,14 @@ allprojects {
     "org.elasticsearch.plugin:elasticsearch-scripting-painless-spi:${version}": ':modules:lang-painless:spi',
     "org.elasticsearch.test:framework:${version}": ':test:framework',
     "org.elasticsearch.distribution.integ-test-zip:elasticsearch:${version}": ':distribution:archives:integ-test-zip',
-    "org.elasticsearch.distribution.zip:elasticsearch:${version}": ':distribution:archives:zip',
-    "org.elasticsearch.distribution.zip:elasticsearch-oss:${version}": ':distribution:archives:oss-zip',
-    "org.elasticsearch.distribution.tar:elasticsearch:${version}": ':distribution:archives:tar',
-    "org.elasticsearch.distribution.tar:elasticsearch-oss:${version}": ':distribution:archives:oss-tar',
-    "org.elasticsearch.distribution.rpm:elasticsearch:${version}": ':distribution:packages:rpm',
-    "org.elasticsearch.distribution.rpm:elasticsearch-oss:${version}": ':distribution:packages:oss-rpm',
-    "org.elasticsearch.distribution.deb:elasticsearch:${version}": ':distribution:packages:deb',
-    "org.elasticsearch.distribution.deb:elasticsearch-oss:${version}": ':distribution:packages:oss-deb',
+    "downloads.zip:elasticsearch:${version}": ':distribution:archives:zip',
+    "downloads.zip:elasticsearch-oss:${version}": ':distribution:archives:oss-zip',
+    "downloads.tar:elasticsearch:${version}": ':distribution:archives:tar',
+    "downloads.tar:elasticsearch-oss:${version}": ':distribution:archives:oss-tar',
+    "downloads.rpm:elasticsearch:${version}": ':distribution:packages:rpm',
+    "downloads.rpm:elasticsearch-oss:${version}": ':distribution:packages:oss-rpm',
+    "downloads.deb:elasticsearch:${version}": ':distribution:packages:deb',
+    "downloads.deb:elasticsearch-oss:${version}": ':distribution:packages:oss-deb',
     "org.elasticsearch.test:logger-usage:${version}": ':test:logger-usage',
     "org.elasticsearch.xpack.test:feature-aware:${version}": ':x-pack:test:feature-aware',
     // for transport client
@@ -244,13 +244,13 @@ allprojects {
   bwcVersions.forPreviousUnreleased { VersionCollection.UnreleasedVersionInfo unreleasedVersion ->
     Version unreleased = unreleasedVersion.version
     String snapshotProject = ":distribution:bwc:${unreleasedVersion.gradleProjectName}"
-    ext.projectSubstitutions["org.elasticsearch.distribution.deb:elasticsearch:${unreleased}"] = snapshotProject
-    ext.projectSubstitutions["org.elasticsearch.distribution.rpm:elasticsearch:${unreleased}"] = snapshotProject
-    ext.projectSubstitutions["org.elasticsearch.distribution.zip:elasticsearch:${unreleased}"] = snapshotProject
+    ext.projectSubstitutions["downloads.deb:elasticsearch:${unreleased}"] = snapshotProject
+    ext.projectSubstitutions["downloads.rpm:elasticsearch:${unreleased}"] = snapshotProject
+    ext.projectSubstitutions["downloads.zip:elasticsearch:${unreleased}"] = snapshotProject
     if (unreleased.onOrAfter('6.3.0')) {
-      ext.projectSubstitutions["org.elasticsearch.distribution.deb:elasticsearch-oss:${unreleased}"] = snapshotProject
-      ext.projectSubstitutions["org.elasticsearch.distribution.rpm:elasticsearch-oss:${unreleased}"] = snapshotProject
-      ext.projectSubstitutions["org.elasticsearch.distribution.zip:elasticsearch-oss:${unreleased}"] = snapshotProject
+      ext.projectSubstitutions["downloads.deb:elasticsearch-oss:${unreleased}"] = snapshotProject
+      ext.projectSubstitutions["downloads.rpm:elasticsearch-oss:${unreleased}"] = snapshotProject
+      ext.projectSubstitutions["downloads.zip:elasticsearch-oss:${unreleased}"] = snapshotProject
     }
   }
 

buildSrc/build.gradle

@@ -178,6 +178,21 @@ if (project != rootProject) {
   jarHell.enabled = false
   thirdPartyAudit.enabled = false
 
+  configurations {
+    distribution
+  }
+
+  dependencies {
+    distribution project(':distribution:archives:zip')
+    distribution project(':distribution:archives:oss-zip')
+  }
+
+  String localDownloads = "${rootProject.buildDir}/local-downloads"
+  task setupLocalDownloads(type:Copy) {
+    from configurations.distribution
+    into localDownloads
+  }
+
   unitTest {
     // The test task is configured to runtimeJava version, but build-tools doesn't support all of them, so test
     // with compiler instead on the ones that are too old.
@@ -192,6 +207,7 @@ if (project != rootProject) {
     dependsOn project.rootProject.allprojects.collect {
       it.tasks.matching { it.name == 'publishNebulaPublicationToLocalTestRepository'}
     }
+    dependsOn setupLocalDownloads
     exclude "**/*Tests.class"
     testClassesDirs = sourceSets.test.output.classesDirs
     classpath = sourceSets.test.runtimeClasspath
@@ -204,6 +220,7 @@ if (project != rootProject) {
             ).asPath,
     )
     systemProperty 'test.local-test-repo-path', "${rootProject.buildDir}/local-test-repo"
+    systemProperty 'test.local-test-downloads-path', localDownloads
     systemProperty 'test.version_under_test', version
     systemProperty 'test.lucene-snapshot-revision', (versions.lucene =~ /\w+-snapshot-([a-z0-9]+)/)[0][1]
   }

buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy

@@ -407,9 +407,16 @@ class BuildPlugin implements Plugin<Project> {
                 if (messages.isEmpty() == false) {
                     throw new GradleException(messages.join('\n'))
                 }
+                rootProject.rootProject.ext.requiredJavaVersions = null // reset to null to indicate the pre-execution checks have executed
             }
+        } else if (rootProject.rootProject.requiredJavaVersions == null) {
+            // check directly if the version is present since we are already executing
+            if (rootProject.javaVersions.get(version) == null) {
+                throw new GradleException("JAVA${version}_HOME required to run task:\n${task}")
+            }
+        } else {
+            rootProject.requiredJavaVersions.get(version).add(task)
         }
-        rootProject.requiredJavaVersions.get(version).add(task)
     }
 
     /** A convenience method for getting java home for a version of java and requiring that version for the given task to execute */
@@ -558,6 +565,12 @@ class BuildPlugin implements Plugin<Project> {
             repos.mavenLocal()
         }
         repos.jcenter()
+        repos.ivy {
+            url "https://artifacts.elastic.co/downloads"
+            patternLayout {
+                artifact "elasticsearch/[module]-[revision](-[classifier]).[ext]"
+            }
+        }
         repos.maven {
             name "elastic"
             url "https://artifacts.elastic.co/maven"

buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/DocsTestPlugin.groovy

@@ -34,7 +34,7 @@ public class DocsTestPlugin extends RestTestPlugin {
         project.pluginManager.apply('elasticsearch.standalone-rest-test')
         super.apply(project)
         // The distribution can be configured with -Dtests.distribution on the command line
-        project.integTestCluster.distribution = System.getProperty('tests.distribution', 'zip')
+        project.integTestCluster.distribution = System.getProperty('tests.distribution', 'default')
         // Docs are published separately so no need to assemble
         project.tasks.assemble.enabled = false
         Map<String, String> defaultSubstitutions = [

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy

@@ -29,7 +29,7 @@ class ClusterConfiguration {
     private final Project project
 
     @Input
-    String distribution = 'zip'
+    String distribution = 'default'
 
     @Input
     int numNodes = 1

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy

@@ -18,20 +18,17 @@
  */
 package org.elasticsearch.gradle.test
 
-import java.util.stream.Collectors
 import org.apache.tools.ant.DefaultLogger
 import org.apache.tools.ant.taskdefs.condition.Os
 import org.elasticsearch.gradle.BuildPlugin
 import org.elasticsearch.gradle.LoggedExec
 import org.elasticsearch.gradle.Version
 import org.elasticsearch.gradle.VersionProperties
-
 import org.elasticsearch.gradle.plugin.PluginBuildPlugin
 import org.elasticsearch.gradle.plugin.PluginPropertiesExtension
 import org.gradle.api.AntBuilder
 import org.gradle.api.DefaultTask
 import org.gradle.api.GradleException
-import org.gradle.api.InvalidUserDataException
 import org.gradle.api.Project
 import org.gradle.api.Task
 import org.gradle.api.artifacts.Configuration
@@ -45,7 +42,7 @@ import org.gradle.api.tasks.Exec
 import java.nio.charset.StandardCharsets
 import java.nio.file.Paths
 import java.util.concurrent.TimeUnit
-
+import java.util.stream.Collectors
 /**
  * A helper for creating tasks to build a cluster that is used by a task, and tear down the cluster when the task is finished.
  */
@@ -89,7 +86,7 @@ class ClusterFormationTasks {
         Configuration currentDistro = project.configurations.create("${prefix}_elasticsearchDistro")
         Configuration bwcDistro = project.configurations.create("${prefix}_elasticsearchBwcDistro")
         Configuration bwcPlugins = project.configurations.create("${prefix}_elasticsearchBwcPlugins")
-        if (System.getProperty('tests.distribution', 'oss-zip') == 'integ-test-zip') {
+        if (System.getProperty('tests.distribution', 'oss') == 'integ-test-zip') {
             throw new Exception("tests.distribution=integ-test-zip is not supported")
         }
         configureDistributionDependency(project, config.distribution, currentDistro, VersionProperties.elasticsearch)
@@ -174,24 +171,31 @@ class ClusterFormationTasks {
 
     /** Adds a dependency on the given distribution */
     static void configureDistributionDependency(Project project, String distro, Configuration configuration, String elasticsearchVersion) {
-        if (Version.fromString(elasticsearchVersion).before('6.3.0') &&
-                distro.startsWith('oss-')
-        ) {
+        // TEMP HACK
+        // The oss docs CI build overrides the distro on the command line. This hack handles backcompat until CI is updated.
+        if (distro.equals('oss-zip')) {
+            distro = 'oss'
+        }
+        if (distro.equals('zip')) {
+            distro = 'default'
+        }
+        // END TEMP HACK
+        if (['integ-test-zip', 'oss', 'default'].contains(distro) == false) {
+            throw new GradleException("Unknown distribution: ${distro} in project ${project.path}")
+        }
+        Version version = Version.fromString(elasticsearchVersion)
+        if (version.before('6.3.0') && distro.startsWith('oss-')) {
             distro = distro.substring('oss-'.length())
         }
-        String packaging = distro
-        if (distro.contains('tar')) {
-            packaging = 'tar.gz'\
-        } else if (distro.contains('zip')) {
-            packaging = 'zip'
+        String group = "downloads.zip"
+        if (distro.equals("integ-test-zip")) {
+            group = "org.elasticsearch.distribution.integ-test-zip"
         }
-        String subgroup = distro
         String artifactName = 'elasticsearch'
-        if (distro.contains('oss')) {
+        if (distro.equals('oss') && Version.fromString(elasticsearchVersion).onOrAfter('6.3.0')) {
             artifactName += '-oss'
-            subgroup = distro.substring('oss-'.length())
         }
-        project.dependencies.add(configuration.name, "org.elasticsearch.distribution.${subgroup}:${artifactName}:${elasticsearchVersion}@${packaging}")
+        project.dependencies.add(configuration.name, "${group}:${artifactName}:${elasticsearchVersion}@zip")
     }
 
     /** Adds a dependency on a different version of the given plugin, which will be retrieved using gradle's dependency resolution */
@@ -314,31 +318,13 @@ class ClusterFormationTasks {
           elasticsearch source tree. If this is a plugin built in the elasticsearch source tree or this is a distro in
           the elasticsearch source tree then this should be the version of elasticsearch built by the source tree.
           If it isn't then Bad Things(TM) will happen. */
-        Task extract
-
-        switch (node.config.distribution) {
-            case 'integ-test-zip':
-            case 'zip':
-            case 'oss-zip':
-                extract = project.tasks.create(name: name, type: Copy, dependsOn: extractDependsOn) {
-                    from {
-                        project.zipTree(configuration.singleFile)
-                    }
-                    into node.baseDir
-                }
-                break;
-            case 'tar':
-            case 'oss-tar':
-                extract = project.tasks.create(name: name, type: Copy, dependsOn: extractDependsOn) {
-                    from {
-                        project.tarTree(project.resources.gzip(configuration.singleFile))
-                    }
-                    into node.baseDir
-                }
-                break;
-            default:
-                throw new InvalidUserDataException("Unknown distribution: ${node.config.distribution}")
+        Task extract = project.tasks.create(name: name, type: Copy, dependsOn: extractDependsOn) {
+            from {
+                project.zipTree(configuration.singleFile)
+            }
+            into node.baseDir
         }
+
         return extract
     }
 

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/NodeInfo.groovy

@@ -29,9 +29,6 @@ import org.gradle.api.Project
 import java.nio.file.Files
 import java.nio.file.Path
 import java.nio.file.Paths
-
-import static org.elasticsearch.gradle.BuildPlugin.getJavaHome
-
 /**
  * A container for the files and configuration associated with a single node in a test cluster.
  */
@@ -125,8 +122,8 @@ class NodeInfo {
         baseDir = new File(project.buildDir, "cluster/${prefix} node${nodeNum}")
         pidFile = new File(baseDir, 'es.pid')
         this.nodeVersion = Version.fromString(nodeVersion)
-        homeDir = homeDir(baseDir, config.distribution, nodeVersion)
-        pathConf = pathConf(baseDir, config.distribution, nodeVersion)
+        homeDir = new File(baseDir, "elasticsearch-${nodeVersion}")
+        pathConf = new File(homeDir, 'config')
         if (config.dataDir != null) {
             dataDir = "${config.dataDir(nodeNum)}"
         } else {
@@ -299,41 +296,4 @@ class NodeInfo {
         }
         return dataDir
     }
-
-    /** Returns the directory elasticsearch home is contained in for the given distribution */
-    static File homeDir(File baseDir, String distro, String nodeVersion) {
-        String path
-        switch (distro) {
-            case 'integ-test-zip':
-            case 'zip':
-            case 'tar':
-            case 'oss-zip':
-            case 'oss-tar':
-                path = "elasticsearch-${nodeVersion}"
-                break
-            case 'rpm':
-            case 'deb':
-                path = "${distro}-extracted/usr/share/elasticsearch"
-                break
-            default:
-                throw new InvalidUserDataException("Unknown distribution: ${distro}")
-        }
-        return new File(baseDir, path)
-    }
-
-    static File pathConf(File baseDir, String distro, String nodeVersion) {
-        switch (distro) {
-            case 'integ-test-zip':
-            case 'zip':
-            case 'oss-zip':
-            case 'tar':
-            case 'oss-tar':
-                return new File(homeDir(baseDir, distro, nodeVersion), 'config')
-            case 'rpm':
-            case 'deb':
-                return new File(baseDir, "${distro}-extracted/etc/elasticsearch")
-            default:
-                throw new InvalidUserDataException("Unknown distribution: ${distro}")
-        }
-    }
 }

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RunTask.groovy

@@ -16,7 +16,7 @@ public class RunTask extends DefaultTask {
         clusterConfig.httpPort = 9200
         clusterConfig.transportPort = 9300
         clusterConfig.daemonize = false
-        clusterConfig.distribution = 'zip'
+        clusterConfig.distribution = 'default'
         project.afterEvaluate {
             ClusterFormationTasks.setup(project, name, this, clusterConfig)
         }

buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantTestPlugin.groovy

@@ -193,10 +193,10 @@ class VagrantTestPlugin implements Plugin<Project> {
         UPGRADE_FROM_ARCHIVES.each {
             // The version of elasticsearch that we upgrade *from*
             project.dependencies.add(PACKAGING_CONFIGURATION,
-                    "org.elasticsearch.distribution.${it}:elasticsearch:${upgradeFromVersion}@${it}")
+                    "downloads.${it}:elasticsearch:${upgradeFromVersion}@${it}")
             if (upgradeFromVersion.onOrAfter('6.3.0')) {
                 project.dependencies.add(PACKAGING_CONFIGURATION,
-                        "org.elasticsearch.distribution.${it}:elasticsearch-oss:${upgradeFromVersion}@${it}")
+                        "downloads.${it}:elasticsearch-oss:${upgradeFromVersion}@${it}")
             }
         }
 

buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java

@@ -324,8 +324,9 @@ private void configure() {
         getConfPathSharedData().mkdirs();
         getConfPathLogs().mkdirs();
         LinkedHashMap<String, String> config = new LinkedHashMap<>();
-        config.put("cluster.name", "cluster-" + safeName(name));
-        config.put("node.name", "node-" + safeName(name));
+        String nodeName = safeName(name);
+        config.put("cluster.name",nodeName);
+        config.put("node.name", nodeName);
         config.put("path.repo", getConfPathRepo().getAbsolutePath());
         config.put("path.data", getConfPathData().getAbsolutePath());
         config.put("path.logs", getConfPathLogs().getAbsolutePath());
@@ -342,6 +343,9 @@ private void configure() {
         if (Version.fromString(version).getMajor() >= 6) {
             config.put("cluster.routing.allocation.disk.watermark.flood_stage", "1b");
         }
+        if (Version.fromString(version).getMajor() >= 7) {
+            config.put("cluster.initial_master_nodes", "[" + nodeName + "]");
+        }
         try {
             Files.write(
                 getConfigFile().toPath(),

buildSrc/src/main/resources/forbidden/es-all-signatures.txt

@@ -50,3 +50,16 @@ java.nio.channels.SocketChannel#connect(java.net.SocketAddress)
 java.lang.Boolean#getBoolean(java.lang.String)
 
 org.apache.lucene.util.IOUtils @ use @org.elasticsearch.core.internal.io instead
+
+@defaultMessage use executors from org.elasticsearch.common.util.concurrent.EsExecutors instead which will properly bubble up Errors
+java.util.concurrent.AbstractExecutorService#<init>()
+java.util.concurrent.ThreadPoolExecutor#<init>(int, int, long, java.util.concurrent.TimeUnit, java.util.concurrent.BlockingQueue)
+java.util.concurrent.ThreadPoolExecutor#<init>(int, int, long, java.util.concurrent.TimeUnit, java.util.concurrent.BlockingQueue, java.util.concurrent.ThreadFactory)
+java.util.concurrent.ThreadPoolExecutor#<init>(int, int, long, java.util.concurrent.TimeUnit, java.util.concurrent.BlockingQueue, java.util.concurrent.RejectedExecutionHandler)
+java.util.concurrent.ThreadPoolExecutor#<init>(int, int, long, java.util.concurrent.TimeUnit, java.util.concurrent.BlockingQueue, java.util.concurrent.ThreadFactory, java.util.concurrent.RejectedExecutionHandler)
+
+@defaultMessage extend org.elasticsearch.threadpool.Scheduler.SafeScheduledThreadPoolExecutor instead which will properly bubble up Errors
+java.util.concurrent.ScheduledThreadPoolExecutor#<init>(int)
+java.util.concurrent.ScheduledThreadPoolExecutor#<init>(int, java.util.concurrent.ThreadFactory)
+java.util.concurrent.ScheduledThreadPoolExecutor#<init>(int, java.util.concurrent.RejectedExecutionHandler)
+java.util.concurrent.ScheduledThreadPoolExecutor#<init>(int, java.util.concurrent.ThreadFactory, java.util.concurrent.RejectedExecutionHandler)