Merge branch 'master' into thread-pool-info-migration

* master: (96 commits)
  TEST: Mute testEnsureWeReconnect
  Mute full cluster restart test recovery test
  REST high-level client: add support for Indices Update Settings API [take 2] (elastic#29327)
  Plugins: Fix native controller confirmation for non-meta plugin (elastic#29434)
  Remove PipelineExecutionService#executeIndexRequest (elastic#29537)
  Fix overflow error in parsing of long geohashes (elastic#29418)
  Remove unused index.ttl.disable_purge setting (elastic#29527)
  FullClusterRestartIT.testRecovery should wait for all initializing shards
  Build: Fail if any libs depend on non-core libs (elastic#29336)
  [TEST] REST client request without leading '/' (elastic#29471)
  Using ObjectParser in UpdateRequest (elastic#29293)
  Prevent accidental changes of default values (elastic#29528)
  [Docs] Add definitions to glossary  (elastic#29127)
  Avoid self-deadlock in the translog (elastic#29520)
  Minor cleanup in NodeInfo.groovy
  Lazy configure build tasks that require older JDKs (elastic#29519)
  Simplify snapshot check in root build file
  Make NodeInfo#nodeVersion strongly-typed as Version (elastic#29515)
  Enable license header exclusions (elastic#29379)
  Use proper Java version for BWC builds (elastic#29493)
  ...

NOTICE.txt

@@ -1,5 +1,5 @@
 Elasticsearch
-Copyright 2009-2017 Elasticsearch
+Copyright 2009-2018 Elasticsearch
 
 This product includes software developed by The Apache Software
 Foundation (http://www.apache.org/).

TESTING.asciidoc

@@ -498,6 +498,13 @@ will contain your change.
 . Push both branches to your remote repository.
 . Run the tests with `./gradlew check -Dtests.bwc.remote=${remote} -Dtests.bwc.refspec.5.x=index_req_bwc_5.x`.
 
+== Skip fetching latest
+
+For some BWC testing scenarios, you want to use the local clone of the
+repository without fetching latest. For these use cases, you can set the system
+property `tests.bwc.git_fetch_latest` to `false` and the BWC builds will skip
+fetching the latest from the remote.
+
 == Test coverage analysis
 
 Generating test coverage reports for Elasticsearch is currently not possible through Gradle.

build.gradle

@@ -36,7 +36,7 @@ import java.security.MessageDigest
 // common maven publishing configuration
 subprojects {
   group = 'org.elasticsearch'
-  version = VersionProperties.elasticsearch
+  version = VersionProperties.elasticsearch.toString()
   description = "Elasticsearch subproject ${project.path}"
 }
 
@@ -80,7 +80,7 @@ configure(subprojects.findAll { it.projectDir.toPath().startsWith(rootPath) }) {
  * in a branch if there are only betas and rcs in the branch so we have
  * *something* to test against. */
 VersionCollection versions = new VersionCollection(file('server/src/main/java/org/elasticsearch/Version.java').readLines('UTF-8'))
-if (versions.currentVersion.toString() != VersionProperties.elasticsearch) {
+if (versions.currentVersion != VersionProperties.elasticsearch) {
   throw new GradleException("The last version in Versions.java [${versions.currentVersion}] does not match " +
           "VersionProperties.elasticsearch [${VersionProperties.elasticsearch}]")
 }
@@ -231,6 +231,23 @@ subprojects {
     }
   }
 
+  /*
+   * Gradle only resolve project substitutions during dependency resolution but
+   * we sometimes want to do the resolution at other times. This creates a
+   * convenient method we can call to do it.
+   */
+  ext.dependencyToProject = { Dependency dep ->
+    if (dep instanceof ProjectDependency) {
+      return dep.dependencyProject
+    } else {
+      String substitution = projectSubstitutions.get("${dep.group}:${dep.name}:${dep.version}")
+      if (substitution != null) {
+        return findProject(substitution)
+      }
+      return null
+    }
+  }
+
   project.afterEvaluate {
     configurations.all {
       resolutionStrategy.dependencySubstitution { DependencySubstitutions subs ->
@@ -245,15 +262,15 @@ subprojects {
     // other packages (e.g org.elasticsearch.client) will point to server rather than
     // their own artifacts.
     if (project.plugins.hasPlugin(BuildPlugin)) {
-      String artifactsHost = VersionProperties.elasticsearch.endsWith("-SNAPSHOT") ? "https://snapshots.elastic.co" : "https://artifacts.elastic.co"
+      String artifactsHost = VersionProperties.elasticsearch.isSnapshot() ? "https://snapshots.elastic.co" : "https://artifacts.elastic.co"
       Closure sortClosure = { a, b -> b.group <=> a.group }
       Closure depJavadocClosure = { dep ->
         if (dep.group != null && dep.group.startsWith('org.elasticsearch')) {
-          String substitution = project.ext.projectSubstitutions.get("${dep.group}:${dep.name}:${dep.version}")
-          if (substitution != null) {
-            project.javadoc.dependsOn substitution + ':javadoc'
+          Project upstreamProject = dependencyToProject(dep)
+          if (upstreamProject != null) {
+            project.javadoc.dependsOn "${upstreamProject.path}:javadoc"
             String artifactPath = dep.group.replaceAll('\\.', '/') + '/' + dep.name.replaceAll('\\.', '/') + '/' + dep.version
-            project.javadoc.options.linksOffline artifactsHost + "/javadoc/" + artifactPath, "${project.project(substitution).buildDir}/docs/javadoc/"
+            project.javadoc.options.linksOffline artifactsHost + "/javadoc/" + artifactPath, "${upstreamProject.buildDir}/docs/javadoc/"
           }
         }
       }
@@ -275,17 +292,7 @@ gradle.projectsEvaluated {
     }
     configurations.all {
       dependencies.all { Dependency dep ->
-        Project upstreamProject = null
-        if (dep instanceof ProjectDependency) {
-          upstreamProject = dep.dependencyProject
-        } else {
-          // gradle doesn't apply substitutions until resolve time, so they won't
-          // show up as a ProjectDependency above
-          String substitution = projectSubstitutions.get("${dep.group}:${dep.name}:${dep.version}")
-          if (substitution != null) {
-            upstreamProject = findProject(substitution)
-          }
-        }
+        Project upstreamProject = dependencyToProject(dep)
         if (upstreamProject != null) {
           if (project.path == upstreamProject.path) {
             // TODO: distribution integ tests depend on themselves (!), fix that

buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy

@@ -97,6 +97,12 @@ class BuildPlugin implements Plugin<Project> {
             String compilerJavaHome = findCompilerJavaHome()
             String runtimeJavaHome = findRuntimeJavaHome(compilerJavaHome)
             File gradleJavaHome = Jvm.current().javaHome
+
+            final Map<Integer, String> javaVersions = [:]
+            for (int version = 7; version <= Integer.parseInt(minimumCompilerVersion.majorVersion); version++) {
+                javaVersions.put(version, findJavaHome(version));
+            }
+
             String javaVendor = System.getProperty('java.vendor')
             String javaVersion = System.getProperty('java.version')
             String gradleJavaVersionDetails = "${javaVendor} ${javaVersion}" +
@@ -158,10 +164,32 @@ class BuildPlugin implements Plugin<Project> {
                 throw new GradleException(message)
             }
 
+            for (final Map.Entry<Integer, String> javaVersionEntry : javaVersions.entrySet()) {
+                final String javaHome = javaVersionEntry.getValue()
+                if (javaHome == null) {
+                    continue
+                }
+                JavaVersion javaVersionEnum = JavaVersion.toVersion(findJavaSpecificationVersion(project, javaHome))
+                final JavaVersion expectedJavaVersionEnum
+                final int version = javaVersionEntry.getKey()
+                if (version < 9) {
+                    expectedJavaVersionEnum = JavaVersion.toVersion("1." + version)
+                } else {
+                    expectedJavaVersionEnum = JavaVersion.toVersion(Integer.toString(version))
+                }
+                if (javaVersionEnum != expectedJavaVersionEnum) {
+                    final String message =
+                            "the environment variable JAVA" + version + "_HOME must be set to a JDK installation directory for Java" +
+                                    " ${expectedJavaVersionEnum} but is [${javaHome}] corresponding to [${javaVersionEnum}]"
+                    throw new GradleException(message)
+                }
+            }
+
             project.rootProject.ext.compilerJavaHome = compilerJavaHome
             project.rootProject.ext.runtimeJavaHome = runtimeJavaHome
             project.rootProject.ext.compilerJavaVersion = compilerJavaVersionEnum
             project.rootProject.ext.runtimeJavaVersion = runtimeJavaVersionEnum
+            project.rootProject.ext.javaVersions = javaVersions
             project.rootProject.ext.buildChecksDone = true
         }
 
@@ -173,6 +201,7 @@ class BuildPlugin implements Plugin<Project> {
         project.ext.runtimeJavaHome = project.rootProject.ext.runtimeJavaHome
         project.ext.compilerJavaVersion = project.rootProject.ext.compilerJavaVersion
         project.ext.runtimeJavaVersion = project.rootProject.ext.runtimeJavaVersion
+        project.ext.javaVersions = project.rootProject.ext.javaVersions
     }
 
     private static String findCompilerJavaHome() {
@@ -188,6 +217,27 @@ class BuildPlugin implements Plugin<Project> {
         return javaHome
     }
 
+    private static String findJavaHome(int version) {
+        return System.getenv('JAVA' + version + '_HOME')
+    }
+
+    /**
+     * Get Java home for the project for the specified version. If the specified version is not configured, an exception with the specified
+     * message is thrown.
+     *
+     * @param project the project
+     * @param version the version of Java home to obtain
+     * @param message the exception message if Java home for the specified version is not configured
+     * @return Java home for the specified version
+     * @throws GradleException if Java home for the specified version is not configured
+     */
+    static String getJavaHome(final Project project, final int version, final String message) {
+        if (project.javaVersions.get(version) == null) {
+            throw new GradleException(message)
+        }
+        return project.javaVersions.get(version)
+    }
+
     private static String findRuntimeJavaHome(final String compilerJavaHome) {
         assert compilerJavaHome != null
         return System.getenv('RUNTIME_JAVA_HOME') ?: compilerJavaHome
@@ -517,17 +567,18 @@ class BuildPlugin implements Plugin<Project> {
             jarTask.destinationDir = new File(project.buildDir, 'distributions')
             // fixup the jar manifest
             jarTask.doFirst {
-                boolean isSnapshot = VersionProperties.elasticsearch.endsWith("-SNAPSHOT");
-                String version = VersionProperties.elasticsearch;
-                if (isSnapshot) {
-                    version = version.substring(0, version.length() - 9)
-                }
+                final Version versionWithoutSnapshot = new Version(
+                        VersionProperties.elasticsearch.major,
+                        VersionProperties.elasticsearch.minor,
+                        VersionProperties.elasticsearch.revision,
+                        VersionProperties.elasticsearch.suffix,
+                        false)
                 // this doFirst is added before the info plugin, therefore it will run
                 // after the doFirst added by the info plugin, and we can override attributes
                 jarTask.manifest.attributes(
-                        'X-Compile-Elasticsearch-Version': version,
+                        'X-Compile-Elasticsearch-Version': versionWithoutSnapshot,
                         'X-Compile-Lucene-Version': VersionProperties.lucene,
-                        'X-Compile-Elasticsearch-Snapshot': isSnapshot,
+                        'X-Compile-Elasticsearch-Snapshot': VersionProperties.elasticsearch.isSnapshot(),
                         'Build-Date': ZonedDateTime.now(ZoneOffset.UTC),
                         'Build-Java-Version': project.compilerJavaVersion)
                 if (jarTask.manifest.attributes.containsKey('Change') == false) {

buildSrc/src/main/groovy/org/elasticsearch/gradle/Version.groovy

@@ -74,20 +74,36 @@ public class Version {
         return "${major}.${minor}.${revision}${suffix}${snapshotStr}"
     }
 
+    public boolean before(Version compareTo) {
+        return id < compareTo.id
+    }
+
     public boolean before(String compareTo) {
-        return id < fromString(compareTo).id
+        return before(fromString(compareTo))
+    }
+
+    public boolean onOrBefore(Version compareTo) {
+        return id <= compareTo.id
     }
 
     public boolean onOrBefore(String compareTo) {
-        return id <= fromString(compareTo).id
+        return onOrBefore(fromString(compareTo))
+    }
+
+    public boolean onOrAfter(Version compareTo) {
+        return id >= compareTo.id
     }
 
     public boolean onOrAfter(String compareTo) {
-        return id >= fromString(compareTo).id
+        return onOrAfter(fromString(compareTo))
+    }
+
+    public boolean after(Version compareTo) {
+        return id > compareTo.id
     }
 
     public boolean after(String compareTo) {
-        return id > fromString(compareTo).id
+        return after(fromString(compareTo))
     }
 
     public boolean onOrBeforeIncludingSuffix(Version otherVersion) {

buildSrc/src/main/groovy/org/elasticsearch/gradle/VersionProperties.groovy

@@ -22,7 +22,7 @@ package org.elasticsearch.gradle
  * Accessor for shared dependency versions used by elasticsearch, namely the elasticsearch and lucene versions.
  */
 class VersionProperties {
-    static final String elasticsearch
+    static final Version elasticsearch
     static final String lucene
     static final Map<String, String> versions = new HashMap<>()
     static {
@@ -32,7 +32,7 @@ class VersionProperties {
             throw new RuntimeException('/version.properties resource missing')
         }
         props.load(propsStream)
-        elasticsearch = props.getProperty('elasticsearch')
+        elasticsearch = Version.fromString(props.getProperty('elasticsearch'))
         lucene = props.getProperty('lucene')
         for (String property : props.stringPropertyNames()) {
             versions.put(property, props.getProperty(property))

buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/DocsTestPlugin.groovy

@@ -41,7 +41,7 @@ public class DocsTestPlugin extends RestTestPlugin {
              * to the version being built for testing but needs to resolve to
              * the last released version for docs. */
             '\\{version\\}':
-                VersionProperties.elasticsearch.replace('-SNAPSHOT', ''),
+                VersionProperties.elasticsearch.toString().replace('-SNAPSHOT', ''),
             '\\{lucene_version\\}' : VersionProperties.lucene.replaceAll('-snapshot-\\w+$', ''),
         ]
         Task listSnippets = project.tasks.create('listSnippets', SnippetsTask)

buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginPropertiesTask.groovy

@@ -77,7 +77,7 @@ class PluginPropertiesTask extends Copy {
             'name': extension.name,
             'description': extension.description,
             'version': stringSnap(extension.version),
-            'elasticsearchVersion': stringSnap(VersionProperties.elasticsearch),
+            'elasticsearchVersion': stringSnap(VersionProperties.elasticsearch.toString()),
             'javaVersion': project.targetCompatibility as String,
             'classname': extension.classname,
             'extendedPlugins': extension.extendedPlugins.join(','),

buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/LicenseHeadersTask.groovy

@@ -49,6 +49,14 @@ public class LicenseHeadersTask extends AntTask {
     @Input
     List<String> approvedLicenses = ['Apache', 'Generated']
 
+    /**
+     * Files that should be excluded from the license header check. Use with extreme care, only in situations where the license on the
+     * source file is compatible with the codebase but we do not want to add the license to the list of approved headers (to avoid the
+     * possibility of inadvertently using the license on our own source files).
+     */
+    @Input
+    List<String> excludes = []
+
     /**
      * Additional license families that may be found. The key is the license category name (5 characters),
      * followed by the family name and the value list of patterns to search for.
@@ -95,7 +103,7 @@ public class LicenseHeadersTask extends AntTask {
                for (File dir: dirSet.srcDirs) {
                    // sometimes these dirs don't exist, e.g. site-plugin has no actual java src/main...
                    if (dir.exists()) {
-                       ant.fileset(dir: dir)
+                       ant.fileset(dir: dir, excludes: excludes.join(' '))
                    }
                }
             }

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy

@@ -18,6 +18,7 @@
  */
 package org.elasticsearch.gradle.test
 
+import org.elasticsearch.gradle.Version
 import org.gradle.api.GradleException
 import org.gradle.api.Project
 import org.gradle.api.tasks.Input
@@ -37,7 +38,7 @@ class ClusterConfiguration {
     int numBwcNodes = 0
 
     @Input
-    String bwcVersion = null
+    Version bwcVersion = null
 
     @Input
     int httpPort = 0
@@ -141,6 +142,8 @@ class ClusterConfiguration {
 
     Map<String, String> keystoreSettings = new HashMap<>()
 
+    Map<String, Object> keystoreFiles = new HashMap<>()
+
     // map from destination path, to source file
     Map<String, Object> extraConfigFiles = new HashMap<>()
 
@@ -167,6 +170,15 @@ class ClusterConfiguration {
         keystoreSettings.put(name, value)
     }
 
+    /**
+     * Adds a file to the keystore. The name is the secure setting name, and the sourceFile
+     * is anything accepted by project.file()
+     */
+    @Input
+    void keystoreFile(String name, Object sourceFile) {
+        keystoreFiles.put(name, sourceFile)
+    }
+
     @Input
     void plugin(String path) {
         Project pluginProject = project.project(path)