Merge branch 'master' into shrink-index-api-reformat

.ci/bwcVersions

@@ -9,5 +9,6 @@ BWC_VERSION:
   - "7.3.1"
   - "7.3.2"
   - "7.4.0"
+  - "7.4.1"
   - "7.5.0"
   - "8.0.0"

.ci/init.gradle

@@ -10,14 +10,18 @@ initscript {
     }
 }
 
-['VAULT_ADDR', 'VAULT_ROLE_ID', 'VAULT_SECRET_ID'].each {
-    if (System.env."$it" == null) {
-        throw new GradleException("$it must be set!")
+boolean USE_ARTIFACTORY=false
 
-    }
+if (System.getenv('VAULT_ADDR') == null) {
+    throw new GradleException("You must set the VAULT_ADDR environment variable to use this init script.")
+}
+
+if (System.getenv('VAULT_ROLE_ID') == null && System.getenv('VAULT_SECRET_ID') == null && System.getenv('VAULT_TOKEN') == null) {
+    throw new GradleException("You must set either the VAULT_ROLE_ID and VAULT_SECRET_ID environment variables, " +
+            "or the VAULT_TOKEN environment variable to use this init script.")
 }
 
-final String vaultToken =  new Vault(
+final String vaultToken = System.getenv('VAULT_TOKEN') ?: new Vault(
         new VaultConfig()
             .address(System.env.VAULT_ADDR)
             .engineVersion(1)
@@ -37,46 +41,52 @@ final Vault vault = new Vault(
 )
         .withRetries(5, 1000)
 
-final Map<String,String> artifactoryCredentials = vault.logical()
-        .read("secret/elasticsearch-ci/artifactory.elstc.co")
-        .getData();
 
-logger.info("Using elastic artifactory repos")
-Closure configCache = {
-    return {
-        name "artifactory-gradle-release"
-        url "https://artifactory.elstc.co/artifactory/gradle-release"
-        credentials {
-            username artifactoryCredentials.get("username")
-            password artifactoryCredentials.get("token")
+if (USE_ARTIFACTORY) {
+    final Map<String,String> artifactoryCredentials = vault.logical()
+            .read("secret/elasticsearch-ci/artifactory.elstc.co")
+            .getData();
+    logger.info("Using elastic artifactory repos")
+    Closure configCache = {
+        return {
+            name "artifactory-gradle-release"
+            url "https://artifactory.elstc.co/artifactory/gradle-release"
+            credentials {
+                username artifactoryCredentials.get("username")
+                password artifactoryCredentials.get("token")
+            }
         }
     }
-}
-settingsEvaluated { settings ->
-    settings.pluginManagement {
-        repositories {
-            maven configCache()
+    settingsEvaluated { settings ->
+        settings.pluginManagement {
+            repositories {
+                maven configCache()
+            }
         }
     }
-}
-projectsLoaded {
-    allprojects {
-        buildscript {
+    projectsLoaded {
+        allprojects {
+            buildscript {
+                repositories {
+                    maven configCache()
+                }
+            }
             repositories {
                 maven configCache()
             }
         }
-        repositories {
-            maven configCache()
-        }
     }
+}
+
+projectsLoaded {
     rootProject {
         project.pluginManager.withPlugin('com.gradle.build-scan') {
             buildScan.server = 'https://gradle-enterprise.elastic.co'
         }
     }
 }
 
+
 final String buildCacheUrl = System.getProperty('org.elasticsearch.build.cache.url')
 final boolean buildCachePush = Boolean.valueOf(System.getProperty('org.elasticsearch.build.cache.push', 'false'))
 

CONTRIBUTING.md

@@ -109,17 +109,16 @@ and `JAVA11_HOME` available so that the tests can pass.
 Elasticsearch uses the Gradle wrapper for its build. You can execute Gradle
 using the wrapper via the `gradlew` script in the root of the repository.
 
-We support development in the Eclipse and IntelliJ IDEs. For Eclipse, the
-minimum version that we support is [Eclipse Oxygen][eclipse] (version 4.7). For
-IntelliJ, the minimum version that we support is [IntelliJ 2017.2][intellij].
+We support development in the Eclipse and IntelliJ IDEs. 
+For Eclipse, the minimum version that we support is [4.13][eclipse]. 
+For IntelliJ, the minimum version that we support is [IntelliJ 2017.2][intellij].
 
 ### Configuring IDEs And Running Tests
 
 Eclipse users can automatically configure their IDE: `./gradlew eclipse`
-then `File: Import: Existing Projects into Workspace`. Select the
-option `Search for nested projects`. Additionally you will want to
-ensure that Eclipse is using 2048m of heap by modifying `eclipse.ini`
-accordingly to avoid GC overhead errors.
+then `File: Import: Gradle : Existing Gradle Project`.
+Additionally you will want to ensure that Eclipse is using 2048m of heap by modifying 
+`eclipse.ini` accordingly to avoid GC overhead and OOM errors.
 
 IntelliJ users can automatically configure their IDE: `./gradlew idea`
 then `File->New Project From Existing Sources`. Point to the root of
@@ -155,19 +154,68 @@ For Eclipse, go to `Preferences->Java->Installed JREs` and add `-ea` to
 
 ### Java Language Formatting Guidelines
 
+Java files in the Elasticsearch codebase are formatted with the Eclipse JDT
+formatter, using the [Spotless
+Gradle](https://github.com/diffplug/spotless/tree/master/plugin-gradle)
+plugin. This plugin is configured on a project-by-project basis, via
+`build.gradle` in the root of the repository. So long as at least one
+project is configured, the formatting check can be run explicitly with:
+
+    ./gradlew spotlessJavaCheck
+
+The code can be formatted with:
+
+    ./gradlew spotlessApply
+
+These tasks can also be run for specific subprojects, e.g.
+
+    ./gradlew server:spotlessJavaCheck
+
 Please follow these formatting guidelines:
 
 * Java indent is 4 spaces
 * Line width is 140 characters
-* Lines of code surrounded by `// tag` and `// end` comments are included in the
-documentation and should only be 76 characters wide not counting
-leading indentation
-* The rest is left to Java coding standards
-* Disable “auto-format on save” to prevent unnecessary format changes. This makes reviews much harder as it generates unnecessary formatting changes. If your IDE supports formatting only modified chunks that is fine to do.
-* Wildcard imports (`import foo.bar.baz.*`) are forbidden and will cause the build to fail. This can be done automatically by your IDE:
- * Eclipse: `Preferences->Java->Code Style->Organize Imports`. There are two boxes labeled "`Number of (static )? imports needed for .*`". Set their values to 99999 or some other absurdly high value.
- * IntelliJ: `Preferences/Settings->Editor->Code Style->Java->Imports`. There are two configuration options: `Class count to use import with '*'` and `Names count to use static import with '*'`. Set their values to 99999 or some other absurdly high value.
-* Don't worry too much about import order. Try not to change it but don't worry about fighting your IDE to stop it from doing so.
+* Lines of code surrounded by `// tag` and `// end` comments are included
+  in the documentation and should only be 76 characters wide not counting
+  leading indentation
+* Wildcard imports (`import foo.bar.baz.*`) are forbidden and will cause
+  the build to fail. This can be done automatically by your IDE:
+   * Eclipse: `Preferences->Java->Code Style->Organize Imports`. There are
+     two boxes labeled "`Number of (static )? imports needed for .*`". Set
+     their values to 99999 or some other absurdly high value.
+   * IntelliJ: `Preferences/Settings->Editor->Code Style->Java->Imports`.
+     There are two configuration options: `Class count to use import with
+     '*'` and `Names count to use static import with '*'`. Set their values
+     to 99999 or some other absurdly high value.
+
+#### Editor / IDE Support
+
+Eclipse IDEs can import the file [elasticsearch.eclipseformat.xml]
+directly.
+
+IntelliJ IDEs can
+[import](https://blog.jetbrains.com/idea/2014/01/intellij-idea-13-importing-code-formatter-settings-from-eclipse/)
+the same settings file, and / or use the [Eclipse Code
+Formatter](https://plugins.jetbrains.com/plugin/6546-eclipse-code-formatter)
+plugin.
+
+You can also tell Spotless to [format a specific
+file](https://github.com/diffplug/spotless/tree/master/plugin-gradle#can-i-apply-spotless-to-specific-files)
+from the command line.
+
+#### Formatting failures
+
+Sometimes Spotless will report a "misbehaving rule which can't make up its
+mind" and will recommend enabling the `paddedCell()` setting. If you
+enabled this settings and run the format check again,
+Spotless will write files to
+`$PROJECT/build/spotless-diagnose-java/` to aid diagnosis. It writes
+different copies of the formatted files, so that you can see how they
+differ and infer what is the problem.
+
+The `paddedCell() option is disabled for normal operation in order to
+detect any misbehaviour. You can enabled the option from the command line
+by running Gradle with `-Dspotless.paddedcell`.
 
 ### License Headers
 
@@ -388,6 +436,6 @@ Finally, we require that you run `./gradlew check` before submitting a
 non-documentation contribution. This is mentioned above, but it is worth
 repeating in this section because it has come up in this context.
 
-[eclipse]: http://www.eclipse.org/community/eclipse_newsletter/2017/june/
+[eclipse]: https://download.eclipse.org/eclipse/downloads/drops4/R-4.13-201909161045/
 [intellij]: https://blog.jetbrains.com/idea/2017/07/intellij-idea-2017-2-is-here-smart-sleek-and-snappy/
 [shadow-plugin]: https://github.com/johnrengelman/shadow

build.gradle

@@ -35,6 +35,7 @@ plugins {
     id 'com.gradle.build-scan' version '2.4'
     id 'lifecycle-base'
     id 'elasticsearch.global-build-info'
+    id "com.diffplug.gradle.spotless" version "3.24.2" apply false
 }
 
 apply plugin: 'nebula.info-scm'
@@ -98,6 +99,34 @@ subprojects {
   plugins.withType(BuildPlugin).whenPluginAdded {
     project.licenseFile = project.rootProject.file('licenses/APACHE-LICENSE-2.0.txt')
     project.noticeFile = project.rootProject.file('NOTICE.txt')
+
+    // Projects that should be formatted and checked with Spotless are
+    // listed here, by project path. Once the number of formatted projects
+    // is greater than the number of unformatted projects, this can be
+    // switched to an exclude list, and eventualy removed completely.
+    def projectPathsToFormat = [
+      // ':build-tools'
+    ]
+
+    if (projectPathsToFormat.contains(project.path)) {
+      project.apply plugin: "com.diffplug.gradle.spotless"
+
+      spotless {
+        java {
+
+          removeUnusedImports()
+          eclipse().configFile rootProject.file('.eclipseformat.xml')
+          trimTrailingWhitespace()
+
+          // See CONTRIBUTING.md for details of when to enabled this.
+          if (System.getProperty('spotless.paddedcell') != null) {
+            paddedCell()
+          }
+        }
+      }
+
+      precommit.dependsOn 'spotlessJavaCheck'
+    }
   }
 }
 
@@ -352,29 +381,19 @@ allprojects {
   }
 
   plugins.withType(JavaBasePlugin) {
-    File eclipseBuild = project.file('build-eclipse')
-    eclipse.classpath.defaultOutputDir = eclipseBuild
-    if (isEclipse) {
-      // set this so generated dirs will be relative to eclipse build
-      project.buildDir = eclipseBuild
-      // Work around https://docs.gradle.org/current/userguide/java_gradle_plugin.html confusing Eclipse by the metadata
-      // it adds to the  classpath
-      project.file("$buildDir/pluginUnderTestMetadata").mkdirs()
-    }
+    eclipse.classpath.defaultOutputDir = file('build-eclipse')
     eclipse.classpath.file.whenMerged { classpath ->
       // give each source folder a unique corresponding output folder
       int i = 0;
       classpath.entries.findAll { it instanceof SourceFolder }.each { folder ->
         i++;
-        // this is *NOT* a path or a file.
         folder.output = "build-eclipse/" + i
       }
     }
   }
 
-  File licenseHeaderFile;
-  String prefix = ':x-pack';
-
+  File licenseHeaderFile
+  String prefix = ':x-pack'
   if (Os.isFamily(Os.FAMILY_WINDOWS)) {
     prefix = prefix.replace(':', '_')
   }

buildSrc/build.gradle

@@ -16,9 +16,8 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-import org.gradle.util.GradleVersion
 
-import java.util.regex.Matcher
+import org.gradle.util.GradleVersion
 
 plugins {
   id 'java-gradle-plugin'
@@ -107,11 +106,13 @@ dependencies {
   compile localGroovy()
 
   compile 'commons-codec:commons-codec:1.12'
+  compile 'org.apache.commons:commons-compress:1.19'
 
   compile 'com.netflix.nebula:gradle-extra-configurations-plugin:3.0.3'
   compile 'com.netflix.nebula:nebula-publishing-plugin:4.4.4'
+  compile 'com.netflix.nebula:gradle-info-plugin:5.1.0'
+  compile 'org.eclipse.jgit:org.eclipse.jgit:5.5.0.201909110433-r'
   compile 'com.netflix.nebula:gradle-info-plugin:3.0.3'
-  compile 'org.eclipse.jgit:org.eclipse.jgit:3.2.0.201312181205-r'
   compile 'com.perforce:p4java:2012.3.551082' // THIS IS SUPPOSED TO BE OPTIONAL IN THE FUTURE....
   compile 'org.apache.rat:apache-rat:0.11'
   compile "org.elasticsearch:jna:4.5.1"
@@ -121,6 +122,7 @@ dependencies {
   testCompile "junit:junit:${props.getProperty('junit')}"
   testCompile "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${props.getProperty('randomizedrunner')}"
   testCompile 'com.github.tomakehurst:wiremock-jre8-standalone:2.23.2'
+  testCompile 'org.mockito:mockito-core:1.9.5'
   minimumRuntimeCompile "junit:junit:${props.getProperty('junit')}"
   minimumRuntimeCompile localGroovy()
   minimumRuntimeCompile gradleApi()
@@ -190,40 +192,6 @@ if (project != rootProject) {
       }
   }
 
-  String localDownloads = "${rootProject.buildDir}/local-downloads"
-  task setupLocalDownloads(type:Copy) {
-    from configurations.distribution
-    into localDownloads
-  }
-
-  task integTest(type: Test) {
-    // integration test requires the local testing repo for example plugin builds
-    dependsOn project.rootProject.allprojects.collect {
-      it.tasks.matching { it.name == 'publishNebulaPublicationToTestRepository'}
-    }
-    dependsOn setupLocalDownloads
-    exclude "**/*Tests.class"
-    inputs.dir(file("src/testKit"))
-    // tell BuildExamplePluginsIT where to find the example plugins
-    systemProperty (
-            'test.build-tools.plugin.examples',
-            files(
-                    project(':example-plugins').subprojects.collect { it.projectDir }
-            ).asPath,
-    )
-    systemProperty 'test.local-test-repo-path', "${rootProject.buildDir}/local-test-repo"
-    systemProperty 'test.local-test-downloads-path', localDownloads
-    systemProperty 'test.version_under_test', version
-    Matcher isLuceneSnapshot = (/\w+-snapshot-([a-z0-9]+)/ =~ versions.lucene)
-    if (isLuceneSnapshot) {
-      systemProperty 'test.lucene-snapshot-revision', isLuceneSnapshot[0][1]
-    }
-    maxParallelForks System.getProperty('tests.jvms', project.rootProject.ext.defaultParallel.toString()) as Integer
-    // These tests run Gradle which doesn't have FIPS support
-    onlyIf { project.inFipsJvm == false }
-  }
-  check.dependsOn(integTest)
-
   // TODO: re-enable once randomizedtesting gradle code is published and removed from here
   licenseHeaders.enabled = false
 
@@ -246,6 +214,14 @@ if (project != rootProject) {
     }
   }
 
+  task integTest(type: Test) {
+    inputs.dir(file("src/testKit")).withPropertyName("testkit dir").withPathSensitivity(PathSensitivity.RELATIVE)
+    systemProperty 'test.version_under_test', version
+    onlyIf { project.inFipsJvm == false }
+    maxParallelForks = System.getProperty('tests.jvms', project.rootProject.ext.defaultParallel.toString()) as Integer
+  }
+  check.dependsOn(integTest)
+
   /*
    * We alread configure publication and we don't need or want this one that
    * comes from the java-gradle-plugin.