Merge branch 'master' into jdk-13-ci

* master: (176 commits)
  Avoid unnecessary persistence of retention leases (elastic#42299)
  [ML][TEST] Fix limits in AutodetectMemoryLimitIT (elastic#42279)
  [ML Data Frame] Persist and restore checkpoint and position (elastic#41942)
  mute failing filerealm hash caching tests (elastic#42304)
  Safer Wait for Snapshot Success in ClusterPrivilegeTests (elastic#40943)
  Remove 7.0.2 (elastic#42282)
  Revert "Remove 7.0.2 (elastic#42282)"
  [DOCS] Copied note on slicing support to Slicing section. Closes 26114 (elastic#40426)
  Remove 7.0.2 (elastic#42282)
  Mute all ml_datafeed_crud rolling upgrade tests
  Move the FIPS configuration back to the build plugin (elastic#41989)
  Remove stray back tick that's messing up table format (elastic#41705)
  Add missing comma in code section (elastic#41678)
  add 7.1.1 and 6.8.1 versions (elastic#42253)
  Use spearate testkit dir for each run (elastic#42013)
  Add experimental and warnings to vector functions (elastic#42205)
  Fix version in tests since elastic#41906 was merged
  Bump version in BWC check after backport
  Prevent in-place downgrades and invalid upgrades (elastic#41731)
  Mute date_histo interval bwc test
  ...

.ci/init.gradle

@@ -6,6 +6,7 @@ if (System.env.ELASTIC_ARTIFACTORY_USERNAME == null || System.env.ELASTIC_ARTIFA
         settings.pluginManagement {
             repositories {
                 maven {
+                    name "artifactory-gradle-plugins"
                     url "https://artifactory.elstc.co/artifactory/gradle-plugins"
                     credentials {
                         username System.env.ELASTIC_ARTIFACTORY_USERNAME
@@ -21,6 +22,7 @@ if (System.env.ELASTIC_ARTIFACTORY_USERNAME == null || System.env.ELASTIC_ARTIFA
             buildscript {
                 repositories {
                     maven {
+                        name "artifactory-gradle-release"
                         url "https://artifactory.elstc.co/artifactory/gradle-release/"
                         credentials {
                             username System.env.ELASTIC_ARTIFACTORY_USERNAME
@@ -31,6 +33,7 @@ if (System.env.ELASTIC_ARTIFACTORY_USERNAME == null || System.env.ELASTIC_ARTIFA
             }
             repositories {
                 maven {
+                    name "artifactory-gradle-release"
                     url "https://artifactory.elstc.co/artifactory/gradle-release/"
                     credentials {
                         username System.env.ELASTIC_ARTIFACTORY_USERNAME

.ci/matrix-runtime-javas.yml

@@ -12,3 +12,4 @@ ES_RUNTIME_JAVA:
   - openjdk13
   - zulu11
   - zulu12
+  - corretto11

Vagrantfile

@@ -67,7 +67,10 @@ Vagrant.configure(2) do |config|
   'debian-8'.tap do |box|
     config.vm.define box, define_opts do |config|
       config.vm.box = 'elastic/debian-8-x86_64'
-      deb_common config, box
+      deb_common config, box, extra: <<-SHELL
+        # this sometimes gets a bad ip, and doesn't appear to be needed
+        rm -f /etc/apt/sources.list.d/http_debian_net_debian.list
+      SHELL
     end
   end
   'debian-9'.tap do |box|
@@ -158,13 +161,17 @@ def deb_common(config, name, extra: '')
       s.privileged = false
       s.inline = "sudo sed -i '/tty/!s/mesg n/tty -s \\&\\& mesg n/' /root/.profile"
   end
+  extra_with_lintian = <<-SHELL
+    #{extra}
+    install lintian
+  SHELL
   linux_common(
     config,
     name,
     update_command: 'apt-get update',
     update_tracking_file: '/var/cache/apt/archives/last_update',
     install_command: 'apt-get install -y',
-    extra: extra
+    extra: extra_with_lintian
   )
 end
 
@@ -249,10 +256,6 @@ def linux_common(config,
     touch /is_vagrant_vm # for consistency between linux and windows
   SHELL
 
-  config.vm.provision 'jdk-11', type: 'shell', inline: <<-SHELL
-    curl -sSL https://download.oracle.com/java/GA/jdk11/9/GPL/openjdk-11.0.2_linux-x64_bin.tar.gz | tar xz -C /opt/
-  SHELL
-
   # This prevents leftovers from previous tests using the
   # same VM from messing up the current test
   config.vm.provision 'clean es installs in tmp', run: 'always', type: 'shell', inline: <<-SHELL
@@ -348,11 +351,10 @@ def sh_install_deps(config,
       return 1
     }
     cat \<\<JAVA > /etc/profile.d/java_home.sh
-if [ -z "\\\$JAVA_HOME" ]; then
-  export JAVA_HOME=/opt/jdk-11.0.2
+if [ ! -z "\\\$JAVA_HOME" ]; then
+  export SYSTEM_JAVA_HOME=\\\$JAVA_HOME
+  unset JAVA_HOME
 fi
-export SYSTEM_JAVA_HOME=\\\$JAVA_HOME
-unset JAVA_HOME
 JAVA
     ensure tar
     ensure curl
@@ -409,16 +411,9 @@ def windows_common(config, name)
     $ps_prompt | Out-File $PsHome/Microsoft.PowerShell_profile.ps1
   SHELL
 
-  config.vm.provision 'windows-jdk-11', type: 'shell', inline: <<-SHELL
-    New-Item -ItemType Directory -Force -Path "C:/java"
-    Invoke-WebRequest "https://download.oracle.com/java/GA/jdk11/9/GPL/openjdk-11.0.2_windows-x64_bin.zip" -OutFile "C:/java/jdk-11.zip"
-    Expand-Archive -Path "C:/java/jdk-11.zip" -DestinationPath "C:/java/"
-  SHELL
-
   config.vm.provision 'set env variables', type: 'shell', inline: <<-SHELL
     $ErrorActionPreference = "Stop"
     [Environment]::SetEnvironmentVariable("PACKAGING_ARCHIVES", "C:/project/build/packaging/archives", "Machine")
-    [Environment]::SetEnvironmentVariable("SYSTEM_JAVA_HOME", "C:\java\jdk-11.0.2", "Machine")
     [Environment]::SetEnvironmentVariable("PACKAGING_TESTS", "C:/project/build/packaging/tests", "Machine")
     [Environment]::SetEnvironmentVariable("JAVA_HOME", $null, "Machine")
   SHELL

build.gradle

@@ -88,7 +88,7 @@ subprojects {
       }
       repositories {
         maven {
-          name = 'localTest'
+          name = 'test'
           url = "${rootProject.buildDir}/local-test-repo"
         }
       }
@@ -619,21 +619,6 @@ allprojects {
   } 
 }
 
-subprojects {
-    // Common config when running with a FIPS-140 runtime JVM
-    if (project.ext.has("inFipsJvm") && project.ext.inFipsJvm) {
-        tasks.withType(Test) {
-          systemProperty 'javax.net.ssl.trustStorePassword', 'password'
-          systemProperty 'javax.net.ssl.keyStorePassword', 'password'
-        }
-        project.pluginManager.withPlugin("elasticsearch.testclusters") {
-          project.testClusters.all {
-            systemProperty 'javax.net.ssl.trustStorePassword', 'password'
-            systemProperty 'javax.net.ssl.keyStorePassword', 'password'
-          }
-        }
-    }
-}
 
 
 

buildSrc/build.gradle

@@ -124,6 +124,7 @@ dependencies {
   compile 'com.avast.gradle:gradle-docker-compose-plugin:0.8.12'
   testCompile "junit:junit:${props.getProperty('junit')}"
   testCompile "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${props.getProperty('randomizedrunner')}"
+  testCompile 'com.github.tomakehurst:wiremock-jre8-standalone:2.23.2'
 }
 
 /*****************************************************************************
@@ -189,7 +190,7 @@ if (project != rootProject) {
   task integTest(type: Test) {
     // integration test requires the local testing repo for example plugin builds
     dependsOn project.rootProject.allprojects.collect {
-      it.tasks.matching { it.name == 'publishNebulaPublicationToLocalTestRepository'}
+      it.tasks.matching { it.name == 'publishNebulaPublicationToTestRepository'}
     }
     dependsOn setupLocalDownloads
     exclude "**/*Tests.class"

buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy

@@ -39,6 +39,7 @@ import org.gradle.api.artifacts.ModuleVersionIdentifier
 import org.gradle.api.artifacts.ProjectDependency
 import org.gradle.api.artifacts.ResolvedArtifact
 import org.gradle.api.artifacts.dsl.RepositoryHandler
+import org.gradle.api.artifacts.repositories.ArtifactRepository
 import org.gradle.api.artifacts.repositories.IvyArtifactRepository
 import org.gradle.api.artifacts.repositories.MavenArtifactRepository
 import org.gradle.api.credentials.HttpHeaderCredentials
@@ -115,6 +116,22 @@ class BuildPlugin implements Plugin<Project> {
         configureTestTasks(project)
         configurePrecommit(project)
         configureDependenciesInfo(project)
+
+        // Common config when running with a FIPS-140 runtime JVM
+        // Need to do it here to support external plugins 
+        if (project.ext.inFipsJvm) {
+            project.tasks.withType(Test) {
+                systemProperty 'javax.net.ssl.trustStorePassword', 'password'
+                systemProperty 'javax.net.ssl.keyStorePassword', 'password'
+            }
+            project.pluginManager.withPlugin("elasticsearch.testclusters") {
+                project.testClusters.all {
+                    systemProperty 'javax.net.ssl.trustStorePassword', 'password'
+                    systemProperty 'javax.net.ssl.keyStorePassword', 'password'
+                }
+            }
+        }
+
     }
 
 
@@ -589,11 +606,11 @@ class BuildPlugin implements Plugin<Project> {
         project.getRepositories().all { repository ->
             if (repository instanceof MavenArtifactRepository) {
                 final MavenArtifactRepository maven = (MavenArtifactRepository) repository
-                assertRepositoryURIUsesHttps(project, maven.getUrl())
+                assertRepositoryURIUsesHttps(maven, project, maven.getUrl())
                 repository.getArtifactUrls().each { uri -> assertRepositoryURIUsesHttps(project, uri) }
             } else if (repository instanceof IvyArtifactRepository) {
                 final IvyArtifactRepository ivy = (IvyArtifactRepository) repository
-                assertRepositoryURIUsesHttps(project, ivy.getUrl())
+                assertRepositoryURIUsesHttps(ivy, project, ivy.getUrl())
             }
         }
         RepositoryHandler repos = project.repositories
@@ -605,6 +622,7 @@ class BuildPlugin implements Plugin<Project> {
         }
         repos.jcenter()
         repos.ivy {
+            name "elasticsearch"
             url "https://artifacts.elastic.co/downloads"
             patternLayout {
                 artifact "elasticsearch/[module]-[revision](-[classifier]).[ext]"
@@ -633,9 +651,9 @@ class BuildPlugin implements Plugin<Project> {
         }
     }
 
-    private static void assertRepositoryURIUsesHttps(final Project project, final URI uri) {
+    private static void assertRepositoryURIUsesHttps(final ArtifactRepository repository, final Project project, final URI uri) {
         if (uri != null && uri.toURL().getProtocol().equals("http")) {
-            throw new GradleException("repository on project with path [${project.path}] is using http for artifacts on [${uri.toURL()}]")
+            throw new GradleException("repository [${repository.name}] on project with path [${project.path}] is using http for artifacts on [${uri.toURL()}]")
         }
     }
 

buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/PrecommitTasks.groovy

@@ -34,6 +34,9 @@ import org.gradle.api.plugins.quality.Checkstyle
 class PrecommitTasks {
 
     /** Adds a precommit task, which depends on non-test verification tasks. */
+
+    public static final String CHECKSTYLE_VERSION = '8.20'
+
     public static Task create(Project project, boolean includeDependencyLicenses) {
         project.configurations.create("forbiddenApisCliJar")
         project.dependencies {
@@ -213,7 +216,7 @@ class PrecommitTasks {
             configProperties = [
                 suppressions: checkstyleSuppressions
             ]
-            toolVersion = '8.10.1'
+            toolVersion = CHECKSTYLE_VERSION
         }
 
         project.tasks.withType(Checkstyle) { task ->

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestIntegTestTask.groovy

@@ -70,7 +70,7 @@ class RestIntegTestTask extends DefaultTask {
             project.testClusters {
                 "$name" {
                     distribution = 'INTEG_TEST'
-                    version = project.version
+                    version = VersionProperties.elasticsearch
                     javaHome = project.file(project.ext.runtimeJavaHome)
                 }
             }

buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/BatsOverVagrantTask.groovy

@@ -27,15 +27,15 @@ import org.gradle.api.tasks.Input
 public class BatsOverVagrantTask extends VagrantCommandTask {
 
     @Input
-    String remoteCommand
+    Object remoteCommand
 
     BatsOverVagrantTask() {
         command = 'ssh'
     }
 
-    void setRemoteCommand(String remoteCommand) {
+    void setRemoteCommand(Object remoteCommand) {
         this.remoteCommand = Objects.requireNonNull(remoteCommand)
-        setArgs(['--command', remoteCommand])
+        setArgs((Iterable<?>) ['--command', remoteCommand])
     }
 
     @Override