Add Docker packaging tests on 7.x

.ci/dockerOnLinuxExclusions

@@ -0,0 +1,12 @@
+# This file specifies the Linux OS versions on which we can't build and
+# test Docker images for some reason. These values correspond to ID and
+# VERSION_ID from /etc/os-release, and a matching value will cause the
+# Docker tests to be skipped on that OS. If /etc/os-release doesn't exist
+# (as is the case on centos-6, for example) then that OS will again be
+# excluded.
+centos-6
+debian-8
+opensuse-15-1
+ol-6.10
+ol-7.7
+sles-12

.ci/os.sh

@@ -6,6 +6,16 @@ if which zypper > /dev/null ; then
     sudo zypper install -y insserv-compat
 fi
 
+if [ -e /etc/sysctl.d/99-gce.conf ]; then
+  # The GCE defaults disable IPv4 forwarding, which breaks the Docker
+  # build. Workaround this by renaming the file so that it is executed
+  # earlier than our own overrides.
+  #
+  # This ultimately needs to be fixed at the image level - see infra
+  # issue 15654.
+  sudo mv /etc/sysctl.d/99-gce.conf /etc/sysctl.d/98-gce.conf
+fi
+
 # Required by bats
 sudo touch /etc/is_vagrant_vm
 sudo useradd vagrant

Vagrantfile

@@ -1,5 +1,5 @@
 # -*- mode: ruby -*-
-# vi: set ft=ruby :
+# vim: ft=ruby ts=2 sw=2 sts=2 et:
 
 # This Vagrantfile exists to test packaging. Read more about its use in the
 # vagrant section in TESTING.asciidoc.
@@ -63,6 +63,7 @@ Vagrant.configure(2) do |config|
         # Install Jayatana so we can work around it being present.
         [ -f /usr/share/java/jayatanaag.jar ] || install jayatana
       SHELL
+      ubuntu_docker config
     end
   end
   'ubuntu-1804'.tap do |box|
@@ -72,6 +73,7 @@ Vagrant.configure(2) do |config|
        # Install Jayatana so we can work around it being present.
        [ -f /usr/share/java/jayatanaag.jar ] || install jayatana
       SHELL
+      ubuntu_docker config
     end
   end
   # Wheezy's backports don't contain Openjdk 8 and the backflips
@@ -90,6 +92,7 @@ Vagrant.configure(2) do |config|
     config.vm.define box, define_opts do |config|
       config.vm.box = 'elastic/debian-9-x86_64'
       deb_common config, box
+      deb_docker config
     end
   end
   'centos-6'.tap do |box|
@@ -102,6 +105,7 @@ Vagrant.configure(2) do |config|
     config.vm.define box, define_opts do |config|
       config.vm.box = 'elastic/centos-7-x86_64'
       rpm_common config, box
+      rpm_docker config
     end
   end
   'oel-6'.tap do |box|
@@ -120,12 +124,14 @@ Vagrant.configure(2) do |config|
     config.vm.define box, define_opts do |config|
       config.vm.box = 'elastic/fedora-28-x86_64'
       dnf_common config, box
+      dnf_docker config
     end
   end
   'fedora-29'.tap do |box|
     config.vm.define box, define_opts do |config|
-      config.vm.box = 'elastic/fedora-28-x86_64'
+      config.vm.box = 'elastic/fedora-29-x86_64'
       dnf_common config, box
+      dnf_docker config
     end
   end
   'opensuse-42'.tap do |box|
@@ -188,6 +194,67 @@ def deb_common(config, name, extra: '')
   )
 end
 
+def ubuntu_docker(config)
+  config.vm.provision 'install Docker using apt', type: 'shell', inline: <<-SHELL
+    # Install packages to allow apt to use a repository over HTTPS
+    apt-get install -y \
+      apt-transport-https \
+      ca-certificates \
+      curl \
+      gnupg2 \
+      software-properties-common
+
+    # Add Docker’s official GPG key
+    curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add -
+
+    # Set up the stable Docker repository
+    add-apt-repository \
+      "deb [arch=amd64] https://download.docker.com/linux/ubuntu \
+      $(lsb_release -cs) \
+      stable"
+
+    # Install Docker. Unlike Fedora and CentOS, this also start the daemon.
+    apt-get update
+    apt-get install -y docker-ce docker-ce-cli containerd.io
+
+    # Add vagrant to the Docker group, so that it can run commands
+    usermod -aG docker vagrant
+
+    # Enable IPv4 forwarding
+    sed -i '/net.ipv4.ip_forward/s/^#//' /etc/sysctl.conf
+    systemctl restart networking
+  SHELL
+end
+
+
+def deb_docker(config)
+  config.vm.provision 'install Docker using apt', type: 'shell', inline: <<-SHELL
+    # Install packages to allow apt to use a repository over HTTPS
+    apt-get install -y \
+      apt-transport-https \
+      ca-certificates \
+      curl \
+      gnupg2 \
+      software-properties-common
+
+    # Add Docker’s official GPG key
+    curl -fsSL https://download.docker.com/linux/debian/gpg | apt-key add -
+
+    # Set up the stable Docker repository
+    add-apt-repository \
+      "deb [arch=amd64] https://download.docker.com/linux/debian \
+      $(lsb_release -cs) \
+      stable"
+
+    # Install Docker. Unlike Fedora and CentOS, this also start the daemon.
+    apt-get update
+    apt-get install -y docker-ce docker-ce-cli containerd.io
+
+    # Add vagrant to the Docker group, so that it can run commands
+    usermod -aG docker vagrant
+  SHELL
+end
+
 def rpm_common(config, name)
   linux_common(
     config,
@@ -198,6 +265,25 @@ def rpm_common(config, name)
   )
 end
 
+def rpm_docker(config)
+  config.vm.provision 'install Docker using yum', type: 'shell', inline: <<-SHELL
+    # Install prerequisites
+    yum install -y yum-utils device-mapper-persistent-data lvm2
+
+    # Add repository
+    yum-config-manager -y --add-repo https://download.docker.com/linux/centos/docker-ce.repo
+
+    # Install Docker
+    yum install -y docker-ce docker-ce-cli containerd.io
+
+    # Start Docker
+    systemctl enable --now docker
+
+    # Add vagrant to the Docker group, so that it can run commands
+    usermod -aG docker vagrant
+  SHELL
+end
+
 def dnf_common(config, name)
   # Autodetect doesn't work....
   if Vagrant.has_plugin?('vagrant-cachier')
@@ -214,6 +300,25 @@ def dnf_common(config, name)
   )
 end
 
+def dnf_docker(config)
+  config.vm.provision 'install Docker using dnf', type: 'shell', inline: <<-SHELL
+    # Install prerequisites
+    dnf -y install dnf-plugins-core
+
+    # Add repository
+    dnf config-manager --add-repo https://download.docker.com/linux/fedora/docker-ce.repo
+
+    # Install Docker
+    dnf install -y docker-ce docker-ce-cli containerd.io
+
+    # Start Docker
+    systemctl enable --now docker
+
+    # Add vagrant to the Docker group, so that it can run commands
+    usermod -aG docker vagrant
+  SHELL
+end
+
 def suse_common(config, name, extra: '')
   linux_common(
     config,

buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy

@@ -77,15 +77,14 @@ import org.gradle.external.javadoc.CoreJavadocOptions
 import org.gradle.internal.jvm.Jvm
 import org.gradle.language.base.plugins.LifecycleBasePlugin
 import org.gradle.process.CommandLineArgumentProvider
-import org.gradle.process.ExecResult
-import org.gradle.process.ExecSpec
 import org.gradle.util.GradleVersion
 
 import java.nio.charset.StandardCharsets
 import java.nio.file.Files
-import java.util.regex.Matcher
 
 import static org.elasticsearch.gradle.tool.Boilerplate.maybeConfigure
+import static org.elasticsearch.gradle.tool.DockerUtils.assertDockerIsAvailable
+import static org.elasticsearch.gradle.tool.DockerUtils.getDockerPath
 
 /**
  * Encapsulates build configuration for elasticsearch projects.
@@ -184,8 +183,7 @@ class BuildPlugin implements Plugin<Project> {
              */
 
             // check if the Docker binary exists and record its path
-            final List<String> maybeDockerBinaries = ['/usr/bin/docker', '/usr/local/bin/docker']
-            final String dockerBinary = maybeDockerBinaries.find { it -> new File(it).exists() }
+            final String dockerBinary = getDockerPath().orElse(null)
 
             final boolean buildDocker
             final String buildDockerProperty = System.getProperty("build.docker")
@@ -204,84 +202,16 @@ class BuildPlugin implements Plugin<Project> {
             ext.set('requiresDocker', [])
             rootProject.gradle.taskGraph.whenReady { TaskExecutionGraph taskGraph ->
                 final List<String> tasks = taskGraph.allTasks.intersect(ext.get('requiresDocker') as List<Task>).collect { "  ${it.path}".toString()}
-                if (tasks.isEmpty() == false) {
-                    /*
-                     * There are tasks in the task graph that require Docker. Now we are failing because either the Docker binary does not
-                     * exist or because execution of a privileged Docker command failed.
-                     */
-                    if (dockerBinary == null) {
-                        final String message = String.format(
-                                Locale.ROOT,
-                                "Docker (checked [%s]) is required to run the following task%s: \n%s",
-                                maybeDockerBinaries.join(","),
-                                tasks.size() > 1 ? "s" : "",
-                                tasks.join('\n'))
-                        throwDockerRequiredException(message)
-                    }
-
-                    // we use a multi-stage Docker build, check the Docker version since 17.05
-                    final ByteArrayOutputStream dockerVersionOutput = new ByteArrayOutputStream()
-                    LoggedExec.exec(
-                            rootProject,
-                            { ExecSpec it ->
-                                it.commandLine = [dockerBinary, '--version']
-                                it.standardOutput = dockerVersionOutput
-                            })
-                    final String dockerVersion = dockerVersionOutput.toString().trim()
-                    checkDockerVersionRecent(dockerVersion)
-
-                    final ByteArrayOutputStream dockerImagesErrorOutput = new ByteArrayOutputStream()
-                    // the Docker binary executes, check that we can execute a privileged command
-                    final ExecResult dockerImagesResult = LoggedExec.exec(
-                            rootProject,
-                            { ExecSpec it ->
-                                it.commandLine = [dockerBinary, "images"]
-                                it.errorOutput = dockerImagesErrorOutput
-                                it.ignoreExitValue = true
-                            })
-
-                    if (dockerImagesResult.exitValue != 0) {
-                        final String message = String.format(
-                                Locale.ROOT,
-                                "a problem occurred running Docker from [%s] yet it is required to run the following task%s: \n%s\n" +
-                                        "the problem is that Docker exited with exit code [%d] with standard error output [%s]",
-                                dockerBinary,
-                                tasks.size() > 1 ? "s" : "",
-                                tasks.join('\n'),
-                                dockerImagesResult.exitValue,
-                                dockerImagesErrorOutput.toString().trim())
-                        throwDockerRequiredException(message)
-                    }
 
+                if (tasks.isEmpty() == false) {
+                    assertDockerIsAvailable(task.project, tasks)
                 }
             }
         }
 
         (ext.get('requiresDocker') as List<Task>).add(task)
     }
 
-    protected static void checkDockerVersionRecent(String dockerVersion) {
-        final Matcher matcher = dockerVersion =~ /Docker version (\d+\.\d+)\.\d+(?:-[a-zA-Z0-9]+)?, build [0-9a-f]{7,40}/
-        assert matcher.matches(): dockerVersion
-        final dockerMajorMinorVersion = matcher.group(1)
-        final String[] majorMinor = dockerMajorMinorVersion.split("\\.")
-        if (Integer.parseInt(majorMinor[0]) < 17
-                || (Integer.parseInt(majorMinor[0]) == 17 && Integer.parseInt(majorMinor[1]) < 5)) {
-            final String message = String.format(
-                    Locale.ROOT,
-                    "building Docker images requires Docker version 17.05+ due to use of multi-stage builds yet was [%s]",
-                    dockerVersion)
-            throwDockerRequiredException(message)
-        }
-    }
-
-    private static void throwDockerRequiredException(final String message) {
-        throw new GradleException(
-                message + "\nyou can address this by attending to the reported issue, "
-                        + "removing the offending tasks from being executed, "
-                        + "or by passing -Dbuild.docker=false")
-    }
-
     /** Add a check before gradle execution phase which ensures java home for the given java version is set. */
     static void requireJavaHome(Task task, int version) {
         // use root project for global accounting