diff --git a/.ci/dockerOnLinuxExclusions b/.ci/dockerOnLinuxExclusions new file mode 100644 index 0000000000000..ab344641baff9 --- /dev/null +++ b/.ci/dockerOnLinuxExclusions @@ -0,0 +1,12 @@ +# This file specifies the Linux OS versions on which we can't build and +# test Docker images for some reason. These values correspond to ID and +# VERSION_ID from /etc/os-release, and a matching value will cause the +# Docker tests to be skipped on that OS. If /etc/os-release doesn't exist +# (as is the case on centos-6, for example) then that OS will again be +# excluded. +centos-6 +debian-8 +opensuse-15-1 +ol-6.10 +ol-7.7 +sles-12 diff --git a/.ci/os.sh b/.ci/os.sh index 8ec110ac183b7..7cb94ab9fa93f 100755 --- a/.ci/os.sh +++ b/.ci/os.sh @@ -6,6 +6,16 @@ if which zypper > /dev/null ; then sudo zypper install -y insserv-compat fi +if [ -e /etc/sysctl.d/99-gce.conf ]; then + # The GCE defaults disable IPv4 forwarding, which breaks the Docker + # build. Workaround this by renaming the file so that it is executed + # earlier than our own overrides. + # + # This ultimately needs to be fixed at the image level - see infra + # issue 15654. + sudo mv /etc/sysctl.d/99-gce.conf /etc/sysctl.d/98-gce.conf +fi + # Required by bats sudo touch /etc/is_vagrant_vm sudo useradd vagrant diff --git a/Vagrantfile b/Vagrantfile index 93b60b46872fd..c86b0a910c239 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -126,7 +126,7 @@ Vagrant.configure(2) do |config| end 'fedora-29'.tap do |box| config.vm.define box, define_opts do |config| - config.vm.box = 'elastic/fedora-28-x86_64' + config.vm.box = 'elastic/fedora-29-x86_64' dnf_common config, box dnf_docker config end @@ -216,6 +216,10 @@ def ubuntu_docker(config) # Add vagrant to the Docker group, so that it can run commands usermod -aG docker vagrant + + # Enable IPv4 forwarding + sed -i '/net.ipv4.ip_forward/s/^#//' /etc/sysctl.conf + systemctl restart networking SHELL end diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy index 7956c2c8d85f2..c085952cc0be8 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy @@ -76,15 +76,14 @@ import org.gradle.external.javadoc.CoreJavadocOptions import org.gradle.internal.jvm.Jvm import org.gradle.language.base.plugins.LifecycleBasePlugin import org.gradle.process.CommandLineArgumentProvider -import org.gradle.process.ExecResult -import org.gradle.process.ExecSpec import org.gradle.util.GradleVersion import java.nio.charset.StandardCharsets import java.nio.file.Files -import java.util.regex.Matcher import static org.elasticsearch.gradle.tool.Boilerplate.maybeConfigure +import static org.elasticsearch.gradle.tool.DockerUtils.assertDockerIsAvailable +import static org.elasticsearch.gradle.tool.DockerUtils.getDockerPath /** * Encapsulates build configuration for elasticsearch projects. @@ -183,8 +182,7 @@ class BuildPlugin implements Plugin { */ // check if the Docker binary exists and record its path - final List maybeDockerBinaries = ['/usr/bin/docker', '/usr/local/bin/docker'] - final String dockerBinary = maybeDockerBinaries.find { it -> new File(it).exists() } + final String dockerBinary = getDockerPath().orElse(null) final boolean buildDocker final String buildDockerProperty = System.getProperty("build.docker") @@ -203,55 +201,9 @@ class BuildPlugin implements Plugin { ext.set('requiresDocker', []) rootProject.gradle.taskGraph.whenReady { TaskExecutionGraph taskGraph -> final List tasks = taskGraph.allTasks.intersect(ext.get('requiresDocker') as List).collect { " ${it.path}".toString()} - if (tasks.isEmpty() == false) { - /* - * There are tasks in the task graph that require Docker. Now we are failing because either the Docker binary does not - * exist or because execution of a privileged Docker command failed. - */ - if (dockerBinary == null) { - final String message = String.format( - Locale.ROOT, - "Docker (checked [%s]) is required to run the following task%s: \n%s", - maybeDockerBinaries.join(","), - tasks.size() > 1 ? "s" : "", - tasks.join('\n')) - throwDockerRequiredException(message) - } - - // we use a multi-stage Docker build, check the Docker version since 17.05 - final ByteArrayOutputStream dockerVersionOutput = new ByteArrayOutputStream() - LoggedExec.exec( - rootProject, - { ExecSpec it -> - it.commandLine = [dockerBinary, '--version'] - it.standardOutput = dockerVersionOutput - }) - final String dockerVersion = dockerVersionOutput.toString().trim() - checkDockerVersionRecent(dockerVersion) - - final ByteArrayOutputStream dockerImagesErrorOutput = new ByteArrayOutputStream() - // the Docker binary executes, check that we can execute a privileged command - final ExecResult dockerImagesResult = LoggedExec.exec( - rootProject, - { ExecSpec it -> - it.commandLine = [dockerBinary, "images"] - it.errorOutput = dockerImagesErrorOutput - it.ignoreExitValue = true - }) - - if (dockerImagesResult.exitValue != 0) { - final String message = String.format( - Locale.ROOT, - "a problem occurred running Docker from [%s] yet it is required to run the following task%s: \n%s\n" + - "the problem is that Docker exited with exit code [%d] with standard error output [%s]", - dockerBinary, - tasks.size() > 1 ? "s" : "", - tasks.join('\n'), - dockerImagesResult.exitValue, - dockerImagesErrorOutput.toString().trim()) - throwDockerRequiredException(message) - } + if (tasks.isEmpty() == false) { + assertDockerIsAvailable(task.project, tasks) } } } @@ -259,28 +211,6 @@ class BuildPlugin implements Plugin { (ext.get('requiresDocker') as List).add(task) } - protected static void checkDockerVersionRecent(String dockerVersion) { - final Matcher matcher = dockerVersion =~ /Docker version (\d+\.\d+)\.\d+(?:-[a-zA-Z0-9]+)?, build [0-9a-f]{7,40}/ - assert matcher.matches(): dockerVersion - final dockerMajorMinorVersion = matcher.group(1) - final String[] majorMinor = dockerMajorMinorVersion.split("\\.") - if (Integer.parseInt(majorMinor[0]) < 17 - || (Integer.parseInt(majorMinor[0]) == 17 && Integer.parseInt(majorMinor[1]) < 5)) { - final String message = String.format( - Locale.ROOT, - "building Docker images requires Docker version 17.05+ due to use of multi-stage builds yet was [%s]", - dockerVersion) - throwDockerRequiredException(message) - } - } - - private static void throwDockerRequiredException(final String message) { - throw new GradleException( - message + "\nyou can address this by attending to the reported issue, " - + "removing the offending tasks from being executed, " - + "or by passing -Dbuild.docker=false") - } - /** Add a check before gradle execution phase which ensures java home for the given java version is set. */ static void requireJavaHome(Task task, int version) { // use root project for global accounting diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/DistroTestPlugin.java b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/DistroTestPlugin.java index 55f4985748325..417e13400c114 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/DistroTestPlugin.java +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/DistroTestPlugin.java @@ -28,18 +28,22 @@ import org.elasticsearch.gradle.ElasticsearchDistribution.Type; import org.elasticsearch.gradle.Jdk; import org.elasticsearch.gradle.JdkDownloadPlugin; +import org.elasticsearch.gradle.OS; import org.elasticsearch.gradle.Version; import org.elasticsearch.gradle.VersionProperties; import org.elasticsearch.gradle.info.BuildParams; import org.elasticsearch.gradle.vagrant.BatsProgressLogger; import org.elasticsearch.gradle.vagrant.VagrantBasePlugin; import org.elasticsearch.gradle.vagrant.VagrantExtension; +import org.gradle.api.GradleException; import org.gradle.api.NamedDomainObjectContainer; import org.gradle.api.Plugin; import org.gradle.api.Project; import org.gradle.api.Task; import org.gradle.api.artifacts.Configuration; import org.gradle.api.file.Directory; +import org.gradle.api.logging.Logger; +import org.gradle.api.logging.Logging; import org.gradle.api.plugins.ExtraPropertiesExtension; import org.gradle.api.plugins.JavaBasePlugin; import org.gradle.api.provider.Provider; @@ -52,6 +56,7 @@ import java.io.UncheckedIOException; import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.Paths; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; @@ -66,6 +71,7 @@ import static org.elasticsearch.gradle.vagrant.VagrantMachine.convertWindowsPath; public class DistroTestPlugin implements Plugin { + private static final Logger logger = Logging.getLogger(DistroTestPlugin.class); private static final String SYSTEM_JDK_VERSION = "11.0.2+9"; private static final String SYSTEM_JDK_VENDOR = "openjdk"; @@ -84,6 +90,8 @@ public class DistroTestPlugin implements Plugin { @Override public void apply(Project project) { + final boolean runDockerTests = shouldRunDockerTests(project); + project.getPluginManager().apply(DistributionDownloadPlugin.class); project.getPluginManager().apply(BuildPlugin.class); @@ -95,15 +103,17 @@ public void apply(Project project) { Provider upgradeDir = project.getLayout().getBuildDirectory().dir("packaging/upgrade"); Provider pluginsDir = project.getLayout().getBuildDirectory().dir("packaging/plugins"); - List distributions = configureDistributions(project, upgradeVersion); + List distributions = configureDistributions(project, upgradeVersion, runDockerTests); TaskProvider copyDistributionsTask = configureCopyDistributionsTask(project, distributionsDir); TaskProvider copyUpgradeTask = configureCopyUpgradeTask(project, upgradeVersion, upgradeDir); TaskProvider copyPluginsTask = configureCopyPluginsTask(project, pluginsDir); TaskProvider destructiveDistroTest = project.getTasks().register("destructiveDistroTest"); for (ElasticsearchDistribution distribution : distributions) { - TaskProvider destructiveTask = configureDistroTest(project, distribution); - destructiveDistroTest.configure(t -> t.dependsOn(destructiveTask)); + if (distribution.getType() != Type.DOCKER || runDockerTests == true) { + TaskProvider destructiveTask = configureDistroTest(project, distribution); + destructiveDistroTest.configure(t -> t.dependsOn(destructiveTask)); + } } Map> batsTests = new HashMap<>(); batsTests.put("bats oss", configureBatsTest(project, "oss", distributionsDir, copyDistributionsTask)); @@ -129,7 +139,23 @@ public void apply(Project project) { TaskProvider vmTask = configureVMWrapperTask(vmProject, distribution.getName() + " distribution", destructiveTaskName, vmDependencies); vmTask.configure(t -> t.dependsOn(distribution)); - distroTest.configure(t -> t.dependsOn(vmTask)); + + distroTest.configure(t -> { + // Only VM sub-projects that are specifically opted-in to testing Docker should + // have the Docker task added as a dependency. Although we control whether Docker + // is installed in the VM via `Vagrantfile` and we could auto-detect its presence + // in the VM, the test tasks e.g. `destructiveDistroTest.default-docker` are defined + // on the host during Gradle's configuration phase and not in the VM, so + // auto-detection doesn't work. + // + // The shouldTestDocker property could be null, hence we use Boolean.TRUE.equals() + boolean shouldExecute = distribution.getType() != Type.DOCKER + || Boolean.TRUE.equals(vmProject.findProperty("shouldTestDocker")) == true; + + if (shouldExecute) { + t.dependsOn(vmTask); + } + }); } } @@ -321,17 +347,17 @@ private static TaskProvider configureBatsTest(Project project, Str }); } - private List configureDistributions(Project project, Version upgradeVersion) { + private List configureDistributions(Project project, Version upgradeVersion, boolean runDockerTests) { NamedDomainObjectContainer distributions = DistributionDownloadPlugin.getContainer(project); List currentDistros = new ArrayList<>(); List upgradeDistros = new ArrayList<>(); - // Docker disabled for https://github.com/elastic/elasticsearch/issues/47639 - for (Type type : Arrays.asList(Type.DEB, Type.RPM /*,Type.DOCKER*/)) { + for (Type type : List.of(Type.DEB, Type.RPM, Type.DOCKER)) { for (Flavor flavor : Flavor.values()) { for (boolean bundledJdk : Arrays.asList(true, false)) { - // We should never add a Docker distro with bundledJdk == false - boolean skip = type == Type.DOCKER && bundledJdk == false; + // All our Docker images include a bundled JDK so it doesn't make sense to test without one + boolean skip = type == Type.DOCKER && (runDockerTests == false || bundledJdk == false); + if (skip == false) { addDistro(distributions, type, null, flavor, bundledJdk, VersionProperties.getElasticsearch(), currentDistros); } @@ -345,6 +371,7 @@ private List configureDistributions(Project project, addDistro(distributions, type, null, Flavor.OSS, true, upgradeVersion.toString(), upgradeDistros); } } + for (Platform platform : Arrays.asList(Platform.LINUX, Platform.WINDOWS)) { for (Flavor flavor : Flavor.values()) { for (boolean bundledJdk : Arrays.asList(true, false)) { @@ -405,4 +432,92 @@ private static String destructiveDistroTestTaskName(ElasticsearchDistribution di distro.getFlavor(), distro.getBundledJdk()); } + + static Map parseOsRelease(final List osReleaseLines) { + final Map values = new HashMap<>(); + + osReleaseLines.stream().map(String::trim).filter(line -> (line.isEmpty() || line.startsWith("#")) == false).forEach(line -> { + final String[] parts = line.split("=", 2); + final String key = parts[0]; + // remove optional leading and trailing quotes and whitespace + final String value = parts[1].replaceAll("^['\"]?\\s*", "").replaceAll("\\s*['\"]?$", ""); + + values.put(key, value); + }); + + return values; + } + + static String deriveId(final Map osRelease) { + return osRelease.get("ID") + "-" + osRelease.get("VERSION_ID"); + } + + private static List getLinuxExclusionList(Project project) { + final String exclusionsFilename = "dockerOnLinuxExclusions"; + final Path exclusionsPath = project.getRootDir().toPath().resolve(Path.of(".ci", exclusionsFilename)); + + try { + return Files.readAllLines(exclusionsPath) + .stream() + .map(String::trim) + .filter(line -> (line.isEmpty() || line.startsWith("#")) == false) + .collect(Collectors.toList()); + } catch (IOException e) { + throw new GradleException("Failed to read .ci/" + exclusionsFilename, e); + } + } + + /** + * The {@link DistroTestPlugin} generates a number of test tasks, some + * of which are Docker packaging tests. When running on the host OS or in CI + * i.e. not in a Vagrant VM, only certain operating systems are supported. This + * method determines whether the Docker tests should be run on the host + * OS. Essentially, unless an OS and version is specifically excluded, we expect + * to be able to run Docker and test the Docker images. + * @param project + */ + private static boolean shouldRunDockerTests(Project project) { + switch (OS.current()) { + case WINDOWS: + // Not yet supported. + return false; + + case MAC: + // Assume that Docker for Mac is installed, since Docker is part of the dev workflow. + return true; + + case LINUX: + // Only some hosts in CI are configured with Docker. We attempt to work out the OS + // and version, so that we know whether to expect to find Docker. We don't attempt + // to probe for whether Docker is available, because that doesn't tell us whether + // Docker is unavailable when it should be. + final Path osRelease = Paths.get("/etc/os-release"); + + if (Files.exists(osRelease)) { + Map values; + + try { + final List osReleaseLines = Files.readAllLines(osRelease); + values = parseOsRelease(osReleaseLines); + } catch (IOException e) { + throw new GradleException("Failed to read /etc/os-release", e); + } + + final String id = deriveId(values); + + final boolean shouldExclude = getLinuxExclusionList(project).contains(id); + + logger.warn("Linux OS id [" + id + "] is " + (shouldExclude ? "" : "not ") + "present in the Docker exclude list"); + + return shouldExclude == false; + } + + logger.warn("/etc/os-release does not exist!"); + return false; + + default: + logger.warn("Unknown OS [" + OS.current() + "], answering false to shouldRunDockerTests()"); + return false; + } + } } diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/tool/DockerUtils.java b/buildSrc/src/main/java/org/elasticsearch/gradle/tool/DockerUtils.java new file mode 100644 index 0000000000000..2442ffce427c8 --- /dev/null +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/tool/DockerUtils.java @@ -0,0 +1,239 @@ +package org.elasticsearch.gradle.tool; + +import org.elasticsearch.gradle.Version; +import org.gradle.api.GradleException; +import org.gradle.api.Project; +import org.gradle.process.ExecResult; + +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.util.List; +import java.util.Locale; +import java.util.Optional; + +/** + * Contains utilities for checking whether Docker is installed, is executable, + * has a recent enough version, and appears to be functional. The Elasticsearch build + * requires Docker >= 17.05 as it uses a multi-stage build. + */ +public class DockerUtils { + /** + * Defines the possible locations of the Docker CLI. These will be searched in order. + */ + private static String[] DOCKER_BINARIES = { "/usr/bin/docker", "/usr/local/bin/docker" }; + + /** + * Searches the entries in {@link #DOCKER_BINARIES} for the Docker CLI. This method does + * not check whether the Docker installation appears usable, see {@link #getDockerAvailability(Project)} + * instead. + * + * @return the path to a CLI, if available. + */ + public static Optional getDockerPath() { + // Check if the Docker binary exists + return List.of(DOCKER_BINARIES) + .stream() + .filter(path -> new File(path).exists()) + .findFirst(); + } + + /** + * Searches for a functional Docker installation, and returns information about the search. + * @return the results of the search. + */ + private static DockerAvailability getDockerAvailability(Project project) { + String dockerPath = null; + Result lastResult = null; + Version version = null; + boolean isVersionHighEnough = false; + + // Check if the Docker binary exists + final Optional dockerBinary = getDockerPath(); + + if (dockerBinary.isPresent()) { + dockerPath = dockerBinary.get(); + + // Since we use a multi-stage Docker build, check the Docker version since 17.05 + lastResult = runCommand(project, dockerPath, "version", "--format", "{{.Server.Version}}"); + + if (lastResult.isSuccess() == true) { + version = Version.fromString(lastResult.stdout.trim(), Version.Mode.RELAXED); + + isVersionHighEnough = version.onOrAfter("17.05.0"); + + if (isVersionHighEnough == true) { + // Check that we can execute a privileged command + lastResult = runCommand(project, dockerPath, "images"); + } + } + } + + boolean isAvailable = isVersionHighEnough && lastResult.isSuccess() == true; + + return new DockerAvailability(isAvailable, isVersionHighEnough, dockerPath, version, lastResult); + } + + /** + * An immutable class that represents the results of a Docker search from {@link #getDockerAvailability(Project)}}. + */ + private static class DockerAvailability { + /** + * Indicates whether Docker is available and meets the required criteria. + * True if, and only if, Docker is: + *
    + *
  • Installed
    • + *
  • Executable
    • + *
  • Is at least version 17.05
    • + *
  • Can execute a command that requires privileges
    • + *
+ */ + final boolean isAvailable; + + /** + * True if the installed Docker version is >= 17.05 + */ + final boolean isVersionHighEnough; + + /** + * The path to the Docker CLI, or null + */ + public final String path; + + /** + * The installed Docker version, or null + */ + public final Version version; + + /** + * Information about the last command executes while probing Docker, or null. + */ + final Result lastCommand; + + DockerAvailability(boolean isAvailable, boolean isVersionHighEnough, String path, Version version, Result lastCommand) { + this.isAvailable = isAvailable; + this.isVersionHighEnough = isVersionHighEnough; + this.path = path; + this.version = version; + this.lastCommand = lastCommand; + } + } + + /** + * Given a list of tasks that requires Docker, check whether Docker is available, otherwise + * throw an exception. + * @param project a Gradle project + * @param tasks the tasks that require Docker + * @throws GradleException if Docker is not available. The exception message gives the reason. + */ + public static void assertDockerIsAvailable(Project project, List tasks) { + DockerAvailability availability = getDockerAvailability(project); + + if (availability.isAvailable == true) { + return; + } + + /* + * There are tasks in the task graph that require Docker. + * Now we are failing because either the Docker binary does + * not exist or because execution of a privileged Docker + * command failed. + */ + if (availability.path == null) { + final String message = String.format( + Locale.ROOT, + "Docker (checked [%s]) is required to run the following task%s: \n%s", + String.join(", ", DOCKER_BINARIES), + tasks.size() > 1 ? "s" : "", + String.join("\n", tasks)); + throwDockerRequiredException(message); + } + + if (availability.version == null) { + final String message = String.format( + Locale.ROOT, + "Docker is required to run the following task%s, but it doesn't appear to be running: \n%s", + tasks.size() > 1 ? "s" : "", + String.join("\n", tasks)); + throwDockerRequiredException(message); + } + + if (availability.isVersionHighEnough == false) { + final String message = String.format( + Locale.ROOT, + "building Docker images requires Docker version 17.05+ due to use of multi-stage builds yet was [%s]", + availability.version); + throwDockerRequiredException(message); + } + + // Some other problem, print the error + final String message = String.format( + Locale.ROOT, + "a problem occurred running Docker from [%s] yet it is required to run the following task%s: \n%s\n" + + "the problem is that Docker exited with exit code [%d] with standard error output [%s]", + availability.path, + tasks.size() > 1 ? "s" : "", + String.join("\n", tasks), + availability.lastCommand.exitCode, + availability.lastCommand.stderr.trim()); + throwDockerRequiredException(message); + } + + private static void throwDockerRequiredException(final String message) { + throwDockerRequiredException(message, null); + } + + private static void throwDockerRequiredException(final String message, Exception e) { + throw new GradleException( + message + "\nyou can address this by attending to the reported issue, " + + "removing the offending tasks from being executed, " + + "or by passing -Dbuild.docker=false", e); + } + + /** + * Runs a command and captures the exit code, standard output and standard error. + * @param args the command and any arguments to execute + * @return a object that captures the result of running the command. If an exception occurring + * while running the command, or the process was killed after reaching the 10s timeout, + * then the exit code will be -1. + */ + private static Result runCommand(Project project, String... args) { + if (args.length == 0) { + throw new IllegalArgumentException("Cannot execute with no command"); + } + + ByteArrayOutputStream stdout = new ByteArrayOutputStream(); + ByteArrayOutputStream stderr = new ByteArrayOutputStream(); + + final ExecResult execResult = project.exec(spec -> { + // The redundant cast is to silence a compiler warning. + spec.setCommandLine((Object[]) args); + spec.setStandardOutput(stdout); + spec.setErrorOutput(stderr); + }); + + return new Result(execResult.getExitValue(), stdout.toString(), stderr.toString()); + } + + /** + * This class models the result of running a command. It captures the exit code, standard output and standard error. + */ + private static class Result { + final int exitCode; + final String stdout; + final String stderr; + + Result(int exitCode, String stdout, String stderr) { + this.exitCode = exitCode; + this.stdout = stdout; + this.stderr = stderr; + } + + boolean isSuccess() { + return exitCode == 0; + } + + public String toString() { + return "exitCode = [" + exitCode + "] " + "stdout = [" + stdout.trim() + "] " + "stderr = [" + stderr.trim() + "]"; + } + } +} diff --git a/buildSrc/src/minimumRuntime/java/org/elasticsearch/gradle/Version.java b/buildSrc/src/minimumRuntime/java/org/elasticsearch/gradle/Version.java index 31738f140878d..d31e15b842b2c 100644 --- a/buildSrc/src/minimumRuntime/java/org/elasticsearch/gradle/Version.java +++ b/buildSrc/src/minimumRuntime/java/org/elasticsearch/gradle/Version.java @@ -13,9 +13,28 @@ public final class Version implements Comparable { private final int revision; private final int id; + /** + * Specifies how a version string should be parsed. + */ + public enum Mode { + /** + * Strict parsing only allows known suffixes after the patch number: "-alpha", "-beta" or "-rc". The + * suffix "-SNAPSHOT" is also allowed, either after the patch number, or after the other suffices. + */ + STRICT, + + /** + * Relaxed parsing allows any alphanumeric suffix after the patch number. + */ + RELAXED + } + private static final Pattern pattern = Pattern.compile("(\\d)+\\.(\\d+)\\.(\\d+)(-alpha\\d+|-beta\\d+|-rc\\d+)?(-SNAPSHOT)?"); + private static final Pattern relaxedPattern = + Pattern.compile("(\\d)+\\.(\\d+)\\.(\\d+)(-[a-zA-Z0-9_]+)*?"); + public Version(int major, int minor, int revision) { Objects.requireNonNull(major, "major version can't be null"); Objects.requireNonNull(minor, "minor version can't be null"); @@ -36,11 +55,18 @@ private static int parseSuffixNumber(String substring) { } public static Version fromString(final String s) { + return fromString(s, Mode.STRICT); + } + + public static Version fromString(final String s, final Mode mode) { Objects.requireNonNull(s); - Matcher matcher = pattern.matcher(s); + Matcher matcher = mode == Mode.STRICT ? pattern.matcher(s) : relaxedPattern.matcher(s); if (matcher.matches() == false) { + String expected = mode == Mode.STRICT == true + ? "major.minor.revision[-(alpha|beta|rc)Number][-SNAPSHOT]" + : "major.minor.revision[-extra]"; throw new IllegalArgumentException( - "Invalid version format: '" + s + "'. Should be major.minor.revision[-(alpha|beta|rc)Number][-SNAPSHOT]" + "Invalid version format: '" + s + "'. Should be " + expected ); } diff --git a/buildSrc/src/test/java/org/elasticsearch/gradle/BuildPluginTests.java b/buildSrc/src/test/java/org/elasticsearch/gradle/BuildPluginTests.java index c61a0a3935898..8d6d0be00dbe8 100644 --- a/buildSrc/src/test/java/org/elasticsearch/gradle/BuildPluginTests.java +++ b/buildSrc/src/test/java/org/elasticsearch/gradle/BuildPluginTests.java @@ -28,17 +28,6 @@ public class BuildPluginTests extends GradleUnitTestCase { - public void testPassingDockerVersions() { - BuildPlugin.checkDockerVersionRecent("Docker version 18.06.1-ce, build e68fc7a215d7"); - BuildPlugin.checkDockerVersionRecent("Docker version 17.05.0, build e68fc7a"); - BuildPlugin.checkDockerVersionRecent("Docker version 17.05.1, build e68fc7a"); - } - - @Test(expected = GradleException.class) - public void testFailingDockerVersions() { - BuildPlugin.checkDockerVersionRecent("Docker version 17.04.0, build e68fc7a"); - } - @Test(expected = GradleException.class) public void testRepositoryURIThatUsesHttpScheme() throws URISyntaxException { final URI uri = new URI("http://s3.amazonaws.com/artifacts.elastic.co/maven"); diff --git a/buildSrc/src/test/java/org/elasticsearch/gradle/VersionTests.java b/buildSrc/src/test/java/org/elasticsearch/gradle/VersionTests.java index 3394285157e17..ae2fb0e6215db 100644 --- a/buildSrc/src/test/java/org/elasticsearch/gradle/VersionTests.java +++ b/buildSrc/src/test/java/org/elasticsearch/gradle/VersionTests.java @@ -40,6 +40,14 @@ public void testVersionParsing() { assertVersionEquals("6.1.2-beta1-SNAPSHOT", 6, 1, 2); } + public void testRelaxedVersionParsing() { + assertVersionEquals("6.1.2", 6, 1, 2, Version.Mode.RELAXED); + assertVersionEquals("6.1.2-SNAPSHOT", 6, 1, 2, Version.Mode.RELAXED); + assertVersionEquals("6.1.2-beta1-SNAPSHOT", 6, 1, 2, Version.Mode.RELAXED); + assertVersionEquals("6.1.2-foo", 6, 1, 2, Version.Mode.RELAXED); + assertVersionEquals("6.1.2-foo-bar", 6, 1, 2, Version.Mode.RELAXED); + } + public void testCompareWithStringVersions() { assertTrue("1.10.20 is not interpreted as before 2.0.0", Version.fromString("1.10.20").before("2.0.0") @@ -100,7 +108,11 @@ private void assertOrder(Version smaller, Version bigger) { } private void assertVersionEquals(String stringVersion, int major, int minor, int revision) { - Version version = Version.fromString(stringVersion); + assertVersionEquals(stringVersion, major, minor, revision, Version.Mode.STRICT); + } + + private void assertVersionEquals(String stringVersion, int major, int minor, int revision, Version.Mode mode) { + Version version = Version.fromString(stringVersion, mode); assertEquals(major, version.getMajor()); assertEquals(minor, version.getMinor()); assertEquals(revision, version.getRevision()); diff --git a/buildSrc/src/test/java/org/elasticsearch/gradle/test/DistroTestPluginTests.java b/buildSrc/src/test/java/org/elasticsearch/gradle/test/DistroTestPluginTests.java new file mode 100644 index 0000000000000..f88a4c11415bc --- /dev/null +++ b/buildSrc/src/test/java/org/elasticsearch/gradle/test/DistroTestPluginTests.java @@ -0,0 +1,85 @@ +package org.elasticsearch.gradle.test; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.elasticsearch.gradle.test.DistroTestPlugin.deriveId; +import static org.elasticsearch.gradle.test.DistroTestPlugin.parseOsRelease; +import static org.hamcrest.CoreMatchers.equalTo; + +public class DistroTestPluginTests extends GradleIntegrationTestCase { + + public void testParseOsReleaseOnOracle() { + final List lines = List + .of( + "NAME=\"Oracle Linux Server\"", + "VERSION=\"6.10\"", + "ID=\"ol\"", + "VERSION_ID=\"6.10\"", + "PRETTY_NAME=\"Oracle Linux Server 6.10\"", + "ANSI_COLOR=\"0;31\"", + "CPE_NAME=\"cpe:/o:oracle:linux:6:10:server\"", + "HOME_URL" + "=\"https://linux.oracle.com/\"", + "BUG_REPORT_URL=\"https://bugzilla.oracle.com/\"", + "", + "ORACLE_BUGZILLA_PRODUCT" + "=\"Oracle Linux 6\"", + "ORACLE_BUGZILLA_PRODUCT_VERSION=6.10", + "ORACLE_SUPPORT_PRODUCT=\"Oracle Linux\"", + "ORACLE_SUPPORT_PRODUCT_VERSION=6.10" + ); + + final Map results = parseOsRelease(lines); + + final Map expected = new HashMap<>(); + expected.put("ANSI_COLOR", "0;31"); + expected.put("BUG_REPORT_URL", "https://bugzilla.oracle.com/"); + expected.put("CPE_NAME", "cpe:/o:oracle:linux:6:10:server"); + expected.put("HOME_URL" + "", "https://linux.oracle.com/"); + expected.put("ID", "ol"); + expected.put("NAME", "Oracle Linux Server"); + expected.put("ORACLE_BUGZILLA_PRODUCT" + "", "Oracle Linux 6"); + expected.put("ORACLE_BUGZILLA_PRODUCT_VERSION", "6.10"); + expected.put("ORACLE_SUPPORT_PRODUCT", "Oracle Linux"); + expected.put("ORACLE_SUPPORT_PRODUCT_VERSION", "6.10"); + expected.put("PRETTY_NAME", "Oracle Linux Server 6.10"); + expected.put("VERSION", "6.10"); + expected.put("VERSION_ID", "6.10"); + + assertThat(expected, equalTo(results)); + } + + /** + * Trailing whitespace should be removed + */ + public void testRemoveTrailingWhitespace() { + final List lines = List.of("NAME=\"Oracle Linux Server\" "); + + final Map results = parseOsRelease(lines); + + final Map expected = Map.of("NAME", "Oracle Linux Server"); + + assertThat(expected, equalTo(results)); + } + + /** + * Comments should be removed + */ + public void testRemoveComments() { + final List lines = List.of("# A comment", "NAME=\"Oracle Linux Server\""); + + final Map results = parseOsRelease(lines); + + final Map expected = Map.of("NAME", "Oracle Linux Server"); + + assertThat(expected, equalTo(results)); + } + + public void testDeriveIdOnOracle() { + final Map osRelease = new HashMap<>(); + osRelease.put("ID", "ol"); + osRelease.put("VERSION_ID", "6.10"); + + assertThat("ol-6.10", equalTo(deriveId(osRelease))); + } +} diff --git a/qa/os/centos-7/build.gradle b/qa/os/centos-7/build.gradle index e69de29bb2d1d..814b04d4aec5f 100644 --- a/qa/os/centos-7/build.gradle +++ b/qa/os/centos-7/build.gradle @@ -0,0 +1 @@ +project.ext.shouldTestDocker = true diff --git a/qa/os/debian-9/build.gradle b/qa/os/debian-9/build.gradle index e69de29bb2d1d..814b04d4aec5f 100644 --- a/qa/os/debian-9/build.gradle +++ b/qa/os/debian-9/build.gradle @@ -0,0 +1 @@ +project.ext.shouldTestDocker = true diff --git a/qa/os/fedora-28/build.gradle b/qa/os/fedora-28/build.gradle index e69de29bb2d1d..814b04d4aec5f 100644 --- a/qa/os/fedora-28/build.gradle +++ b/qa/os/fedora-28/build.gradle @@ -0,0 +1 @@ +project.ext.shouldTestDocker = true diff --git a/qa/os/fedora-29/build.gradle b/qa/os/fedora-29/build.gradle index e69de29bb2d1d..814b04d4aec5f 100644 --- a/qa/os/fedora-29/build.gradle +++ b/qa/os/fedora-29/build.gradle @@ -0,0 +1 @@ +project.ext.shouldTestDocker = true diff --git a/qa/os/src/test/java/org/elasticsearch/packaging/test/DockerTests.java b/qa/os/src/test/java/org/elasticsearch/packaging/test/DockerTests.java index daad67f7fb111..52205263d3ed3 100644 --- a/qa/os/src/test/java/org/elasticsearch/packaging/test/DockerTests.java +++ b/qa/os/src/test/java/org/elasticsearch/packaging/test/DockerTests.java @@ -28,7 +28,6 @@ import org.junit.AfterClass; import org.junit.Before; import org.junit.BeforeClass; -import org.junit.Ignore; import java.nio.file.Files; import java.nio.file.Path; @@ -55,7 +54,6 @@ import static org.hamcrest.Matchers.emptyString; import static org.junit.Assume.assumeTrue; -@Ignore("https://github.com/elastic/elasticsearch/issues/47639") public class DockerTests extends PackagingTestCase { protected DockerShell sh; diff --git a/qa/os/src/test/java/org/elasticsearch/packaging/util/Docker.java b/qa/os/src/test/java/org/elasticsearch/packaging/util/Docker.java index d78b60236bc4a..8e6bc4a46e13c 100644 --- a/qa/os/src/test/java/org/elasticsearch/packaging/util/Docker.java +++ b/qa/os/src/test/java/org/elasticsearch/packaging/util/Docker.java @@ -137,20 +137,24 @@ private static void waitForElasticsearchToStart() throws InterruptedException { boolean isElasticsearchRunning = false; int attempt = 0; + String psOutput; + do { - String psOutput = dockerShell.run("ps ax").stdout; + // Give the container a chance to crash out + Thread.sleep(1000); + + psOutput = dockerShell.run("ps ax").stdout; - if (psOutput.contains("/usr/share/elasticsearch/jdk/bin/java -X")) { + if (psOutput.contains("/usr/share/elasticsearch/jdk/bin/java")) { isElasticsearchRunning = true; break; } - Thread.sleep(1000); } while (attempt++ < 5); if (!isElasticsearchRunning) { - final String logs = sh.run("docker logs " + containerId).stdout; - fail("Elasticsearch container did start successfully.\n\n" + logs); + final String dockerLogs = sh.run("docker logs " + containerId).stdout; + fail("Elasticsearch container did start successfully.\n\n" + psOutput + "\n\n" + dockerLogs); } } @@ -240,7 +244,7 @@ public static void assertPermissionsAndOwnership(Path path, Set