Skip to content

Commit

Permalink
Merge branch 'master' into ccr-replicate-aliases
Browse files Browse the repository at this point in the history
* master: (82 commits)
  Fix off-by-one error in an index shard test
  Cleanup Redundant BlobStoreFormat Class (elastic#42195)
  remove backcompat handling of 6.2.x versions (elastic#42044)
  Mute testDelayedOperationsBeforeAndAfterRelocated
  Execute actions under permit in primary mode only (elastic#42241)
  Mute another transforms_stats yaml test
  Deprecate support for chained multi-fields. (elastic#41926)
  Mute transforms_stats yaml test
  Make unwrapCorrupt Check Suppressed Ex. (elastic#41889)
  Remove Dead Code from Azure Repo Plugin (elastic#42178)
  Reorganize Painless doc structure (elastic#42303)
  Avoid unnecessary persistence of retention leases (elastic#42299)
  [ML][TEST] Fix limits in AutodetectMemoryLimitIT (elastic#42279)
  [ML Data Frame] Persist and restore checkpoint and position (elastic#41942)
  mute failing filerealm hash caching tests (elastic#42304)
  Safer Wait for Snapshot Success in ClusterPrivilegeTests (elastic#40943)
  Remove 7.0.2 (elastic#42282)
  Revert "Remove 7.0.2 (elastic#42282)"
  [DOCS] Copied note on slicing support to Slicing section. Closes 26114 (elastic#40426)
  Remove 7.0.2 (elastic#42282)
  ...
  • Loading branch information
jasontedor committed May 22, 2019
2 parents 2b1fe34 + bb2ec18 commit db4ad80
Show file tree
Hide file tree
Showing 497 changed files with 9,068 additions and 6,058 deletions.
20 changes: 4 additions & 16 deletions Vagrantfile
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,7 @@ Vagrant.configure(2) do |config|
config.vm.box = 'elastic/debian-8-x86_64'
deb_common config, box, extra: <<-SHELL
# this sometimes gets a bad ip, and doesn't appear to be needed
rm /etc/apt/sources.list.d/http_debian_net_debian.list
rm -f /etc/apt/sources.list.d/http_debian_net_debian.list
SHELL
end
end
Expand Down Expand Up @@ -256,10 +256,6 @@ def linux_common(config,
touch /is_vagrant_vm # for consistency between linux and windows
SHELL

config.vm.provision 'jdk-11', type: 'shell', inline: <<-SHELL
curl -sSL https://download.oracle.com/java/GA/jdk11/9/GPL/openjdk-11.0.2_linux-x64_bin.tar.gz | tar xz -C /opt/
SHELL

# This prevents leftovers from previous tests using the
# same VM from messing up the current test
config.vm.provision 'clean es installs in tmp', run: 'always', type: 'shell', inline: <<-SHELL
Expand Down Expand Up @@ -355,11 +351,10 @@ def sh_install_deps(config,
return 1
}
cat \<\<JAVA > /etc/profile.d/java_home.sh
if [ -z "\\\$JAVA_HOME" ]; then
export JAVA_HOME=/opt/jdk-11.0.2
if [ ! -z "\\\$JAVA_HOME" ]; then
export SYSTEM_JAVA_HOME=\\\$JAVA_HOME
unset JAVA_HOME
fi
export SYSTEM_JAVA_HOME=\\\$JAVA_HOME
unset JAVA_HOME
JAVA
ensure tar
ensure curl
Expand Down Expand Up @@ -416,16 +411,9 @@ def windows_common(config, name)
$ps_prompt | Out-File $PsHome/Microsoft.PowerShell_profile.ps1
SHELL

config.vm.provision 'windows-jdk-11', type: 'shell', inline: <<-SHELL
New-Item -ItemType Directory -Force -Path "C:/java"
Invoke-WebRequest "https://download.oracle.com/java/GA/jdk11/9/GPL/openjdk-11.0.2_windows-x64_bin.zip" -OutFile "C:/java/jdk-11.zip"
Expand-Archive -Path "C:/java/jdk-11.zip" -DestinationPath "C:/java/"
SHELL

config.vm.provision 'set env variables', type: 'shell', inline: <<-SHELL
$ErrorActionPreference = "Stop"
[Environment]::SetEnvironmentVariable("PACKAGING_ARCHIVES", "C:/project/build/packaging/archives", "Machine")
[Environment]::SetEnvironmentVariable("SYSTEM_JAVA_HOME", "C:\java\jdk-11.0.2", "Machine")
[Environment]::SetEnvironmentVariable("PACKAGING_TESTS", "C:/project/build/packaging/tests", "Machine")
[Environment]::SetEnvironmentVariable("JAVA_HOME", $null, "Machine")
SHELL
Expand Down
15 changes: 0 additions & 15 deletions build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -619,21 +619,6 @@ allprojects {
}
}

subprojects {
// Common config when running with a FIPS-140 runtime JVM
if (project.ext.has("inFipsJvm") && project.ext.inFipsJvm) {
tasks.withType(Test) {
systemProperty 'javax.net.ssl.trustStorePassword', 'password'
systemProperty 'javax.net.ssl.keyStorePassword', 'password'
}
project.pluginManager.withPlugin("elasticsearch.testclusters") {
project.testClusters.all {
systemProperty 'javax.net.ssl.trustStorePassword', 'password'
systemProperty 'javax.net.ssl.keyStorePassword', 'password'
}
}
}
}



Original file line number Diff line number Diff line change
Expand Up @@ -116,6 +116,22 @@ class BuildPlugin implements Plugin<Project> {
configureTestTasks(project)
configurePrecommit(project)
configureDependenciesInfo(project)

// Common config when running with a FIPS-140 runtime JVM
// Need to do it here to support external plugins
if (project.ext.inFipsJvm) {
project.tasks.withType(Test) {
systemProperty 'javax.net.ssl.trustStorePassword', 'password'
systemProperty 'javax.net.ssl.keyStorePassword', 'password'
}
project.pluginManager.withPlugin("elasticsearch.testclusters") {
project.testClusters.all {
systemProperty 'javax.net.ssl.trustStorePassword', 'password'
systemProperty 'javax.net.ssl.keyStorePassword', 'password'
}
}
}

}


Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,15 +27,15 @@ import org.gradle.api.tasks.Input
public class BatsOverVagrantTask extends VagrantCommandTask {

@Input
String remoteCommand
Object remoteCommand

BatsOverVagrantTask() {
command = 'ssh'
}

void setRemoteCommand(String remoteCommand) {
void setRemoteCommand(Object remoteCommand) {
this.remoteCommand = Objects.requireNonNull(remoteCommand)
setArgs(['--command', remoteCommand])
setArgs((Iterable<?>) ['--command', remoteCommand])
}

@Override
Expand Down
Original file line number Diff line number Diff line change
@@ -1,11 +1,18 @@
package org.elasticsearch.gradle.vagrant

import org.apache.tools.ant.taskdefs.condition.Os
import org.elasticsearch.gradle.BwcVersions
import org.elasticsearch.gradle.FileContentsTask
import org.elasticsearch.gradle.Jdk
import org.elasticsearch.gradle.JdkDownloadPlugin
import org.elasticsearch.gradle.LoggedExec
import org.elasticsearch.gradle.Version
import org.elasticsearch.gradle.BwcVersions
import org.gradle.api.*
import org.gradle.api.GradleException
import org.gradle.api.InvalidUserDataException
import org.gradle.api.NamedDomainObjectContainer
import org.gradle.api.Plugin
import org.gradle.api.Project
import org.gradle.api.Task
import org.gradle.api.artifacts.dsl.RepositoryHandler
import org.gradle.api.execution.TaskExecutionAdapter
import org.gradle.api.internal.artifacts.dependencies.DefaultProjectDependency
Expand All @@ -15,6 +22,8 @@ import org.gradle.api.tasks.Exec
import org.gradle.api.tasks.StopExecutionException
import org.gradle.api.tasks.TaskState

import java.nio.file.Paths

import static java.util.Collections.unmodifiableList

class VagrantTestPlugin implements Plugin<Project> {
Expand Down Expand Up @@ -85,8 +94,33 @@ class VagrantTestPlugin implements Plugin<Project> {
/** extra env vars to pass to vagrant for box configuration **/
Map<String, String> vagrantBoxEnvVars = [:]

private static final String SYSTEM_JDK_VERSION = "11.0.2+9"
private static final String GRADLE_JDK_VERSION = "12.0.1+12@69cfe15208a647278a19ef0990eea691"
private Jdk linuxSystemJdk;
private Jdk linuxGradleJdk;
private Jdk windowsSystemJdk;
private Jdk windowsGradleJdk;

@Override
void apply(Project project) {
project.pluginManager.apply(JdkDownloadPlugin.class)
NamedDomainObjectContainer<Jdk> jdksContainer = (NamedDomainObjectContainer<Jdk>) project.getExtensions().getByName("jdks");
linuxSystemJdk = jdksContainer.create("linux_system") {
version = SYSTEM_JDK_VERSION
platform = "linux"
}
linuxGradleJdk = jdksContainer.create("linux_gradle") {
version = GRADLE_JDK_VERSION
platform = "linux"
}
windowsSystemJdk = jdksContainer.create("windows_system") {
version = SYSTEM_JDK_VERSION
platform = "windows"
}
windowsGradleJdk = jdksContainer.create("windows_gradle") {
version = GRADLE_JDK_VERSION
platform = "windows"
}

collectAvailableBoxes(project)

Expand Down Expand Up @@ -264,7 +298,7 @@ class VagrantTestPlugin implements Plugin<Project> {
}
}

private static void createPrepareVagrantTestEnvTask(Project project) {
private void createPrepareVagrantTestEnvTask(Project project) {
File packagingDir = new File(project.buildDir, PACKAGING_CONFIGURATION)

File archivesDir = new File(packagingDir, 'archives')
Expand All @@ -280,19 +314,23 @@ class VagrantTestPlugin implements Plugin<Project> {
}

Task createLinuxRunnerScript = project.tasks.create('createLinuxRunnerScript', FileContentsTask) {
dependsOn copyPackagingTests
dependsOn copyPackagingTests, linuxGradleJdk, linuxSystemJdk
file "${testsDir}/run-tests.sh"
contents """\
if [ "\$#" -eq 0 ]; then
test_args=( "${-> project.extensions.esvagrant.testClass}" )
else
test_args=( "\$@" )
fi
"\$SYSTEM_JAVA_HOME"/bin/java -cp "\$PACKAGING_TESTS/*" org.elasticsearch.packaging.VMTestRunner "\${test_args[@]}"
if [ -z "\$SYSTEM_JAVA_HOME" ]; then
export SYSTEM_JAVA_HOME="${-> convertPath(project, linuxSystemJdk.toString()) }"
fi
"${-> convertPath(project, linuxGradleJdk.toString()) }"/bin/java -cp "\$PACKAGING_TESTS/*" org.elasticsearch.packaging.VMTestRunner "\${test_args[@]}"
"""
}
Task createWindowsRunnerScript = project.tasks.create('createWindowsRunnerScript', FileContentsTask) {
dependsOn copyPackagingTests
dependsOn copyPackagingTests, windowsGradleJdk, windowsSystemJdk
file "${testsDir}/run-tests.ps1"
// the use of $args rather than param() here is deliberate because the syntax for array (multivalued) parameters is likely
// a little trappy for those unfamiliar with powershell
Expand All @@ -302,7 +340,8 @@ class VagrantTestPlugin implements Plugin<Project> {
} else {
\$testArgs = \$args
}
& "\$Env:SYSTEM_JAVA_HOME"/bin/java -cp "\$Env:PACKAGING_TESTS/*" org.elasticsearch.packaging.VMTestRunner @testArgs
\$Env:SYSTEM_JAVA_HOME = "${-> convertPath(project, windowsSystemJdk.toString()) }"
& "${-> convertPath(project, windowsGradleJdk.toString()) }"/bin/java -cp "\$Env:PACKAGING_TESTS/*" org.elasticsearch.packaging.VMTestRunner @testArgs
exit \$LASTEXITCODE
"""
}
Expand Down Expand Up @@ -539,10 +578,10 @@ class VagrantTestPlugin implements Plugin<Project> {

if (LINUX_BOXES.contains(box)) {
Task batsPackagingTest = project.tasks.create("vagrant${boxTask}#batsPackagingTest", BatsOverVagrantTask) {
remoteCommand BATS_TEST_COMMAND
remoteCommand "export SYSTEM_JAVA_HOME=\"${-> convertPath(project, linuxSystemJdk.toString())}\"; " + BATS_TEST_COMMAND
boxName box
environmentVars vagrantEnvVars
dependsOn up, setupPackagingTest
dependsOn up, setupPackagingTest, linuxSystemJdk
finalizedBy halt
}

Expand Down Expand Up @@ -617,4 +656,9 @@ class VagrantTestPlugin implements Plugin<Project> {
}
}
}

// convert the given path from an elasticsearch repo path to a VM path
private String convertPath(Project project, String path) {
return "/elasticsearch/" + project.rootDir.toPath().relativize(Paths.get(path));
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -28,11 +28,13 @@
import org.gradle.api.file.FileTree;
import org.gradle.api.specs.Spec;
import org.gradle.api.tasks.CacheableTask;
import org.gradle.api.tasks.Classpath;
import org.gradle.api.tasks.Input;
import org.gradle.api.tasks.InputFile;
import org.gradle.api.tasks.InputFiles;
import org.gradle.api.tasks.Internal;
import org.gradle.api.tasks.Optional;
import org.gradle.api.tasks.OutputDirectory;
import org.gradle.api.tasks.OutputFile;
import org.gradle.api.tasks.PathSensitive;
import org.gradle.api.tasks.PathSensitivity;
import org.gradle.api.tasks.SkipWhenEmpty;
Expand All @@ -45,6 +47,7 @@
import java.net.URISyntaxException;
import java.net.URL;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.util.Arrays;
import java.util.Collections;
import java.util.Set;
Expand Down Expand Up @@ -113,14 +116,19 @@ public void setJavaHome(String javaHome) {
this.javaHome = javaHome;
}

@OutputDirectory
@Internal
public File getJarExpandDir() {
return new File(
new File(getProject().getBuildDir(), "precommit/thirdPartyAudit"),
getName()
);
}

@OutputFile
public File getSuccessMarker() {
return new File(getProject().getBuildDir(), "markers/" + getName());
}

public void ignoreMissingClasses(String... classesOrPackages) {
if (classesOrPackages.length == 0) {
missingClassExcludes = null;
Expand Down Expand Up @@ -157,8 +165,7 @@ public Set<String> getMissingClassExcludes() {
return missingClassExcludes;
}

@InputFiles
@PathSensitive(PathSensitivity.NAME_ONLY)
@Classpath
@SkipWhenEmpty
public Set<File> getJarsToScan() {
// These are SelfResolvingDependency, and some of them backed by file collections, like the Gradle API files,
Expand Down Expand Up @@ -241,6 +248,10 @@ public void runThirdPartyAudit() throws IOException {
}

assertNoJarHell(jdkJarHellClasses);

// Mark successful third party audit check
getSuccessMarker().getParentFile().mkdirs();
Files.write(getSuccessMarker().toPath(), new byte[]{});
}

private void logForbiddenAPIsOutput(String forbiddenApisOutput) {
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,67 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

package org.elasticsearch.gradle;

import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStream;

/**
* An outputstream to a File that is lazily opened on the first write.
*/
class LazyFileOutputStream extends OutputStream {
private OutputStream delegate;

LazyFileOutputStream(File file) {
// use an initial dummy delegate to avoid doing a conditional on every write
this.delegate = new OutputStream() {
private void bootstrap() throws IOException {
file.getParentFile().mkdirs();
delegate = new FileOutputStream(file);
}
@Override
public void write(int b) throws IOException {
bootstrap();
delegate.write(b);
}
@Override
public void write(byte b[], int off, int len) throws IOException {
bootstrap();
delegate.write(b, off, len);
}
};
}

@Override
public void write(int b) throws IOException {
delegate.write(b);
}

@Override
public void write(byte b[], int off, int len) throws IOException {
delegate.write(b, off, len);
}

@Override
public void close() throws IOException {
delegate.close();
}
}
Loading

0 comments on commit db4ad80

Please sign in to comment.