Skip to content

Commit

Permalink
Merge remote-tracking branch 'elastic/master' into global-checkpoint-…
Browse files Browse the repository at this point in the history 
…listeners

* elastic/master: (58 commits)
  [ML] Partition-wise maximum scores (elastic#32748)
  [DOCS] XContentBuilder#bytes method removed, using BytesReference.bytes(docBuilder) (elastic#32771)
  HLRC: migration get assistance API (elastic#32744)
  Add a task to run forbiddenapis using cli (elastic#32076)
  [Kerberos] Add debug log statement for exceptions (elastic#32663)
  Make x-pack core pull transport-nio (elastic#32757)
  Painless: Clean Up Whitelist Names (elastic#32791)
  Cat apis: Fix index creation time to use strict date format (elastic#32510)
  Clear Job#finished_time when it is opened (elastic#32605) (elastic#32755)
  Test: Only sniff host metadata for node_selectors (elastic#32750)
  Update scripted metric docs to use `state` variable (elastic#32695)
  Painless: Clean up PainlessCast (elastic#32754)
  [TEST] Certificate NONE not allowed in FIPS JVM (elastic#32753)
  [ML] Refactor ProcessCtrl into Autodetect and Normalizer builders (elastic#32720)
  Access build tools resources (elastic#32201)
  Tests: Disable rolling upgrade tests with system key on fips JVM (elastic#32775)
  HLRC: Ban LoggingDeprecationHandler (elastic#32756)
  Fix test reproducability in AbstractBuilderTestCase setup (elastic#32403)
  Only require java<version>_home env var if needed
  Tests: Muted ScriptDocValuesDatesTests.testJodaTimeBwc
  ...
  • Loading branch information
@jasontedor
jasontedor committed Aug 13, 2018
2 parents ea3c198 + d147cd7 commit 50a9a6c
Show file tree
Hide file tree
Showing 356 changed files with 10,469 additions and 4,696 deletions.
80 changes: 60 additions & 20 deletions buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,8 @@ class BuildPlugin implements Plugin<Project> {
project.pluginManager.apply('nebula.info-scm')
project.pluginManager.apply('nebula.info-jar')

project.getTasks().create("buildResources", ExportElasticsearchBuildResourcesTask)

globalBuildInfo(project)
configureRepositories(project)
configureConfigurations(project)
Expand All @@ -101,6 +103,7 @@ class BuildPlugin implements Plugin<Project> {
configureDependenciesInfo(project)
}


/** Performs checks on the build environment and prints information about the build environment. */
static void globalBuildInfo(Project project) {
if (project.rootProject.ext.has('buildChecksDone') == false) {
Expand All @@ -116,12 +119,14 @@ class BuildPlugin implements Plugin<Project> {

final Map<Integer, String> javaVersions = [:]
for (int version = 7; version <= Integer.parseInt(minimumCompilerVersion.majorVersion); version++) {
javaVersions.put(version, findJavaHome(version));
if(System.getenv(getJavaHomeEnvVarName(version.toString())) != null) {
javaVersions.put(version, findJavaHome(version.toString()));
}
}

String javaVendor = System.getProperty('java.vendor')
String javaVersion = System.getProperty('java.version')
String gradleJavaVersionDetails = "${javaVendor} ${javaVersion}" +
String gradleJavaVersion = System.getProperty('java.version')
String gradleJavaVersionDetails = "${javaVendor} ${gradleJavaVersion}" +
" [${System.getProperty('java.vm.name')} ${System.getProperty('java.vm.version')}]"

String compilerJavaVersionDetails = gradleJavaVersionDetails
Expand All @@ -144,33 +149,33 @@ class BuildPlugin implements Plugin<Project> {
// Build debugging info
println '======================================='
println 'Elasticsearch Build Hamster says Hello!'
println '======================================='
println " Gradle Version : ${project.gradle.gradleVersion}"
println " OS Info : ${System.getProperty('os.name')} ${System.getProperty('os.version')} (${System.getProperty('os.arch')})"
if (gradleJavaVersionDetails != compilerJavaVersionDetails || gradleJavaVersionDetails != runtimeJavaVersionDetails) {
println " JDK Version (gradle) : ${gradleJavaVersionDetails}"
println " JAVA_HOME (gradle) : ${gradleJavaHome}"
println " JDK Version (compile) : ${compilerJavaVersionDetails}"
println " JAVA_HOME (compile) : ${compilerJavaHome}"
println " JDK Version (runtime) : ${runtimeJavaVersionDetails}"
println " JAVA_HOME (runtime) : ${runtimeJavaHome}"
println " Compiler JDK Version : ${getPaddedMajorVersion(compilerJavaVersionEnum)} (${compilerJavaVersionDetails})"
println " Compiler java.home : ${compilerJavaHome}"
println " Runtime JDK Version : ${getPaddedMajorVersion(runtimeJavaVersionEnum)} (${runtimeJavaVersionDetails})"
println " Runtime java.home : ${runtimeJavaHome}"
println " Gradle JDK Version : ${getPaddedMajorVersion(JavaVersion.toVersion(gradleJavaVersion))} (${gradleJavaVersionDetails})"
println " Gradle java.home : ${gradleJavaHome}"
} else {
println " JDK Version : ${gradleJavaVersionDetails}"
println " JDK Version : ${getPaddedMajorVersion(JavaVersion.toVersion(gradleJavaVersion))} (${gradleJavaVersionDetails})"
println " JAVA_HOME : ${gradleJavaHome}"
}
println " Random Testing Seed : ${project.testSeed}"
println '======================================='

// enforce Java version
if (compilerJavaVersionEnum < minimumCompilerVersion) {
final String message =
"the environment variable JAVA_HOME must be set to a JDK installation directory for Java ${minimumCompilerVersion}" +
"the compiler java.home must be set to a JDK installation directory for Java ${minimumCompilerVersion}" +
" but is [${compilerJavaHome}] corresponding to [${compilerJavaVersionEnum}]"
throw new GradleException(message)
}

if (runtimeJavaVersionEnum < minimumRuntimeVersion) {
final String message =
"the environment variable RUNTIME_JAVA_HOME must be set to a JDK installation directory for Java ${minimumRuntimeVersion}" +
"the runtime java.home must be set to a JDK installation directory for Java ${minimumRuntimeVersion}" +
" but is [${runtimeJavaHome}] corresponding to [${runtimeJavaVersionEnum}]"
throw new GradleException(message)
}
Expand Down Expand Up @@ -205,6 +210,7 @@ class BuildPlugin implements Plugin<Project> {
project.rootProject.ext.minimumCompilerVersion = minimumCompilerVersion
project.rootProject.ext.minimumRuntimeVersion = minimumRuntimeVersion
project.rootProject.ext.inFipsJvm = inFipsJvm
project.rootProject.ext.gradleJavaVersion = JavaVersion.toVersion(gradleJavaVersion)
}

project.targetCompatibility = project.rootProject.ext.minimumRuntimeVersion
Expand All @@ -217,11 +223,20 @@ class BuildPlugin implements Plugin<Project> {
project.ext.runtimeJavaVersion = project.rootProject.ext.runtimeJavaVersion
project.ext.javaVersions = project.rootProject.ext.javaVersions
project.ext.inFipsJvm = project.rootProject.ext.inFipsJvm
project.ext.gradleJavaVersion = project.rootProject.ext.gradleJavaVersion
}

private static String getPaddedMajorVersion(JavaVersion compilerJavaVersionEnum) {
compilerJavaVersionEnum.getMajorVersion().toString().padLeft(2)
}

private static String findCompilerJavaHome() {
final String javaHome = System.getenv('JAVA_HOME')
if (javaHome == null) {
final String compilerJavaHome = System.getenv('JAVA_HOME')
final String compilerJavaProperty = System.getProperty('compiler.java')
if (compilerJavaProperty != null) {
compilerJavaHome = findJavaHome(compilerJavaProperty)
}
if (compilerJavaHome == null) {
if (System.getProperty("idea.active") != null || System.getProperty("eclipse.launcher") != null) {
// IntelliJ does not set JAVA_HOME, so we use the JDK that Gradle was run with
return Jvm.current().javaHome
Expand All @@ -233,11 +248,24 @@ class BuildPlugin implements Plugin<Project> {
)
}
}
return javaHome
return compilerJavaHome
}

private static String findJavaHome(int version) {
return System.getenv('JAVA' + version + '_HOME')
private static String findJavaHome(String version) {
String versionedVarName = getJavaHomeEnvVarName(version)
String versionedJavaHome = System.getenv(versionedVarName);
if (versionedJavaHome == null) {
throw new GradleException(
"$versionedVarName must be set to build Elasticsearch. " +
"Note that if the variable was just set you might have to run `./gradlew --stop` for " +
"it to be picked up. See https://github.com/elastic/elasticsearch/issues/31399 details."
)
}
return versionedJavaHome
}

private static String getJavaHomeEnvVarName(String version) {
return 'JAVA' + version + '_HOME'
}

/** Add a check before gradle execution phase which ensures java home for the given java version is set. */
Expand Down Expand Up @@ -271,7 +299,10 @@ class BuildPlugin implements Plugin<Project> {
}

private static String findRuntimeJavaHome(final String compilerJavaHome) {
assert compilerJavaHome != null
String runtimeJavaProperty = System.getProperty("runtime.java")
if (runtimeJavaProperty != null) {
return findJavaHome(runtimeJavaProperty)
}
return System.getenv('RUNTIME_JAVA_HOME') ?: compilerJavaHome
}

Expand Down Expand Up @@ -768,6 +799,12 @@ class BuildPlugin implements Plugin<Project> {
systemProperty 'tests.task', path
systemProperty 'tests.security.manager', 'true'
systemProperty 'jna.nosys', 'true'
systemProperty 'compiler.java', project.ext.compilerJavaVersion.getMajorVersion()
if (project.ext.inFipsJvm) {
systemProperty 'runtime.java', project.ext.runtimeJavaVersion.getMajorVersion() + "FIPS"
} else {
systemProperty 'runtime.java', project.ext.runtimeJavaVersion.getMajorVersion()
}
// TODO: remove setting logging level via system property
systemProperty 'tests.logger.level', 'WARN'
for (Map.Entry<String, String> property : System.properties.entrySet()) {
Expand All @@ -782,9 +819,12 @@ class BuildPlugin implements Plugin<Project> {
}
}

// TODO: remove this once joda time is removed from scriptin in 7.0
// TODO: remove this once joda time is removed from scripting in 7.0
systemProperty 'es.scripting.use_java_time', 'true'

// TODO: remove this once ctx isn't added to update script params in 7.0
systemProperty 'es.scripting.update.ctx_in_params', 'false'

// Set the system keystore/truststore password if we're running tests in a FIPS-140 JVM
if (project.inFipsJvm) {
systemProperty 'javax.net.ssl.trustStorePassword', 'password'
Expand Down
59 changes: 58 additions & 1 deletion buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/PrecommitTasks.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,10 +20,14 @@ package org.elasticsearch.gradle.precommit

import de.thetaphi.forbiddenapis.gradle.CheckForbiddenApis
import de.thetaphi.forbiddenapis.gradle.ForbiddenApisPlugin
import org.elasticsearch.gradle.ExportElasticsearchBuildResourcesTask
import org.gradle.api.Project
import org.gradle.api.Task
import org.gradle.api.file.FileCollection
import org.gradle.api.plugins.JavaBasePlugin
import org.gradle.api.plugins.quality.Checkstyle
import org.gradle.api.tasks.JavaExec
import org.gradle.api.tasks.StopExecutionException

/**
* Validation tasks which should be run before committing. These run before tests.
Expand All @@ -40,7 +44,11 @@ class PrecommitTasks {
project.tasks.create('licenseHeaders', LicenseHeadersTask.class),
project.tasks.create('filepermissions', FilePermissionsTask.class),
project.tasks.create('jarHell', JarHellTask.class),
project.tasks.create('thirdPartyAudit', ThirdPartyAuditTask.class)]
project.tasks.create('thirdPartyAudit', ThirdPartyAuditTask.class)
]

// Configure it but don't add it as a dependency yet
configureForbiddenApisCli(project)

// tasks with just tests don't need dependency licenses, so this flag makes adding
// the task optional
Expand Down Expand Up @@ -96,9 +104,58 @@ class PrecommitTasks {
}
Task forbiddenApis = project.tasks.findByName('forbiddenApis')
forbiddenApis.group = "" // clear group, so this does not show up under verification tasks

return forbiddenApis
}

private static Task configureForbiddenApisCli(Project project) {
project.configurations.create("forbiddenApisCliJar")
project.dependencies {
forbiddenApisCliJar 'de.thetaphi:forbiddenapis:2.5'
}
Task forbiddenApisCli = project.tasks.create('forbiddenApisCli')

project.sourceSets.forEach { sourceSet ->
forbiddenApisCli.dependsOn(
project.tasks.create(sourceSet.getTaskName('forbiddenApisCli', null), JavaExec) {
ExportElasticsearchBuildResourcesTask buildResources = project.tasks.getByName('buildResources')
dependsOn(buildResources)
classpath = project.files(
project.configurations.forbiddenApisCliJar,
sourceSet.compileClasspath,
sourceSet.runtimeClasspath
)
main = 'de.thetaphi.forbiddenapis.cli.CliMain'
executable = "${project.runtimeJavaHome}/bin/java"
args "-b", 'jdk-unsafe-1.8'
args "-b", 'jdk-deprecated-1.8'
args "-b", 'jdk-non-portable'
args "-b", 'jdk-system-out'
args "-f", buildResources.copy("forbidden/jdk-signatures.txt")
args "-f", buildResources.copy("forbidden/es-all-signatures.txt")
args "--suppressannotation", '**.SuppressForbidden'
if (sourceSet.name == 'test') {
args "-f", buildResources.copy("forbidden/es-test-signatures.txt")
args "-f", buildResources.copy("forbidden/http-signatures.txt")
} else {
args "-f", buildResources.copy("forbidden/es-server-signatures.txt")
}
dependsOn sourceSet.classesTaskName
doFirst {
// Forbidden APIs expects only existing dirs, and requires at least one
FileCollection existingOutputs = sourceSet.output.classesDirs
.filter { it.exists() }
if (existingOutputs.isEmpty()) {
throw new StopExecutionException("${sourceSet.name} has no outputs")
}
existingOutputs.forEach { args "-d", it }
}
}
)
}
return forbiddenApisCli
}

private static Task configureCheckstyle(Project project) {
// Always copy the checkstyle configuration files to 'buildDir/checkstyle' since the resources could be located in a jar
// file. If the resources are located in a jar, Gradle will fail when it tries to turn the URL into a file
Expand Down
4 changes: 2 additions & 2 deletions buildSrc/src/main/groovy/org/elasticsearch/gradle/test/StandaloneRestTestPlugin.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,15 +22,14 @@ package org.elasticsearch.gradle.test

import com.carrotsearch.gradle.junit4.RandomizedTestingPlugin
import org.elasticsearch.gradle.BuildPlugin
import org.elasticsearch.gradle.ExportElasticsearchBuildResourcesTask
import org.elasticsearch.gradle.VersionProperties
import org.elasticsearch.gradle.precommit.PrecommitTasks
import org.gradle.api.InvalidUserDataException
import org.gradle.api.Plugin
import org.gradle.api.Project
import org.gradle.api.Task
import org.gradle.api.plugins.JavaBasePlugin
import org.gradle.api.tasks.compile.JavaCompile

/**
* Configures the build to compile tests against Elasticsearch's test framework
* and run REST tests. Use BuildPlugin if you want to build main code as well
Expand All @@ -48,6 +47,7 @@ public class StandaloneRestTestPlugin implements Plugin<Project> {
project.pluginManager.apply(JavaBasePlugin)
project.pluginManager.apply(RandomizedTestingPlugin)

project.getTasks().create("buildResources", ExportElasticsearchBuildResourcesTask)
BuildPlugin.globalBuildInfo(project)
BuildPlugin.configureRepositories(project)

Expand Down
Loading

0 comments on commit 50a9a6c

Please sign in to comment.