From 3779576c5170fe4bd9ea781087ac54cd7917a040 Mon Sep 17 00:00:00 2001 From: Andriy Redko Date: Wed, 13 Oct 2021 17:25:48 -0400 Subject: [PATCH] Modernize and consolidate JDKs usage across all stages of the build. Use JDK-17 as bundled JDK distribution to run tests (#1358) * Modernize and consolidate JDKs usage across all stages of the build. Use JDK-17 as bundled JDK distribution to run tests Signed-off-by: Andriy Redko * Using -Djava.security.egd=file:/dev/urandom explicitly for cli tests Signed-off-by: Andriy Redko --- .../main/java/org/opensearch/gradle/Jdk.java | 2 +- .../opensearch/gradle/JdkDownloadPlugin.java | 19 ++++++++++++++++--- .../gradle/JdkDownloadPluginTests.java | 2 +- buildSrc/version.properties | 6 +++--- distribution/tools/plugin-cli/build.gradle | 1 + libs/nio/build.gradle | 10 ++++++++++ libs/ssl-config/build.gradle | 10 ++++++++++ .../ssl/DefaultJdkTrustConfigTests.java | 4 +--- plugins/repository-hdfs/build.gradle | 4 ++++ qa/evil-tests/build.gradle | 8 ++++++++ server/build.gradle | 6 ++++++ server/licenses/joda-time-2.10.12.jar.sha1 | 1 + server/licenses/joda-time-2.10.4.jar.sha1 | 1 - .../common/time/DateUtilsTests.java | 5 +++-- .../resources/provision/krb5.conf.template | 2 ++ 15 files changed, 67 insertions(+), 14 deletions(-) create mode 100644 server/licenses/joda-time-2.10.12.jar.sha1 delete mode 100644 server/licenses/joda-time-2.10.4.jar.sha1 diff --git a/buildSrc/src/main/java/org/opensearch/gradle/Jdk.java b/buildSrc/src/main/java/org/opensearch/gradle/Jdk.java index 60256dc91dc4e..f305aee7fcbaa 100644 --- a/buildSrc/src/main/java/org/opensearch/gradle/Jdk.java +++ b/buildSrc/src/main/java/org/opensearch/gradle/Jdk.java @@ -49,7 +49,7 @@ public class Jdk implements Buildable, Iterable { private static final List ALLOWED_ARCHITECTURES = Collections.unmodifiableList(Arrays.asList("aarch64", "x64")); - private static final List ALLOWED_VENDORS = Collections.unmodifiableList(Arrays.asList("adoptopenjdk", "openjdk")); + private static final List ALLOWED_VENDORS = Collections.unmodifiableList(Arrays.asList("adoptium", "adoptopenjdk", "openjdk")); private static final List ALLOWED_PLATFORMS = Collections.unmodifiableList(Arrays.asList("darwin", "linux", "windows", "mac")); private static final Pattern VERSION_PATTERN = Pattern.compile("(\\d+)(\\.\\d+\\.\\d+)?\\+(\\d+(?:\\.\\d+)?)(@([a-f0-9]{32}))?"); private static final Pattern LEGACY_VERSION_PATTERN = Pattern.compile("(\\d)(u\\d+)\\+(b\\d+?)(@([a-f0-9]{32}))?"); diff --git a/buildSrc/src/main/java/org/opensearch/gradle/JdkDownloadPlugin.java b/buildSrc/src/main/java/org/opensearch/gradle/JdkDownloadPlugin.java index b8d4bcb007e8d..00f4e026d33b0 100644 --- a/buildSrc/src/main/java/org/opensearch/gradle/JdkDownloadPlugin.java +++ b/buildSrc/src/main/java/org/opensearch/gradle/JdkDownloadPlugin.java @@ -46,7 +46,7 @@ import org.gradle.api.internal.artifacts.ArtifactAttributes; public class JdkDownloadPlugin implements Plugin { - + public static final String VENDOR_ADOPTIUM = "adoptium"; public static final String VENDOR_ADOPTOPENJDK = "adoptopenjdk"; public static final String VENDOR_OPENJDK = "openjdk"; @@ -108,7 +108,20 @@ private void setupRepository(Project project, Jdk jdk) { String repoUrl; String artifactPattern; - if (jdk.getVendor().equals(VENDOR_ADOPTOPENJDK)) { + if (jdk.getVendor().equals(VENDOR_ADOPTIUM)) { + repoUrl = "https://github.com/adoptium/temurin" + jdk.getMajor() + "-binaries/releases/download/"; + artifactPattern = "jdk-" + + jdk.getBaseVersion() + + "+" + + jdk.getBuild() + + "/OpenJDK" + + jdk.getMajor() + + "-jdk_[classifier]_[module]_hotspot_" + + jdk.getBaseVersion() + + "_" + + jdk.getBuild() + + ".[ext]"; + } else if (jdk.getVendor().equals(VENDOR_ADOPTOPENJDK)) { repoUrl = "https://api.adoptopenjdk.net/v3/binary/version/"; if (jdk.getMajor().equals("8")) { // legacy pattern for JDK 8 @@ -167,7 +180,7 @@ public static NamedDomainObjectContainer getContainer(Project project) { private static String dependencyNotation(Jdk jdk) { String platformDep = jdk.getPlatform().equals("darwin") || jdk.getPlatform().equals("mac") - ? (jdk.getVendor().equals(VENDOR_ADOPTOPENJDK) ? "mac" : "osx") + ? (jdk.getVendor().equals(VENDOR_OPENJDK) ? "osx" : "mac") : jdk.getPlatform(); String extension = jdk.getPlatform().equals("windows") ? "zip" : "tar.gz"; diff --git a/buildSrc/src/test/java/org/opensearch/gradle/JdkDownloadPluginTests.java b/buildSrc/src/test/java/org/opensearch/gradle/JdkDownloadPluginTests.java index 1d3ab3935c692..7facf603c0133 100644 --- a/buildSrc/src/test/java/org/opensearch/gradle/JdkDownloadPluginTests.java +++ b/buildSrc/src/test/java/org/opensearch/gradle/JdkDownloadPluginTests.java @@ -60,7 +60,7 @@ public void testUnknownVendor() { "11.0.2+33", "linux", "x64", - "unknown vendor [unknown] for jdk [testjdk], must be one of [adoptopenjdk, openjdk]" + "unknown vendor [unknown] for jdk [testjdk], must be one of [adoptium, adoptopenjdk, openjdk]" ); } diff --git a/buildSrc/version.properties b/buildSrc/version.properties index cda7419e50ed8..6270d7207a4d3 100644 --- a/buildSrc/version.properties +++ b/buildSrc/version.properties @@ -1,8 +1,8 @@ opensearch = 2.0.0 lucene = 8.9.0 -bundled_jdk_vendor = adoptopenjdk -bundled_jdk = 15.0.1+9 +bundled_jdk_vendor = adoptium +bundled_jdk = 17+35 checkstyle = 8.29 @@ -21,7 +21,7 @@ slf4j = 1.6.2 jna = 5.5.0 netty = 4.1.59.Final -joda = 2.10.4 +joda = 2.10.12 # when updating this version, you need to ensure compatibility with: # - plugins/ingest-attachment (transitive dependency, check the upstream POM) diff --git a/distribution/tools/plugin-cli/build.gradle b/distribution/tools/plugin-cli/build.gradle index db6a033cdb699..21f59a8777488 100644 --- a/distribution/tools/plugin-cli/build.gradle +++ b/distribution/tools/plugin-cli/build.gradle @@ -49,6 +49,7 @@ tasks.named("dependencyLicenses").configure { test { // TODO: find a way to add permissions for the tests in this module systemProperty 'tests.security.manager', 'false' + jvmArgs += [ "-Djava.security.egd=file:/dev/urandom" ] } /* diff --git a/libs/nio/build.gradle b/libs/nio/build.gradle index 794a544607c20..cae9f7e6feb26 100644 --- a/libs/nio/build.gradle +++ b/libs/nio/build.gradle @@ -27,6 +27,9 @@ * specific language governing permissions and limitations * under the License. */ + +import org.opensearch.gradle.info.BuildParams + apply plugin: 'opensearch.publish' dependencies { @@ -47,3 +50,10 @@ tasks.named('forbiddenApisMain').configure { // es-all is not checked as we connect and accept sockets replaceSignatureFiles 'jdk-signatures' } + +tasks.test { + if (BuildParams.runtimeJavaVersion > JavaVersion.VERSION_1_8) { + jvmArgs += ["--add-opens", "java.base/java.nio.channels=ALL-UNNAMED"] + jvmArgs += ["--add-opens", "java.base/java.net=ALL-UNNAMED"] + } +} diff --git a/libs/ssl-config/build.gradle b/libs/ssl-config/build.gradle index 0186f30696a6e..740d5e309350c 100644 --- a/libs/ssl-config/build.gradle +++ b/libs/ssl-config/build.gradle @@ -27,6 +27,9 @@ * specific language governing permissions and limitations * under the License. */ + +import org.opensearch.gradle.info.BuildParams + apply plugin: "opensearch.publish" dependencies { @@ -52,3 +55,10 @@ forbiddenPatterns { exclude '**/*.p12' exclude '**/*.jks' } + +tasks.test { + if (BuildParams.runtimeJavaVersion > JavaVersion.VERSION_1_8) { + jvmArgs += ["--add-opens", "java.base/java.security.cert=ALL-UNNAMED"] + } +} + diff --git a/libs/ssl-config/src/test/java/org/opensearch/common/ssl/DefaultJdkTrustConfigTests.java b/libs/ssl-config/src/test/java/org/opensearch/common/ssl/DefaultJdkTrustConfigTests.java index bc83f00575481..e767787d67d6d 100644 --- a/libs/ssl-config/src/test/java/org/opensearch/common/ssl/DefaultJdkTrustConfigTests.java +++ b/libs/ssl-config/src/test/java/org/opensearch/common/ssl/DefaultJdkTrustConfigTests.java @@ -68,10 +68,8 @@ private void assertStandardIssuers(X509ExtendedTrustManager trustManager) { assertThat(trustManager.getAcceptedIssuers(), not(emptyArray())); // This is a sample of the CAs that we expect on every JRE. // We can safely change this list if the JRE's issuer list changes, but we want to assert something useful. - assertHasTrustedIssuer(trustManager, "VeriSign"); - assertHasTrustedIssuer(trustManager, "GeoTrust"); + // - https://bugs.openjdk.java.net/browse/JDK-8215012: VeriSign, GeoTrust" and "thawte" are gone assertHasTrustedIssuer(trustManager, "DigiCert"); - assertHasTrustedIssuer(trustManager, "thawte"); assertHasTrustedIssuer(trustManager, "COMODO"); } diff --git a/plugins/repository-hdfs/build.gradle b/plugins/repository-hdfs/build.gradle index 91fcf3828ba77..dfe94c266b392 100644 --- a/plugins/repository-hdfs/build.gradle +++ b/plugins/repository-hdfs/build.gradle @@ -223,6 +223,10 @@ for (String integTestTaskName : ['integTestHa', 'integTestSecure', 'integTestSec ) } } + + if (BuildParams.runtimeJavaVersion > JavaVersion.VERSION_1_8) { + jvmArgs += ["--add-opens", "java.security.jgss/sun.security.krb5=ALL-UNNAMED"] + } } testClusters."${integTestTaskName}" { diff --git a/qa/evil-tests/build.gradle b/qa/evil-tests/build.gradle index dde39e56df9df..691115864de16 100644 --- a/qa/evil-tests/build.gradle +++ b/qa/evil-tests/build.gradle @@ -33,6 +33,8 @@ * integration, change default filesystem impl, mess with arbitrary * threads, etc. */ + +import org.opensearch.gradle.info.BuildParams apply plugin: 'opensearch.testclusters' apply plugin: 'opensearch.standalone-test' @@ -61,3 +63,9 @@ thirdPartyAudit { 'com.google.common.primitives.UnsignedBytes$LexicographicalComparatorHolder$UnsafeComparator$1' ) } + +tasks.test { + if (BuildParams.runtimeJavaVersion > JavaVersion.VERSION_1_8) { + jvmArgs += ["--add-opens", "java.base/java.lang=ALL-UNNAMED"] + } +} diff --git a/server/build.gradle b/server/build.gradle index 923e80d61d63a..b925604d4e405 100644 --- a/server/build.gradle +++ b/server/build.gradle @@ -355,3 +355,9 @@ tasks.named("licenseHeaders").configure { excludes << 'org/apache/lucene/search/RegexpQuery87*' excludes << 'org/opensearch/client/documentation/placeholder.txt' } + +tasks.test { + if (BuildParams.runtimeJavaVersion > JavaVersion.VERSION_1_8) { + jvmArgs += ["--add-opens", "java.base/java.nio.file=ALL-UNNAMED"] + } +} diff --git a/server/licenses/joda-time-2.10.12.jar.sha1 b/server/licenses/joda-time-2.10.12.jar.sha1 new file mode 100644 index 0000000000000..538f23152f69d --- /dev/null +++ b/server/licenses/joda-time-2.10.12.jar.sha1 @@ -0,0 +1 @@ +95b3f193ad0493d94dcd7daa9ea575c30e6be5f5 \ No newline at end of file diff --git a/server/licenses/joda-time-2.10.4.jar.sha1 b/server/licenses/joda-time-2.10.4.jar.sha1 deleted file mode 100644 index 878998e54db2b..0000000000000 --- a/server/licenses/joda-time-2.10.4.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -8c10bb8815109067ce3c91a8e547b5a52e8a1c1a \ No newline at end of file diff --git a/server/src/test/java/org/opensearch/common/time/DateUtilsTests.java b/server/src/test/java/org/opensearch/common/time/DateUtilsTests.java index f0d1093b3eabd..3c6bc3ea20e61 100644 --- a/server/src/test/java/org/opensearch/common/time/DateUtilsTests.java +++ b/server/src/test/java/org/opensearch/common/time/DateUtilsTests.java @@ -59,8 +59,9 @@ public class DateUtilsTests extends OpenSearchTestCase { private static final Set IGNORE = new HashSet<>(Arrays.asList( - "Eire", "Europe/Dublin", // dublin timezone in joda does not account for DST - "Asia/Qostanay" // this has been added in joda 2.10.2 but is not part of the JDK 12.0.1 tzdata yet + "Pacific/Enderbury", + "Pacific/Kanton", + "Pacific/Niue" )); public void testTimezoneIds() { diff --git a/test/fixtures/krb5kdc-fixture/src/main/resources/provision/krb5.conf.template b/test/fixtures/krb5kdc-fixture/src/main/resources/provision/krb5.conf.template index 57fd8ac849708..ba0832b2b7d99 100644 --- a/test/fixtures/krb5kdc-fixture/src/main/resources/provision/krb5.conf.template +++ b/test/fixtures/krb5kdc-fixture/src/main/resources/provision/krb5.conf.template @@ -43,6 +43,8 @@ # udp_preference_limit = 1 kdc_timeout = 3000 canonicalize = true + # See please https://seanjmullan.org/blog/2021/09/14/jdk17 (deprecate 3DES and RC4 in Kerberos) + allow_weak_crypto = true [realms] ${REALM_NAME} = {