Modernize and consolidate JDKs usage across all stages of the build. …

…Use JDK-17 as bundled JDK distribution to run tests (#1358) * Modernize and consolidate JDKs usage across all stages of the build. Use JDK-17 as bundled JDK distribution to run tests Signed-off-by: Andriy Redko <[email protected]> * Using -Djava.security.egd=file:/dev/urandom explicitly for cli tests Signed-off-by: Andriy Redko <[email protected]>
opensearch-project · Oct 13, 2021 · 3779576 · 3779576
1 parent 5a29b47
commit 3779576
Show file tree

Hide file tree

Showing 15 changed files with 67 additions and 14 deletions.
diff --git a/buildSrc/src/main/java/org/opensearch/gradle/Jdk.java b/buildSrc/src/main/java/org/opensearch/gradle/Jdk.java
@@ -49,7 +49,7 @@
 public class Jdk implements Buildable, Iterable<File> {
 
     private static final List<String> ALLOWED_ARCHITECTURES = Collections.unmodifiableList(Arrays.asList("aarch64", "x64"));
-    private static final List<String> ALLOWED_VENDORS = Collections.unmodifiableList(Arrays.asList("adoptopenjdk", "openjdk"));
+    private static final List<String> ALLOWED_VENDORS = Collections.unmodifiableList(Arrays.asList("adoptium", "adoptopenjdk", "openjdk"));
     private static final List<String> ALLOWED_PLATFORMS = Collections.unmodifiableList(Arrays.asList("darwin", "linux", "windows", "mac"));
     private static final Pattern VERSION_PATTERN = Pattern.compile("(\\d+)(\\.\\d+\\.\\d+)?\\+(\\d+(?:\\.\\d+)?)(@([a-f0-9]{32}))?");
     private static final Pattern LEGACY_VERSION_PATTERN = Pattern.compile("(\\d)(u\\d+)\\+(b\\d+?)(@([a-f0-9]{32}))?");

diff --git a/buildSrc/src/main/java/org/opensearch/gradle/JdkDownloadPlugin.java b/buildSrc/src/main/java/org/opensearch/gradle/JdkDownloadPlugin.java
@@ -46,7 +46,7 @@
 import org.gradle.api.internal.artifacts.ArtifactAttributes;
 
 public class JdkDownloadPlugin implements Plugin<Project> {
-
+    public static final String VENDOR_ADOPTIUM = "adoptium";
     public static final String VENDOR_ADOPTOPENJDK = "adoptopenjdk";
     public static final String VENDOR_OPENJDK = "openjdk";
 
@@ -108,7 +108,20 @@ private void setupRepository(Project project, Jdk jdk) {
         String repoUrl;
         String artifactPattern;
 
-        if (jdk.getVendor().equals(VENDOR_ADOPTOPENJDK)) {
+        if (jdk.getVendor().equals(VENDOR_ADOPTIUM)) {
+            repoUrl = "https://github.com/adoptium/temurin" + jdk.getMajor() + "-binaries/releases/download/";
+            artifactPattern = "jdk-"
+                + jdk.getBaseVersion()
+                + "+"
+                + jdk.getBuild()
+                + "/OpenJDK"
+                + jdk.getMajor()
+                + "-jdk_[classifier]_[module]_hotspot_"
+                + jdk.getBaseVersion()
+                + "_"
+                + jdk.getBuild()
+                + ".[ext]";
+        } else if (jdk.getVendor().equals(VENDOR_ADOPTOPENJDK)) {
             repoUrl = "https://api.adoptopenjdk.net/v3/binary/version/";
             if (jdk.getMajor().equals("8")) {
                 // legacy pattern for JDK 8
@@ -167,7 +180,7 @@ public static NamedDomainObjectContainer<Jdk> getContainer(Project project) {
 
     private static String dependencyNotation(Jdk jdk) {
         String platformDep = jdk.getPlatform().equals("darwin") || jdk.getPlatform().equals("mac")
-            ? (jdk.getVendor().equals(VENDOR_ADOPTOPENJDK) ? "mac" : "osx")
+            ? (jdk.getVendor().equals(VENDOR_OPENJDK) ? "osx" : "mac")
             : jdk.getPlatform();
         String extension = jdk.getPlatform().equals("windows") ? "zip" : "tar.gz";
 

diff --git a/buildSrc/src/test/java/org/opensearch/gradle/JdkDownloadPluginTests.java b/buildSrc/src/test/java/org/opensearch/gradle/JdkDownloadPluginTests.java
@@ -60,7 +60,7 @@ public void testUnknownVendor() {
             "11.0.2+33",
             "linux",
             "x64",
-            "unknown vendor [unknown] for jdk [testjdk], must be one of [adoptopenjdk, openjdk]"
+            "unknown vendor [unknown] for jdk [testjdk], must be one of [adoptium, adoptopenjdk, openjdk]"
         );
     }
 

diff --git a/buildSrc/version.properties b/buildSrc/version.properties
@@ -1,8 +1,8 @@
 opensearch        = 2.0.0
 lucene            = 8.9.0
 
-bundled_jdk_vendor = adoptopenjdk
-bundled_jdk = 15.0.1+9
+bundled_jdk_vendor = adoptium
+bundled_jdk = 17+35
 
 checkstyle = 8.29
 
@@ -21,7 +21,7 @@ slf4j             = 1.6.2
 jna               = 5.5.0
 
 netty             = 4.1.59.Final
-joda              = 2.10.4
+joda              = 2.10.12
 
 # when updating this version, you need to ensure compatibility with:
 #  - plugins/ingest-attachment (transitive dependency, check the upstream POM)

diff --git a/distribution/tools/plugin-cli/build.gradle b/distribution/tools/plugin-cli/build.gradle
@@ -49,6 +49,7 @@ tasks.named("dependencyLicenses").configure {
 test {
   // TODO: find a way to add permissions for the tests in this module
   systemProperty 'tests.security.manager', 'false'
+  jvmArgs += [ "-Djava.security.egd=file:/dev/urandom" ]
 }
 
 /*

diff --git a/libs/nio/build.gradle b/libs/nio/build.gradle
@@ -27,6 +27,9 @@
  * specific language governing permissions and limitations
  * under the License.
  */
+
+import org.opensearch.gradle.info.BuildParams
+
 apply plugin: 'opensearch.publish'
 
 dependencies {
@@ -47,3 +50,10 @@ tasks.named('forbiddenApisMain').configure {
   // es-all is not checked as we connect and accept sockets
   replaceSignatureFiles 'jdk-signatures'
 }
+
+tasks.test {
+    if (BuildParams.runtimeJavaVersion > JavaVersion.VERSION_1_8) {
+        jvmArgs += ["--add-opens", "java.base/java.nio.channels=ALL-UNNAMED"]
+        jvmArgs += ["--add-opens", "java.base/java.net=ALL-UNNAMED"]
+    }
+}
diff --git a/libs/ssl-config/build.gradle b/libs/ssl-config/build.gradle
@@ -27,6 +27,9 @@
  * specific language governing permissions and limitations
  * under the License.
  */
+
+import org.opensearch.gradle.info.BuildParams
+
 apply plugin: "opensearch.publish"
 
 dependencies {
@@ -52,3 +55,10 @@ forbiddenPatterns {
   exclude '**/*.p12'
   exclude '**/*.jks'
 }
+
+tasks.test {
+    if (BuildParams.runtimeJavaVersion > JavaVersion.VERSION_1_8) {
+        jvmArgs += ["--add-opens", "java.base/java.security.cert=ALL-UNNAMED"]
+    }
+}
+
diff --git a/libs/ssl-config/src/test/java/org/opensearch/common/ssl/DefaultJdkTrustConfigTests.java b/libs/ssl-config/src/test/java/org/opensearch/common/ssl/DefaultJdkTrustConfigTests.java
@@ -68,10 +68,8 @@ private void assertStandardIssuers(X509ExtendedTrustManager trustManager) {
         assertThat(trustManager.getAcceptedIssuers(), not(emptyArray()));
         // This is a sample of the CAs that we expect on every JRE.
         // We can safely change this list if the JRE's issuer list changes, but we want to assert something useful.
-        assertHasTrustedIssuer(trustManager, "VeriSign");
-        assertHasTrustedIssuer(trustManager, "GeoTrust");
+        // - https://bugs.openjdk.java.net/browse/JDK-8215012: VeriSign, GeoTrust" and "thawte" are gone
         assertHasTrustedIssuer(trustManager, "DigiCert");
-        assertHasTrustedIssuer(trustManager, "thawte");
         assertHasTrustedIssuer(trustManager, "COMODO");
     }
 

diff --git a/plugins/repository-hdfs/build.gradle b/plugins/repository-hdfs/build.gradle
@@ -223,6 +223,10 @@ for (String integTestTaskName : ['integTestHa', 'integTestSecure', 'integTestSec
           )
         }
       }
+
+      if (BuildParams.runtimeJavaVersion > JavaVersion.VERSION_1_8) {
+        jvmArgs += ["--add-opens", "java.security.jgss/sun.security.krb5=ALL-UNNAMED"]
+      }
   }
 
   testClusters."${integTestTaskName}" {

diff --git a/qa/evil-tests/build.gradle b/qa/evil-tests/build.gradle
@@ -33,6 +33,8 @@
  * integration, change default filesystem impl, mess with arbitrary
  * threads, etc.
  */
+
+import org.opensearch.gradle.info.BuildParams
 
 apply plugin: 'opensearch.testclusters'
 apply plugin: 'opensearch.standalone-test'
@@ -61,3 +63,9 @@ thirdPartyAudit {
     'com.google.common.primitives.UnsignedBytes$LexicographicalComparatorHolder$UnsafeComparator$1'
   )
 }
+
+tasks.test {
+    if (BuildParams.runtimeJavaVersion > JavaVersion.VERSION_1_8) {
+        jvmArgs += ["--add-opens", "java.base/java.lang=ALL-UNNAMED"]
+    }
+}
diff --git a/server/build.gradle b/server/build.gradle
@@ -355,3 +355,9 @@ tasks.named("licenseHeaders").configure {
     excludes << 'org/apache/lucene/search/RegexpQuery87*'
     excludes << 'org/opensearch/client/documentation/placeholder.txt'
 }
+
+tasks.test {
+    if (BuildParams.runtimeJavaVersion > JavaVersion.VERSION_1_8) {
+        jvmArgs += ["--add-opens", "java.base/java.nio.file=ALL-UNNAMED"]
+    }
+}
diff --git a/server/licenses/joda-time-2.10.12.jar.sha1 b/server/licenses/joda-time-2.10.12.jar.sha1
@@ -0,0 +1 @@
+95b3f193ad0493d94dcd7daa9ea575c30e6be5f5
diff --git a/server/licenses/joda-time-2.10.4.jar.sha1 b/server/licenses/joda-time-2.10.4.jar.sha1
diff --git a/server/src/test/java/org/opensearch/common/time/DateUtilsTests.java b/server/src/test/java/org/opensearch/common/time/DateUtilsTests.java
@@ -59,8 +59,9 @@
 
 public class DateUtilsTests extends OpenSearchTestCase {
     private static final Set<String> IGNORE = new HashSet<>(Arrays.asList(
-        "Eire", "Europe/Dublin", // dublin timezone in joda does not account for DST
-        "Asia/Qostanay" // this has been added in joda 2.10.2 but is not part of the JDK 12.0.1 tzdata yet
+        "Pacific/Enderbury",
+        "Pacific/Kanton",
+        "Pacific/Niue"
     ));
 
     public void testTimezoneIds() {

diff --git a/test/fixtures/krb5kdc-fixture/src/main/resources/provision/krb5.conf.template b/test/fixtures/krb5kdc-fixture/src/main/resources/provision/krb5.conf.template
@@ -43,6 +43,8 @@
     # udp_preference_limit = 1
     kdc_timeout = 3000
     canonicalize = true
+    # See please https://seanjmullan.org/blog/2021/09/14/jdk17 (deprecate 3DES and RC4 in Kerberos)
+    allow_weak_crypto = true
 
 [realms]
     ${REALM_NAME} = {
-Original file line number
+Diff line change
@@ Expand Up / @@ -49,6 +49,7 @@ tasks.named("dependencyLicenses").configure { @@
     test {
       // TODO: find a way to add permissions for the tests in this module
       systemProperty 'tests.security.manager', 'false'
+      jvmArgs += [ "-Djava.security.egd=file:/dev/urandom" ]
     }
     /*
@@ Expand Down @@