Fixing issues caused by 3.0 version bumping (#696)

* Update BaseNodeRequest dependency Signed-off-by: Jackie Han <[email protected]> * Migrate client transports to Apache HttpClient / Core 5.x Signed-off-by: Jackie Han <[email protected]> * Address spotlessCheck failures Signed-off-by: Jackie Han <[email protected]> * Upgrade bwc version because 1.1.0 cannot be upgraded directly to version 3.0.0 Signed-off-by: Jackie Han <[email protected]> * update bwc version to 2.4 Signed-off-by: Jackie Han <[email protected]> * Update bwc version Signed-off-by: Jackie Han <[email protected]> * update bwc download link Signed-off-by: Jackie Han <[email protected]> * use latest 2.4 build for bwc test Signed-off-by: Jackie Han <[email protected]> * move bwc constants to build script block Signed-off-by: Jackie Han <[email protected]> Signed-off-by: Jackie Han <[email protected]>
opensearch-project · Oct 21, 2022 · 7a05826 · 7a05826
1 parent 24232ae
commit 7a05826
Show file tree

Hide file tree

Showing 18 changed files with 97 additions and 78 deletions.
diff --git a/build.gradle b/build.gradle
@@ -38,10 +38,15 @@ buildscript {
                 '/latest/linux/x64/tar/builds/opensearch/plugins/opensearch-job-scheduler-' + plugin_no_snapshot + '.zip'
         anomaly_detection_build_download = 'https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/' + opensearch_no_snapshot +
                 '/latest/linux/x64/tar/builds/opensearch/plugins/opensearch-anomaly-detection-' + plugin_no_snapshot + '.zip'
-        bwcOpenSearchADDownload = 'https://ci.opensearch.org/ci/dbc/bundle-build/1.1.0/20210930/linux/x64/builds/opensearch/plugins/' +
-                'opensearch-anomaly-detection-1.1.0.0.zip'
-        bwcOpenSearchJSDownload = 'https://ci.opensearch.org/ci/dbc/bundle-build/1.1.0/20210930/linux/x64/builds/opensearch/plugins/' +
-                'opensearch-job-scheduler-1.1.0.0.zip'
+        bwcOpenSearchADDownload = 'https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.4.0/latest/linux/x64/tar/builds/' +
+                'opensearch/plugins/opensearch-anomaly-detection-2.4.0.0.zip'
+        bwcOpenSearchJSDownload = 'https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.4.0/latest/linux/x64/tar/builds/' +
+                'opensearch/plugins/opensearch-job-scheduler-2.4.0.0.zip'
+        bwcVersion = "2.4.0.0"
+        baseName = "adBwcCluster"
+        bwcFilePath = "src/test/resources/org/opensearch/ad/bwc/"
+        bwcJobSchedulerPath = bwcFilePath + "job-scheduler/"
+        bwcAnomalyDetectionPath = bwcFilePath + "anomaly-detection/"
 
         // gradle build won't print logs during test by default unless there is a failure.
         // It is useful to record intermediately information like prediction precision and recall.
@@ -136,7 +141,7 @@ configurations.all {
         force "joda-time:joda-time:${versions.joda}"
         force "com.fasterxml.jackson.core:jackson-core:2.13.4"
         force "commons-logging:commons-logging:${versions.commonslogging}"
-        force "org.apache.httpcomponents:httpcore:${versions.httpcore}"
+        force "org.apache.httpcomponents:httpcore5:${versions.httpcore5}"
         force "commons-codec:commons-codec:${versions.commonscodec}"
 
         force "org.mockito:mockito-core:2.25.0"
@@ -372,17 +377,11 @@ task integTestRemote(type: RestIntegTestTask) {
     }
 }
 
-String bwcVersion = "1.1.0.0"
-String baseName = "adBwcCluster"
-String bwcFilePath = "src/test/resources/org/opensearch/ad/bwc/"
-String bwcJobSchedulerPath = bwcFilePath + "job-scheduler/"
-String bwcAnomalyDetectionPath = bwcFilePath + "anomaly-detection/"
-
 2.times {i -> 
     testClusters {
         "${baseName}$i" {
             testDistribution = "ARCHIVE"
-            versions = ["1.1.0", opensearch_version]
+            versions = ["2.4.0", opensearch_version]
             numberOfNodes = 3
             plugin(provider(new Callable<RegularFile>(){
                 @Override

diff --git a/src/main/java/org/opensearch/ad/transport/ADCancelTaskNodeRequest.java b/src/main/java/org/opensearch/ad/transport/ADCancelTaskNodeRequest.java
@@ -13,11 +13,11 @@
 
 import java.io.IOException;
 
-import org.opensearch.action.support.nodes.BaseNodeRequest;
 import org.opensearch.common.io.stream.StreamInput;
 import org.opensearch.common.io.stream.StreamOutput;
+import org.opensearch.transport.TransportRequest;
 
-public class ADCancelTaskNodeRequest extends BaseNodeRequest {
+public class ADCancelTaskNodeRequest extends TransportRequest {
     private String detectorId;
     private String detectorTaskId;
     private String userName;

diff --git a/src/main/java/org/opensearch/ad/transport/ADStatsNodeRequest.java b/src/main/java/org/opensearch/ad/transport/ADStatsNodeRequest.java
@@ -13,14 +13,14 @@
 
 import java.io.IOException;
 
-import org.opensearch.action.support.nodes.BaseNodeRequest;
 import org.opensearch.common.io.stream.StreamInput;
 import org.opensearch.common.io.stream.StreamOutput;
+import org.opensearch.transport.TransportRequest;
 
 /**
  *  ADStatsNodeRequest to get a nodes stat
  */
-public class ADStatsNodeRequest extends BaseNodeRequest {
+public class ADStatsNodeRequest extends TransportRequest {
     private ADStatsRequest request;
 
     /**

diff --git a/src/main/java/org/opensearch/ad/transport/ADTaskProfileNodeRequest.java b/src/main/java/org/opensearch/ad/transport/ADTaskProfileNodeRequest.java
@@ -13,11 +13,11 @@
 
 import java.io.IOException;
 
-import org.opensearch.action.support.nodes.BaseNodeRequest;
 import org.opensearch.common.io.stream.StreamInput;
 import org.opensearch.common.io.stream.StreamOutput;
+import org.opensearch.transport.TransportRequest;
 
-public class ADTaskProfileNodeRequest extends BaseNodeRequest {
+public class ADTaskProfileNodeRequest extends TransportRequest {
     private String detectorId;
 
     public ADTaskProfileNodeRequest(StreamInput in) throws IOException {

diff --git a/src/main/java/org/opensearch/ad/transport/CronNodeRequest.java b/src/main/java/org/opensearch/ad/transport/CronNodeRequest.java
@@ -13,13 +13,13 @@
 
 import java.io.IOException;
 
-import org.opensearch.action.support.nodes.BaseNodeRequest;
 import org.opensearch.common.io.stream.StreamInput;
+import org.opensearch.transport.TransportRequest;
 
 /**
  *  Delete model represents the request to an individual node
  */
-public class CronNodeRequest extends BaseNodeRequest {
+public class CronNodeRequest extends TransportRequest {
 
     public CronNodeRequest() {}
 

diff --git a/src/main/java/org/opensearch/ad/transport/DeleteModelNodeRequest.java b/src/main/java/org/opensearch/ad/transport/DeleteModelNodeRequest.java
@@ -13,14 +13,14 @@
 
 import java.io.IOException;
 
-import org.opensearch.action.support.nodes.BaseNodeRequest;
 import org.opensearch.common.io.stream.StreamInput;
 import org.opensearch.common.io.stream.StreamOutput;
+import org.opensearch.transport.TransportRequest;
 
 /**
  *  Delete model represents the request to an individual node
  */
-public class DeleteModelNodeRequest extends BaseNodeRequest {
+public class DeleteModelNodeRequest extends TransportRequest {
 
     private String adID;
 

diff --git a/src/main/java/org/opensearch/ad/transport/ProfileNodeRequest.java b/src/main/java/org/opensearch/ad/transport/ProfileNodeRequest.java
@@ -14,15 +14,15 @@
 import java.io.IOException;
 import java.util.Set;
 
-import org.opensearch.action.support.nodes.BaseNodeRequest;
 import org.opensearch.ad.model.DetectorProfileName;
 import org.opensearch.common.io.stream.StreamInput;
 import org.opensearch.common.io.stream.StreamOutput;
+import org.opensearch.transport.TransportRequest;
 
 /**
  *  Class representing a nodes's profile request
  */
-public class ProfileNodeRequest extends BaseNodeRequest {
+public class ProfileNodeRequest extends TransportRequest {
     private ProfileRequest request;
 
     public ProfileNodeRequest(StreamInput in) throws IOException {

diff --git a/src/test/java/org/opensearch/ad/AnomalyDetectorRestTestCase.java b/src/test/java/org/opensearch/ad/AnomalyDetectorRestTestCase.java
@@ -21,8 +21,8 @@
 import java.util.Locale;
 import java.util.Map;
 
-import org.apache.http.HttpHeaders;
-import org.apache.http.message.BasicHeader;
+import org.apache.hc.core5.http.HttpHeaders;
+import org.apache.hc.core5.http.message.BasicHeader;
 import org.opensearch.ad.model.ADTask;
 import org.opensearch.ad.model.AnomalyDetector;
 import org.opensearch.ad.model.AnomalyDetectorExecutionInput;

diff --git a/src/test/java/org/opensearch/ad/HistoricalAnalysisRestTestCase.java b/src/test/java/org/opensearch/ad/HistoricalAnalysisRestTestCase.java
@@ -22,9 +22,10 @@
 import java.util.Set;
 import java.util.function.ToDoubleFunction;
 
-import org.apache.http.HttpHeaders;
-import org.apache.http.message.BasicHeader;
-import org.apache.http.util.EntityUtils;
+import org.apache.hc.core5.http.HttpHeaders;
+import org.apache.hc.core5.http.ParseException;
+import org.apache.hc.core5.http.io.entity.EntityUtils;
+import org.apache.hc.core5.http.message.BasicHeader;
 import org.junit.Before;
 import org.opensearch.ad.mock.model.MockSimpleLog;
 import org.opensearch.ad.model.ADTaskProfile;
@@ -62,7 +63,7 @@ public ToXContentObject[] getHistoricalAnomalyDetector(String detectorId, boolea
         return getAnomalyDetector(detectorId, header, false, returnTask, client);
     }
 
-    public ADTaskProfile getADTaskProfile(String detectorId) throws IOException {
+    public ADTaskProfile getADTaskProfile(String detectorId) throws IOException, ParseException {
         Response profileResponse = TestHelpers
             .makeRequest(
                 client(),
@@ -99,7 +100,8 @@ public Response ingestSimpleMockLog(
         ToDoubleFunction<Integer> valueFunc,
         int ipSize,
         int categorySize
-    ) throws IOException {
+    ) throws IOException,
+        ParseException {
         TestHelpers
             .makeRequest(
                 client(),
@@ -150,7 +152,7 @@ public Response ingestSimpleMockLog(
         return bulkResponse;
     }
 
-    public ADTaskProfile parseADTaskProfile(Response profileResponse) throws IOException {
+    public ADTaskProfile parseADTaskProfile(Response profileResponse) throws IOException, ParseException {
         String profileResult = EntityUtils.toString(profileResponse.getEntity());
         XContentParser parser = TestHelpers.parser(profileResult);
         ADTaskProfile adTaskProfile = null;
@@ -166,7 +168,8 @@ public ADTaskProfile parseADTaskProfile(Response profileResponse) throws IOExcep
         return adTaskProfile;
     }
 
-    protected void ingestTestDataForHistoricalAnalysis(String indexName, int detectionIntervalInMinutes) throws IOException {
+    protected void ingestTestDataForHistoricalAnalysis(String indexName, int detectionIntervalInMinutes) throws IOException,
+        ParseException {
         ingestSimpleMockLog(indexName, 10, 3000, detectionIntervalInMinutes, (i) -> {
             if (i % 500 == 0) {
                 return randomDoubleBetween(100, 1000, true);

diff --git a/src/test/java/org/opensearch/ad/ODFERestTestCase.java b/src/test/java/org/opensearch/ad/ODFERestTestCase.java
@@ -28,15 +28,19 @@
 import java.util.Optional;
 import java.util.stream.Collectors;
 
-import org.apache.http.Header;
-import org.apache.http.HttpHost;
-import org.apache.http.auth.AuthScope;
-import org.apache.http.auth.UsernamePasswordCredentials;
-import org.apache.http.client.CredentialsProvider;
-import org.apache.http.conn.ssl.NoopHostnameVerifier;
-import org.apache.http.impl.client.BasicCredentialsProvider;
-import org.apache.http.message.BasicHeader;
-import org.apache.http.ssl.SSLContextBuilder;
+import org.apache.hc.client5.http.auth.AuthScope;
+import org.apache.hc.client5.http.auth.UsernamePasswordCredentials;
+import org.apache.hc.client5.http.impl.auth.BasicCredentialsProvider;
+import org.apache.hc.client5.http.impl.nio.PoolingAsyncClientConnectionManager;
+import org.apache.hc.client5.http.impl.nio.PoolingAsyncClientConnectionManagerBuilder;
+import org.apache.hc.client5.http.ssl.ClientTlsStrategyBuilder;
+import org.apache.hc.client5.http.ssl.NoopHostnameVerifier;
+import org.apache.hc.core5.http.Header;
+import org.apache.hc.core5.http.HttpHost;
+import org.apache.hc.core5.http.message.BasicHeader;
+import org.apache.hc.core5.http.nio.ssl.TlsStrategy;
+import org.apache.hc.core5.ssl.SSLContextBuilder;
+import org.apache.hc.core5.util.Timeout;
 import org.junit.After;
 import org.opensearch.client.Request;
 import org.opensearch.client.Response;
@@ -139,7 +143,7 @@ protected RestClient buildClient(Settings settings, HttpHost[] hosts) throws IOE
     @After
     protected void wipeAllODFEIndices() throws IOException {
         Response response = adminClient().performRequest(new Request("GET", "/_cat/indices?format=json&expand_wildcards=all"));
-        XContentType xContentType = XContentType.fromMediaType(response.getEntity().getContentType().getValue());
+        XContentType xContentType = XContentType.fromMediaType(response.getEntity().getContentType());
         try (
             XContentParser parser = xContentType
                 .xContent()
@@ -181,14 +185,25 @@ protected static void configureHttpsClient(RestClientBuilder builder, Settings s
             String password = Optional
                 .ofNullable(System.getProperty("password"))
                 .orElseThrow(() -> new RuntimeException("password is missing"));
-            CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
-            credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(userName, password));
+            BasicCredentialsProvider credentialsProvider = new BasicCredentialsProvider();
+            credentialsProvider
+                .setCredentials(
+                    new AuthScope(new HttpHost("localhost", 9200)),
+                    new UsernamePasswordCredentials(userName, password.toCharArray())
+                );
             try {
-                return httpClientBuilder
-                    .setDefaultCredentialsProvider(credentialsProvider)
+                final TlsStrategy tlsStrategy = ClientTlsStrategyBuilder
+                    .create()
+                    .setSslContext(SSLContextBuilder.create().loadTrustMaterial(null, (chains, authType) -> true).build())
                     // disable the certificate since our testing cluster just uses the default security configuration
-                    .setSSLHostnameVerifier(NoopHostnameVerifier.INSTANCE)
-                    .setSSLContext(SSLContextBuilder.create().loadTrustMaterial(null, (chains, authType) -> true).build());
+                    .setHostnameVerifier(NoopHostnameVerifier.INSTANCE)
+                    .build();
+
+                final PoolingAsyncClientConnectionManager connectionManager = PoolingAsyncClientConnectionManagerBuilder
+                    .create()
+                    .setTlsStrategy(tlsStrategy)
+                    .build();
+                return httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider).setConnectionManager(connectionManager);
             } catch (Exception e) {
                 throw new RuntimeException(e);
             }
@@ -197,7 +212,8 @@ protected static void configureHttpsClient(RestClientBuilder builder, Settings s
         final String socketTimeoutString = settings.get(CLIENT_SOCKET_TIMEOUT);
         final TimeValue socketTimeout = TimeValue
             .parseTimeValue(socketTimeoutString == null ? "60s" : socketTimeoutString, CLIENT_SOCKET_TIMEOUT);
-        builder.setRequestConfigCallback(conf -> conf.setSocketTimeout(Math.toIntExact(socketTimeout.getMillis())));
+        builder
+            .setRequestConfigCallback(conf -> conf.setResponseTimeout(Timeout.ofMilliseconds(Math.toIntExact(socketTimeout.getMillis()))));
         if (settings.hasValue(CLIENT_PATH_PREFIX)) {
             builder.setPathPrefix(settings.get(CLIENT_PATH_PREFIX));
         }

diff --git a/src/test/java/org/opensearch/ad/TestHelpers.java b/src/test/java/org/opensearch/ad/TestHelpers.java
@@ -11,7 +11,7 @@
 
 package org.opensearch.ad;
 
-import static org.apache.http.entity.ContentType.APPLICATION_JSON;
+import static org.apache.hc.core5.http.ContentType.APPLICATION_JSON;
 import static org.opensearch.ad.model.AnomalyDetectorJob.ANOMALY_DETECTOR_JOB_INDEX;
 import static org.opensearch.cluster.node.DiscoveryNodeRole.BUILT_IN_ROLES;
 import static org.opensearch.common.xcontent.XContentParserUtils.ensureExpectedToken;
@@ -39,11 +39,10 @@
 import java.util.function.Consumer;
 import java.util.stream.IntStream;
 
-import org.apache.http.Header;
-import org.apache.http.HttpEntity;
-import org.apache.http.entity.ContentType;
-import org.apache.http.entity.StringEntity;
-import org.apache.http.nio.entity.NStringEntity;
+import org.apache.hc.core5.http.ContentType;
+import org.apache.hc.core5.http.Header;
+import org.apache.hc.core5.http.HttpEntity;
+import org.apache.hc.core5.http.io.entity.StringEntity;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.util.Strings;
@@ -172,7 +171,7 @@ public static Response makeRequest(
         String jsonEntity,
         List<Header> headers
     ) throws IOException {
-        HttpEntity httpEntity = Strings.isBlank(jsonEntity) ? null : new NStringEntity(jsonEntity, ContentType.APPLICATION_JSON);
+        HttpEntity httpEntity = Strings.isBlank(jsonEntity) ? null : new StringEntity(jsonEntity, ContentType.APPLICATION_JSON);
         return makeRequest(client, method, endpoint, params, httpEntity, headers);
     }
 

diff --git a/src/test/java/org/opensearch/ad/bwc/ADBackwardsCompatibilityIT.java b/src/test/java/org/opensearch/ad/bwc/ADBackwardsCompatibilityIT.java
@@ -36,7 +36,7 @@
 import java.util.Set;
 import java.util.stream.Collectors;
 
-import org.apache.http.HttpEntity;
+import org.apache.hc.core5.http.HttpEntity;
 import org.junit.Assert;
 import org.junit.Before;
 import org.opensearch.ad.TestHelpers;

diff --git a/src/test/java/org/opensearch/ad/e2e/DetectionResultEvalutationIT.java b/src/test/java/org/opensearch/ad/e2e/DetectionResultEvalutationIT.java
@@ -38,8 +38,8 @@
 import java.util.TimeZone;
 import java.util.concurrent.TimeUnit;
 
-import org.apache.http.HttpHeaders;
-import org.apache.http.message.BasicHeader;
+import org.apache.hc.core5.http.HttpHeaders;
+import org.apache.hc.core5.http.message.BasicHeader;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.core.Logger;
 import org.opensearch.ad.ODFERestTestCase;

diff --git a/src/test/java/org/opensearch/ad/mock/transport/MockADCancelTaskNodeRequest_1_0.java b/src/test/java/org/opensearch/ad/mock/transport/MockADCancelTaskNodeRequest_1_0.java
@@ -13,11 +13,11 @@
 
 import java.io.IOException;
 
-import org.opensearch.action.support.nodes.BaseNodeRequest;
 import org.opensearch.common.io.stream.StreamInput;
 import org.opensearch.common.io.stream.StreamOutput;
+import org.opensearch.transport.TransportRequest;
 
-public class MockADCancelTaskNodeRequest_1_0 extends BaseNodeRequest {
+public class MockADCancelTaskNodeRequest_1_0 extends TransportRequest {
     private String detectorId;
     private String userName;
 

diff --git a/src/test/java/org/opensearch/ad/rest/ADRestTestUtils.java b/src/test/java/org/opensearch/ad/rest/ADRestTestUtils.java
@@ -29,10 +29,11 @@
 import java.util.Set;
 import java.util.function.ToDoubleFunction;
 
-import org.apache.http.HttpEntity;
-import org.apache.http.HttpHeaders;
-import org.apache.http.message.BasicHeader;
-import org.apache.http.util.EntityUtils;
+import org.apache.hc.core5.http.HttpEntity;
+import org.apache.hc.core5.http.HttpHeaders;
+import org.apache.hc.core5.http.ParseException;
+import org.apache.hc.core5.http.io.entity.EntityUtils;
+import org.apache.hc.core5.http.message.BasicHeader;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.core.Logger;
 import org.opensearch.ad.TestHelpers;
@@ -486,7 +487,7 @@ public static ADTaskProfile waitUntilTaskReachState(RestClient client, String de
         return adTaskProfile;
     }
 
-    public static ADTaskProfile getADTaskProfile(RestClient client, String detectorId) throws IOException {
+    public static ADTaskProfile getADTaskProfile(RestClient client, String detectorId) throws IOException, ParseException {
         Response profileResponse = TestHelpers
             .makeRequest(
                 client,
@@ -499,7 +500,7 @@ public static ADTaskProfile getADTaskProfile(RestClient client, String detectorI
         return parseADTaskProfile(profileResponse);
     }
 
-    public static ADTaskProfile parseADTaskProfile(Response profileResponse) throws IOException {
+    public static ADTaskProfile parseADTaskProfile(Response profileResponse) throws IOException, ParseException {
         String profileResult = EntityUtils.toString(profileResponse.getEntity());
         XContentParser parser = TestHelpers.parser(profileResult);
         ADTaskProfile adTaskProfile = null;