From e974d2654d72a843f2106095ea9491069226e6c0 Mon Sep 17 00:00:00 2001 From: Heemin Kim Date: Thu, 13 Oct 2022 13:53:49 -0700 Subject: [PATCH] Update http client package to resolve build failure (#168) Signed-off-by: Heemin Kim --- .../geospatial/GeospatialRestTestCase.java | 16 ++--- .../OpenSearchSecureRestTestCase.java | 58 +++++++++++++------ .../mapper/xypoint/XYPointFieldMapperIT.java | 19 +++--- .../mapper/xyshape/XYShapeFieldMapperIT.java | 6 +- .../geospatial/plugin/GeospatialPluginIT.java | 6 +- .../processor/FeatureProcessorIT.java | 7 +-- .../geojson/RestUploadGeoJSONActionIT.java | 6 +- .../stats/upload/RestUploadStatsActionIT.java | 10 ++-- 8 files changed, 72 insertions(+), 56 deletions(-) diff --git a/src/test/java/org/opensearch/geospatial/GeospatialRestTestCase.java b/src/test/java/org/opensearch/geospatial/GeospatialRestTestCase.java index 0ce1a1e5..6b71b696 100644 --- a/src/test/java/org/opensearch/geospatial/GeospatialRestTestCase.java +++ b/src/test/java/org/opensearch/geospatial/GeospatialRestTestCase.java @@ -24,7 +24,7 @@ import java.util.Optional; import java.util.stream.IntStream; -import org.apache.http.util.EntityUtils; +import org.apache.hc.core5.http.io.entity.EntityUtils; import org.hamcrest.MatcherAssert; import org.hamcrest.Matchers; import org.json.JSONArray; @@ -142,7 +142,7 @@ protected Map buildGeoJSONFeatureProcessorConfig(Map getDocument(String docID, String indexName) throws IOException { + public Map getDocument(String docID, String indexName) throws Exception { String path = String.join(URL_DELIMITER, indexName, DOC, docID); final Request request = new Request("GET", path); final Response response = client().performRequest(request); @@ -187,7 +187,7 @@ protected void assertIndexNotExists(String indexName) throws IOException { /* Get index mapping as map */ - protected Map getIndexMapping(String index) throws IOException { + protected Map getIndexMapping(String index) throws Exception { String indexMappingURL = String.join(URL_DELIMITER, index, MAPPING); Request request = new Request("GET", indexMappingURL); Response response = client().performRequest(request); @@ -201,13 +201,13 @@ protected Map getIndexMapping(String index) throws IOException { /* Get index mapping's properties as map */ - protected Map getIndexProperties(String index) throws IOException { + protected Map getIndexProperties(String index) throws Exception { final Map indexMapping = getIndexMapping(index); MatcherAssert.assertThat("No properties found for index: " + index, indexMapping, Matchers.hasKey(MAPPING_PROPERTIES_KEY)); return (Map) indexMapping.get(MAPPING_PROPERTIES_KEY); } - protected int getIndexDocumentCount(String index) throws IOException { + protected int getIndexDocumentCount(String index) throws Exception { String indexDocumentCountPath = String.join(URL_DELIMITER, index, COUNT); Request request = new Request("GET", indexDocumentCountPath); Response response = client().performRequest(request); @@ -264,7 +264,7 @@ public String buildSearchBodyAsString( }); } - public SearchResponse searchIndex(String indexName, String entity) throws IOException { + public SearchResponse searchIndex(String indexName, String entity) throws Exception { String path = String.join(URL_DELIMITER, indexName, SEARCH); final Request request = new Request("GET", path); request.setJsonEntity(entity); @@ -298,7 +298,7 @@ public String indexDocumentUsingGeoJSON(String indexName, String fieldName, Geom } public SearchResponse searchUsingShapeRelation(String indexName, String fieldName, Geometry geometry, ShapeRelation shapeRelation) - throws IOException { + throws Exception { String searchEntity = buildSearchBodyAsString(builder -> { builder.field(DEFAULT_SHAPE_FIELD_NAME); GeoJson.toXContent(geometry, builder, EMPTY_PARAMS); @@ -320,7 +320,7 @@ public SearchResponse searchUsingIndexedShapeIndex( String indexedShapePath, String docId, String fieldName - ) throws IOException { + ) throws Exception { String searchEntity = buildSearchBodyAsString(builder -> { builder.startObject(INDEXED_SHAPE_FIELD); builder.field(SHAPE_INDEX_FIELD, indexedShapeIndex); diff --git a/src/test/java/org/opensearch/geospatial/OpenSearchSecureRestTestCase.java b/src/test/java/org/opensearch/geospatial/OpenSearchSecureRestTestCase.java index 851595e9..59667296 100644 --- a/src/test/java/org/opensearch/geospatial/OpenSearchSecureRestTestCase.java +++ b/src/test/java/org/opensearch/geospatial/OpenSearchSecureRestTestCase.java @@ -8,19 +8,29 @@ package org.opensearch.geospatial; +import static org.opensearch.client.RestClientBuilder.DEFAULT_MAX_CONN_PER_ROUTE; +import static org.opensearch.client.RestClientBuilder.DEFAULT_MAX_CONN_TOTAL; + import java.io.IOException; -import java.util.*; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Optional; import java.util.stream.Collectors; -import org.apache.http.Header; -import org.apache.http.HttpHost; -import org.apache.http.auth.AuthScope; -import org.apache.http.auth.UsernamePasswordCredentials; -import org.apache.http.client.CredentialsProvider; -import org.apache.http.conn.ssl.NoopHostnameVerifier; -import org.apache.http.impl.client.BasicCredentialsProvider; -import org.apache.http.message.BasicHeader; -import org.apache.http.ssl.SSLContextBuilder; +import org.apache.hc.client5.http.auth.AuthScope; +import org.apache.hc.client5.http.auth.UsernamePasswordCredentials; +import org.apache.hc.client5.http.impl.auth.BasicCredentialsProvider; +import org.apache.hc.client5.http.impl.nio.PoolingAsyncClientConnectionManager; +import org.apache.hc.client5.http.impl.nio.PoolingAsyncClientConnectionManagerBuilder; +import org.apache.hc.client5.http.ssl.ClientTlsStrategyBuilder; +import org.apache.hc.client5.http.ssl.NoopHostnameVerifier; +import org.apache.hc.core5.http.Header; +import org.apache.hc.core5.http.HttpHost; +import org.apache.hc.core5.http.message.BasicHeader; +import org.apache.hc.core5.http.nio.ssl.TlsStrategy; +import org.apache.hc.core5.ssl.SSLContextBuilder; +import org.apache.hc.core5.util.Timeout; import org.junit.After; import org.opensearch.client.Request; import org.opensearch.client.Response; @@ -95,13 +105,20 @@ private void configureHttpsClient(RestClientBuilder builder, Settings settings) .orElseThrow(() -> new RuntimeException("user name is missing")); String password = Optional.ofNullable(System.getProperty(SYS_PROPERTY_KEY_PASSWORD)) .orElseThrow(() -> new RuntimeException("password is missing")); - CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); - credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(userName, password)); + BasicCredentialsProvider credentialsProvider = new BasicCredentialsProvider(); + final AuthScope anyScope = new AuthScope(null, -1); + credentialsProvider.setCredentials(anyScope, new UsernamePasswordCredentials(userName, password.toCharArray())); try { - return httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider) - // disable the certificate since our testing cluster just uses the default security configuration - .setSSLHostnameVerifier(NoopHostnameVerifier.INSTANCE) - .setSSLContext(SSLContextBuilder.create().loadTrustMaterial(null, (chains, authType) -> true).build()); + final TlsStrategy tlsStrategy = ClientTlsStrategyBuilder.create() + .setHostnameVerifier(NoopHostnameVerifier.INSTANCE) + .setSslContext(SSLContextBuilder.create().loadTrustMaterial(null, (chains, authType) -> true).build()) + .build(); + final PoolingAsyncClientConnectionManager connectionManager = PoolingAsyncClientConnectionManagerBuilder.create() + .setMaxConnPerRoute(DEFAULT_MAX_CONN_PER_ROUTE) + .setMaxConnTotal(DEFAULT_MAX_CONN_TOTAL) + .setTlsStrategy(tlsStrategy) + .build(); + return httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider).setConnectionManager(connectionManager); } catch (Exception e) { throw new RuntimeException(e); } @@ -112,7 +129,12 @@ private void configureHttpsClient(RestClientBuilder builder, Settings settings) socketTimeoutString == null ? DEFAULT_SOCKET_TIMEOUT : socketTimeoutString, CLIENT_SOCKET_TIMEOUT ); - builder.setRequestConfigCallback(conf -> conf.setSocketTimeout(Math.toIntExact(socketTimeout.getMillis()))); + builder.setRequestConfigCallback(conf -> { + Timeout timeout = Timeout.ofMilliseconds(Math.toIntExact(socketTimeout.getMillis())); + conf.setConnectTimeout(timeout); + conf.setResponseTimeout(timeout); + return conf; + }); if (settings.hasValue(CLIENT_PATH_PREFIX)) { builder.setPathPrefix(settings.get(CLIENT_PATH_PREFIX)); } @@ -129,7 +151,7 @@ protected boolean preserveIndicesUponCompletion() { @After public void deleteExternalIndices() throws IOException { Response response = client().performRequest(new Request("GET", "/_cat/indices?format=json&expand_wildcards=all")); - XContentType xContentType = XContentType.fromMediaType(response.getEntity().getContentType().getValue()); + XContentType xContentType = XContentType.fromMediaType(response.getEntity().getContentType()); try ( XContentParser parser = xContentType.xContent() .createParser( diff --git a/src/test/java/org/opensearch/geospatial/index/mapper/xypoint/XYPointFieldMapperIT.java b/src/test/java/org/opensearch/geospatial/index/mapper/xypoint/XYPointFieldMapperIT.java index 60758cd5..f7e4c90b 100644 --- a/src/test/java/org/opensearch/geospatial/index/mapper/xypoint/XYPointFieldMapperIT.java +++ b/src/test/java/org/opensearch/geospatial/index/mapper/xypoint/XYPointFieldMapperIT.java @@ -5,7 +5,6 @@ package org.opensearch.geospatial.index.mapper.xypoint; -import java.io.IOException; import java.util.List; import java.util.Locale; import java.util.Map; @@ -21,7 +20,7 @@ public class XYPointFieldMapperIT extends GeospatialRestTestCase { private static final String FIELD_X_KEY = "x"; private static final String FIELD_Y_KEY = "y"; - public void testMappingWithXYPointField() throws IOException { + public void testMappingWithXYPointField() throws Exception { String indexName = GeospatialTestHelper.randomLowerCaseString(); String fieldName = GeospatialTestHelper.randomLowerCaseString(); createIndex(indexName, Settings.EMPTY, Map.of(fieldName, XYPointFieldMapper.CONTENT_TYPE)); @@ -32,7 +31,7 @@ public void testMappingWithXYPointField() throws IOException { deleteIndex(indexName); } - public void testIndexWithXYPointFieldAsWKTFormat() throws IOException { + public void testIndexWithXYPointFieldAsWKTFormat() throws Exception { String indexName = GeospatialTestHelper.randomLowerCaseString(); String fieldName = GeospatialTestHelper.randomLowerCaseString(); createIndex(indexName, Settings.EMPTY, Map.of(fieldName, XYPointFieldMapper.CONTENT_TYPE)); @@ -45,7 +44,7 @@ public void testIndexWithXYPointFieldAsWKTFormat() throws IOException { deleteIndex(indexName); } - public void testIndexWithXYPointFieldAsArrayFormat() throws IOException { + public void testIndexWithXYPointFieldAsArrayFormat() throws Exception { String indexName = GeospatialTestHelper.randomLowerCaseString(); String fieldName = GeospatialTestHelper.randomLowerCaseString(); createIndex(indexName, Settings.EMPTY, Map.of(fieldName, XYPointFieldMapper.CONTENT_TYPE)); @@ -58,7 +57,7 @@ public void testIndexWithXYPointFieldAsArrayFormat() throws IOException { deleteIndex(indexName); } - public void testIndexWithXYPointFieldAsStringFormat() throws IOException { + public void testIndexWithXYPointFieldAsStringFormat() throws Exception { String indexName = GeospatialTestHelper.randomLowerCaseString(); String fieldName = GeospatialTestHelper.randomLowerCaseString(); createIndex(indexName, Settings.EMPTY, Map.of(fieldName, XYPointFieldMapper.CONTENT_TYPE)); @@ -72,7 +71,7 @@ public void testIndexWithXYPointFieldAsStringFormat() throws IOException { deleteIndex(indexName); } - public void testIndexWithXYPointFieldAsObjectFormat() throws IOException { + public void testIndexWithXYPointFieldAsObjectFormat() throws Exception { String indexName = GeospatialTestHelper.randomLowerCaseString(); String fieldName = GeospatialTestHelper.randomLowerCaseString(); createIndex(indexName, Settings.EMPTY, Map.of(fieldName, XYPointFieldMapper.CONTENT_TYPE)); @@ -86,19 +85,19 @@ public void testIndexWithXYPointFieldAsObjectFormat() throws IOException { deleteIndex(indexName); } - private String getDocumentWithWKTValueForXYPoint(String fieldName, Geometry geometry) throws IOException { + private String getDocumentWithWKTValueForXYPoint(String fieldName, Geometry geometry) throws Exception { return buildContentAsString(build -> build.field(fieldName, geometry.toString())); } - private String getDocumentWithArrayValueForXYPoint(String fieldName, Point point) throws IOException { + private String getDocumentWithArrayValueForXYPoint(String fieldName, Point point) throws Exception { return buildContentAsString(build -> build.field(fieldName, new double[] { point.getY(), point.getX() })); } - private String getDocumentWithStringValueForXYPoint(String fieldName, String pointAsString) throws IOException { + private String getDocumentWithStringValueForXYPoint(String fieldName, String pointAsString) throws Exception { return buildContentAsString(build -> build.field(fieldName, pointAsString)); } - private String getDocumentWithObjectValueForXYPoint(String fieldName, Point point) throws IOException { + private String getDocumentWithObjectValueForXYPoint(String fieldName, Point point) throws Exception { return buildContentAsString(build -> { build.startObject(fieldName); build.field(FIELD_X_KEY, point.getX()); diff --git a/src/test/java/org/opensearch/geospatial/index/mapper/xyshape/XYShapeFieldMapperIT.java b/src/test/java/org/opensearch/geospatial/index/mapper/xyshape/XYShapeFieldMapperIT.java index c3ee8c75..268f79f8 100644 --- a/src/test/java/org/opensearch/geospatial/index/mapper/xyshape/XYShapeFieldMapperIT.java +++ b/src/test/java/org/opensearch/geospatial/index/mapper/xyshape/XYShapeFieldMapperIT.java @@ -33,7 +33,7 @@ private String getDocumentWithWKTValueForXYShape(String fieldName, Geometry geom return buildContentAsString(build -> build.field(fieldName, geometry.toString())); } - public void testMappingWithXYShapeField() throws IOException { + public void testMappingWithXYShapeField() throws Exception { String indexName = GeospatialTestHelper.randomLowerCaseString(); String fieldName = GeospatialTestHelper.randomLowerCaseString(); createIndex(indexName, Settings.EMPTY, Map.of(fieldName, XYShapeFieldMapper.CONTENT_TYPE)); @@ -44,7 +44,7 @@ public void testMappingWithXYShapeField() throws IOException { deleteIndex(indexName); } - public void testIndexWithXYShapeFieldAsWKTFormat() throws IOException { + public void testIndexWithXYShapeFieldAsWKTFormat() throws Exception { String indexName = GeospatialTestHelper.randomLowerCaseString(); String fieldName = GeospatialTestHelper.randomLowerCaseString(); createIndex(indexName, Settings.EMPTY, Map.of(fieldName, XYShapeFieldMapper.CONTENT_TYPE)); @@ -57,7 +57,7 @@ public void testIndexWithXYShapeFieldAsWKTFormat() throws IOException { deleteIndex(indexName); } - public void testIndexWithXYShapeFieldAsGeoJSONFormat() throws IOException { + public void testIndexWithXYShapeFieldAsGeoJSONFormat() throws Exception { String indexName = GeospatialTestHelper.randomLowerCaseString(); String fieldName = GeospatialTestHelper.randomLowerCaseString(); createIndex(indexName, Settings.EMPTY, Map.of(fieldName, XYShapeFieldMapper.CONTENT_TYPE)); diff --git a/src/test/java/org/opensearch/geospatial/plugin/GeospatialPluginIT.java b/src/test/java/org/opensearch/geospatial/plugin/GeospatialPluginIT.java index ba91a43f..daf9b913 100644 --- a/src/test/java/org/opensearch/geospatial/plugin/GeospatialPluginIT.java +++ b/src/test/java/org/opensearch/geospatial/plugin/GeospatialPluginIT.java @@ -5,9 +5,7 @@ package org.opensearch.geospatial.plugin; -import java.io.IOException; - -import org.apache.http.util.EntityUtils; +import org.apache.hc.core5.http.io.entity.EntityUtils; import org.opensearch.client.Request; import org.opensearch.client.Response; import org.opensearch.geospatial.GeospatialRestTestCase; @@ -18,7 +16,7 @@ public class GeospatialPluginIT extends GeospatialRestTestCase { /** * Tests whether plugin is installed or not */ - public void testPluginInstalled() throws IOException { + public void testPluginInstalled() throws Exception { String restURI = String.join("/", "_cat", "plugins"); Request request = new Request("GET", restURI); diff --git a/src/test/java/org/opensearch/geospatial/processor/FeatureProcessorIT.java b/src/test/java/org/opensearch/geospatial/processor/FeatureProcessorIT.java index afe4a2e4..8e256aae 100644 --- a/src/test/java/org/opensearch/geospatial/processor/FeatureProcessorIT.java +++ b/src/test/java/org/opensearch/geospatial/processor/FeatureProcessorIT.java @@ -10,14 +10,13 @@ import static org.opensearch.geospatial.GeospatialObjectBuilder.randomGeoJSONFeature; import static org.opensearch.geospatial.GeospatialTestHelper.randomLowerCaseString; -import java.io.IOException; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; -import org.apache.http.util.EntityUtils; +import org.apache.hc.core5.http.io.entity.EntityUtils; import org.json.JSONObject; import org.opensearch.client.Request; import org.opensearch.client.Response; @@ -27,7 +26,7 @@ public class FeatureProcessorIT extends GeospatialRestTestCase { - public void testProcessorAvailable() throws IOException { + public void testProcessorAvailable() throws Exception { String nodeIngestURL = String.join("/", "_nodes", "ingest"); String endpoint = nodeIngestURL + "?filter_path=nodes.*.ingest.processors&pretty"; Request request = new Request("GET", endpoint); @@ -39,7 +38,7 @@ public void testProcessorAvailable() throws IOException { assertTrue(responseBody.contains(FeatureProcessor.TYPE)); } - public void testIndexGeoJSONSuccess() throws IOException { + public void testIndexGeoJSONSuccess() throws Exception { String indexName = randomLowerCaseString(); String geoShapeField = randomLowerCaseString(); diff --git a/src/test/java/org/opensearch/geospatial/rest/action/upload/geojson/RestUploadGeoJSONActionIT.java b/src/test/java/org/opensearch/geospatial/rest/action/upload/geojson/RestUploadGeoJSONActionIT.java index 4c42cbb7..f677001a 100644 --- a/src/test/java/org/opensearch/geospatial/rest/action/upload/geojson/RestUploadGeoJSONActionIT.java +++ b/src/test/java/org/opensearch/geospatial/rest/action/upload/geojson/RestUploadGeoJSONActionIT.java @@ -29,7 +29,7 @@ public class RestUploadGeoJSONActionIT extends GeospatialRestTestCase { public static final int NUMBER_OF_FEATURES_TO_ADD = 3; - public void testGeoJSONUploadSuccessPostMethod() throws IOException { + public void testGeoJSONUploadSuccessPostMethod() throws Exception { final String index = randomLowerCaseStringWithSuffix(ACCEPTED_INDEX_SUFFIX_PATH); assertIndexNotExists(index); @@ -55,7 +55,7 @@ public void testGeoJSONUploadFailIndexExists() throws IOException { assertTrue("Not an expected exception", responseException.getMessage().contains("resource_already_exists_exception")); } - public void testGeoJSONUploadSuccessPutMethod() throws IOException { + public void testGeoJSONUploadSuccessPutMethod() throws Exception { String index = randomLowerCaseStringWithSuffix(ACCEPTED_INDEX_SUFFIX_PATH); Response response = uploadGeoJSONFeaturesIntoExistingIndex(NUMBER_OF_FEATURES_TO_ADD, index, null); @@ -64,7 +64,7 @@ public void testGeoJSONUploadSuccessPutMethod() throws IOException { assertEquals("failed to index documents", NUMBER_OF_FEATURES_TO_ADD, getIndexDocumentCount(index)); } - public void testGeoJSONPutMethodUploadIndexExists() throws IOException { + public void testGeoJSONPutMethodUploadIndexExists() throws Exception { String index = randomLowerCaseStringWithSuffix(ACCEPTED_INDEX_SUFFIX_PATH); String geoFieldName = randomLowerCaseString(); diff --git a/src/test/java/org/opensearch/geospatial/stats/upload/RestUploadStatsActionIT.java b/src/test/java/org/opensearch/geospatial/stats/upload/RestUploadStatsActionIT.java index e702d232..303e7ccf 100644 --- a/src/test/java/org/opensearch/geospatial/stats/upload/RestUploadStatsActionIT.java +++ b/src/test/java/org/opensearch/geospatial/stats/upload/RestUploadStatsActionIT.java @@ -9,9 +9,7 @@ import static org.opensearch.geospatial.stats.upload.RestUploadStatsAction.ACTION_OBJECT; import static org.opensearch.geospatial.stats.upload.RestUploadStatsAction.ACTION_STATS; -import java.io.IOException; - -import org.apache.http.util.EntityUtils; +import org.apache.hc.core5.http.io.entity.EntityUtils; import org.opensearch.client.Request; import org.opensearch.client.Response; import org.opensearch.geospatial.GeospatialRestTestCase; @@ -25,20 +23,20 @@ private String getUploadStatsPath() { return String.join(URL_DELIMITER, getPluginURLPrefix(), ACTION_OBJECT, ACTION_STATS); } - private String getStatsResponseAsString() throws IOException { + private String getStatsResponseAsString() throws Exception { Request statsRequest = new Request("GET", getUploadStatsPath()); Response statsResponse = client().performRequest(statsRequest); return EntityUtils.toString(statsResponse.getEntity()); } - public void testStatsAPISuccess() throws IOException { + public void testStatsAPISuccess() throws Exception { Request request = new Request("GET", getUploadStatsPath()); Response response = client().performRequest(request); assertEquals("Failed to retrieve stats", RestStatus.OK, RestStatus.fromCode(response.getStatusLine().getStatusCode())); } - public void testStatsAreUpdatedAfterUpload() throws IOException { + public void testStatsAreUpdatedAfterUpload() throws Exception { // get current stats response final String currentUploadStats = getStatsResponseAsString(); assertNotNull(currentUploadStats);