Upgrade AWS SDK v1 to v2 (#7372)

* AWS SDK v2. * Migrating com.amazonaws.sdk.ec2MetadataServiceEndpointOverride to aws.ec2MetadataServiceEndpoint per https://docs.aws.amazon.com/sdk-for-java/latest/developer-guide/migration-client-changes.html. * Fetch host address for EC2 metadata service from SdkSettings. * Deprecations of running with partial keys were removed in AWS SDK v2. * Underscore is not a valid host name, thus URI.create doesn't treat it as a host. * Added missing JARs, all tests pass. * Setup a default AWS profile under config, don't read ~/.aws. * Re-enable availability zone node attribute implementation. * Cleaning up privileged calls using SocketAccess * Consistently setup and teardown AWS settings in repository-s3. * Using SdkException instead of RuntimeException in S3BlobContainer * Removing AwsSessionCredentials object from S3BasicSesstionCredentials * Removing S3BasicCredentials and S3BasicSessionCredentials Signed-off-by: dblock <[email protected]> Co-authored-by: Raghuvansh Raj <[email protected]>
opensearch-project · Jun 2, 2023 · b15ffc9 · b15ffc9
1 parent 5d5e8ad
commit b15ffc9
Show file tree

Hide file tree

Showing 187 changed files with 10,379 additions and 2,373 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -41,8 +41,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Bump `com.netflix.nebula:nebula-publishing-plugin` from 19.2.0 to 20.3.0
 - Bump `com.diffplug.spotless` from 6.17.0 to 6.18.0
 - Bump `io.opencensus:opencensus-api` from 0.18.0 to 0.31.1 ([#7291](https://github.com/opensearch-project/OpenSearch/pull/7291))
+- Add `org.reactivestreams` 1.0.4 ([7372](https://github.com/opensearch-project/OpenSearch/pull/7372/))
 - Add  `com.github.luben:zstd-jni` version 1.5.5-3 ([#2996](https://github.com/opensearch-project/OpenSearch/pull/2996))
 - OpenJDK Update (April 2023 Patch releases) ([#7344](https://github.com/opensearch-project/OpenSearch/pull/7344)
+- Bump `com.amazonaws` 1.12.270 to `software.amazon.awssdk` 2.20.55 ([7372](https://github.com/opensearch-project/OpenSearch/pull/7372/))
 
 ### Changed
 - [CCR] Add getHistoryOperationsFromTranslog method to fetch the history snapshot from translogs ([#3948](https://github.com/opensearch-project/OpenSearch/pull/3948))
@@ -53,6 +55,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Improve summary error message for invalid setting updates ([#4792](https://github.com/opensearch-project/OpenSearch/pull/4792))
 - Reduce memory copy in zstd compression ([#7681](https://github.com/opensearch-project/OpenSearch/pull/7681))
 - Add ZSTD compression for snapshotting ([#2996](https://github.com/opensearch-project/OpenSearch/pull/2996))
+- Change `com.amazonaws.sdk.ec2MetadataServiceEndpointOverride` to `aws.ec2MetadataServiceEndpoint` ([7372](https://github.com/opensearch-project/OpenSearch/pull/7372/))
+- Change `com.amazonaws.sdk.stsEndpointOverride` to `aws.stsEndpointOverride` ([7372](https://github.com/opensearch-project/OpenSearch/pull/7372/))
 
 ### Deprecated
 

diff --git a/buildSrc/version.properties b/buildSrc/version.properties
@@ -42,7 +42,8 @@ commonslogging    = 1.2
 commonscodec      = 1.15
 
 # plugin dependencies
-aws               = 1.12.270
+aws               = 2.20.55
+reactivestreams   = 1.0.4
 
 # when updating this version, you need to ensure compatibility with:
 #  - plugins/ingest-attachment (transitive dependency, check the upstream POM)

diff --git a/plugins/discovery-ec2/build.gradle b/plugins/discovery-ec2/build.gradle
@@ -39,15 +39,32 @@ opensearchplugin {
 }
 
 dependencies {
-  api "com.amazonaws:aws-java-sdk-ec2:${versions.aws}"
-  api "com.amazonaws:aws-java-sdk-core:${versions.aws}"
+  api "software.amazon.awssdk:sdk-core:${versions.aws}"
+  api "software.amazon.awssdk:aws-core:${versions.aws}"
+  api "software.amazon.awssdk:utils:${versions.aws}"
+  api "software.amazon.awssdk:auth:${versions.aws}"
+  api "software.amazon.awssdk:ec2:${versions.aws}"
+  api "software.amazon.awssdk:http-client-spi:${versions.aws}"
+  api "software.amazon.awssdk:apache-client:${versions.aws}"
+  api "software.amazon.awssdk:regions:${versions.aws}"
+  api "software.amazon.awssdk:profiles:${versions.aws}"
+  api "software.amazon.awssdk:endpoints-spi:${versions.aws}"
+  api "software.amazon.awssdk:annotations:${versions.aws}"
+  api "software.amazon.awssdk:metrics-spi:${versions.aws}"
+  api "software.amazon.awssdk:json-utils:${versions.aws}"
+  api "software.amazon.awssdk:protocol-core:${versions.aws}"
+  api "software.amazon.awssdk:aws-query-protocol:${versions.aws}"
+  api "software.amazon.awssdk:aws-json-protocol:${versions.aws}"
+  api "software.amazon.awssdk:third-party-jackson-core:${versions.aws}"
   api "org.apache.httpcomponents:httpclient:${versions.httpclient}"
   api "org.apache.httpcomponents:httpcore:${versions.httpcore}"
   api "commons-logging:commons-logging:${versions.commonslogging}"
   api "org.apache.logging.log4j:log4j-1.2-api:${versions.log4j}"
+  api "org.slf4j:slf4j-api:${versions.slf4j}"
   api "commons-codec:commons-codec:${versions.commonscodec}"
   api "com.fasterxml.jackson.core:jackson-databind:${versions.jackson_databind}"
   api "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}"
+  api "org.reactivestreams:reactive-streams:${versions.reactivestreams}"
 }
 
 restResources {
@@ -57,7 +74,7 @@ restResources {
 }
 
 tasks.named("dependencyLicenses").configure {
-  mapping from: /aws-java-sdk-.*/, to: 'aws-java-sdk'
+  mapping from: /software.amazon.awssdk-.*/, to: 'software.amazon.awssdk'
   mapping from: /jackson-.*/, to: 'jackson'
 }
 
@@ -90,15 +107,17 @@ tasks.register("writeTestJavaPolicy") {
           "  permission org.bouncycastle.crypto.CryptoServicesPermission \"exportSecretKey\";",
           "  permission org.bouncycastle.crypto.CryptoServicesPermission \"exportPrivateKey\";",
           "  permission java.io.FilePermission \"\${javax.net.ssl.trustStore}\", \"read\";",
-          "  permission java.util.PropertyPermission \"com.amazonaws.sdk.ec2MetadataServiceEndpointOverride\", \"write\";",
+          "  permission java.util.PropertyPermission \"aws.ec2MetadataServiceEndpoint\", \"write\";",
+          "  permission java.io.FilePermission \"config\", \"read\";",
           "};"
         ].join("\n")
       )
     } else {
       javaPolicy.write(
         [
           "grant {",
-          "  permission java.util.PropertyPermission \"com.amazonaws.sdk.ec2MetadataServiceEndpointOverride\", \"write\";",
+          "  permission java.util.PropertyPermission \"aws.ec2MetadataServiceEndpoint\", \"write\";",
+          "  permission java.io.FilePermission \"config\", \"read\";",
           "};"
         ].join("\n"))
     }
@@ -110,7 +129,7 @@ tasks.named("test").configure {
   // this is needed for insecure plugins, remove if possible!
   systemProperty 'tests.artifact', project.name
 
-  // Setting a custom policy to manipulate com.amazonaws.sdk.ec2MetadataServiceEndpointOverride system property
+  // Setting a custom policy to manipulate aws.ec2MetadataServiceEndpoint system property
   // it is better rather disable security manager at all with `systemProperty 'tests.security.manager', 'false'`
   if (BuildParams.inFipsJvm){
     // Using the key==value format to override default JVM security settings and policy
@@ -128,30 +147,17 @@ tasks.named("check").configure {
 
 tasks.named("thirdPartyAudit").configure {
   ignoreMissingClasses(
-          // classes are missing
-          'javax.jms.Message',
-          'com.amazonaws.jmespath.JmesPathExpression',
-          'com.amazonaws.jmespath.ObjectMapperSingleton',
-          'software.amazon.ion.IonReader',
-          'software.amazon.ion.IonSystem',
-          'software.amazon.ion.IonType',
-          'software.amazon.ion.IonWriter',
-          'software.amazon.ion.Timestamp',
-          'software.amazon.ion.system.IonBinaryWriterBuilder',
-          'software.amazon.ion.system.IonSystemBuilder',
-          'software.amazon.ion.system.IonTextWriterBuilder',
-          'software.amazon.ion.system.IonWriterBuilder',
-          'javax.servlet.ServletContextEvent',
-          'javax.servlet.ServletContextListener',
-          'org.apache.avalon.framework.logger.Logger',
-          'org.apache.log.Hierarchy',
-          'org.apache.log.Logger'
+    'javax.jms.Message',
+    'javax.servlet.ServletContextEvent',
+    'javax.servlet.ServletContextListener',
+    'org.apache.avalon.framework.logger.Logger',
+    'org.apache.log.Hierarchy',
+    'org.apache.log.Logger',
+    'org.slf4j.impl.StaticLoggerBinder',
+    'org.slf4j.impl.StaticMDCBinder',
+    'org.slf4j.impl.StaticMarkerBinder',
+    'software.amazon.eventstream.HeaderValue',
+    'software.amazon.eventstream.Message',
+    'software.amazon.eventstream.MessageDecoder'
   )
-
-  if (BuildParams.runtimeJavaVersion > JavaVersion.VERSION_1_8) {
-    ignoreMissingClasses(
-            'javax.xml.bind.DatatypeConverter',
-            'javax.xml.bind.JAXBContext'
-    )
-  }
 }
diff --git a/plugins/discovery-ec2/config/discovery-ec2/log4j2.properties b/plugins/discovery-ec2/config/discovery-ec2/log4j2.properties
@@ -9,17 +9,17 @@
 # GitHub history for details.
 #
 
-logger.com_amazonaws.name = com.amazonaws
+logger.com_amazonaws.name = software.amazon.awssdk
 logger.com_amazonaws.level = warn
 
-logger.com_amazonaws_jmx_SdkMBeanRegistrySupport.name = com.amazonaws.jmx.SdkMBeanRegistrySupport
+logger.com_amazonaws_jmx_SdkMBeanRegistrySupport.name = software.amazon.awssdk.jmx.SdkMBeanRegistrySupport
 logger.com_amazonaws_jmx_SdkMBeanRegistrySupport.level = error
 
-logger.com_amazonaws_metrics_AwsSdkMetrics.name = com.amazonaws.metrics.AwsSdkMetrics
+logger.com_amazonaws_metrics_AwsSdkMetrics.name = software.amazon.awssdk.metrics.AwsSdkMetrics
 logger.com_amazonaws_metrics_AwsSdkMetrics.level = error
 
-logger.com_amazonaws_auth_profile_internal_BasicProfileConfigFileLoader.name = com.amazonaws.auth.profile.internal.BasicProfileConfigFileLoader
+logger.com_amazonaws_auth_profile_internal_BasicProfileConfigFileLoader.name = software.amazon.awssdk.auth.profile.internal.BasicProfileConfigFileLoader
 logger.com_amazonaws_auth_profile_internal_BasicProfileConfigFileLoader.level = error
 
-logger.com_amazonaws_services_s3_internal_UseArnRegionResolver.name = com.amazonaws.services.s3.internal.UseArnRegionResolver
+logger.com_amazonaws_services_s3_internal_UseArnRegionResolver.name = software.amazon.awssdk.services.s3.internal.UseArnRegionResolver
 logger.com_amazonaws_services_s3_internal_UseArnRegionResolver.level = error
diff --git a/plugins/discovery-ec2/licenses/annotations-2.20.55.jar.sha1 b/plugins/discovery-ec2/licenses/annotations-2.20.55.jar.sha1
@@ -0,0 +1 @@
+330e9d0e5f2401fffba5afe30f3740f400e8308d