HADOOP-18382. SDK upgrade prerequisites

hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSCredentialProviderList.java

@@ -58,9 +58,13 @@
  *   rethrown; exceptions other than 'no credentials' have priority.</li>
  *   <li>Special handling of {@link AnonymousAWSCredentials}.</li>
  * </ol>
+ *
+ * @deprecated This class will be replaced by one that implements AWS SDK V2's AwsCredentialProvider
+ * as part of upgrading S3A to SDK V2. See HADOOP-18073.
  */
 @InterfaceAudience.Private
 @InterfaceStability.Evolving
+@Deprecated
 public class AWSCredentialProviderList implements AWSCredentialsProvider,
     AutoCloseable {
 

hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AnonymousAWSCredentialsProvider.java

@@ -34,9 +34,13 @@
  * Please note that users may reference this class name from configuration
  * property fs.s3a.aws.credentials.provider.  Therefore, changing the class name
  * would be a backward-incompatible change.
+ *
+ * @deprecated This class will be replaced by one that implements AWS SDK V2's AwsCredentialProvider
+ * as part of upgrading S3A to SDK V2. See HADOOP-18073.
  */
 @InterfaceAudience.Private
 @InterfaceStability.Stable
+@Deprecated
 public class AnonymousAWSCredentialsProvider implements AWSCredentialsProvider {
 
   public static final String NAME

hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java

@@ -140,6 +140,7 @@ private Constants() {
   public static final String ASSUMED_ROLE_POLICY =
       "fs.s3a.assumed.role.policy";
 
+  @SuppressWarnings("deprecation")
   public static final String ASSUMED_ROLE_CREDENTIALS_DEFAULT =
       SimpleAWSCredentialsProvider.NAME;
 
@@ -732,6 +733,7 @@ private Constants() {
 
   @InterfaceAudience.Private
   @InterfaceStability.Unstable
+  @SuppressWarnings("deprecation")
   public static final Class<? extends S3ClientFactory>
       DEFAULT_S3_CLIENT_FACTORY_IMPL =
           DefaultS3ClientFactory.class;
@@ -1203,4 +1205,9 @@ private Constants() {
    * Default maximum read size in bytes during vectored reads : {@value}.
    */
   public static final int DEFAULT_AWS_S3_VECTOR_READS_MAX_MERGED_READ_SIZE = 1253376; //1M
+
+  /**
+   * Prefix of auth classes in AWS SDK V1.
+   */
+  public static final String AWS_AUTH_CLASS_PREFIX = "com.amazonaws.auth";
 }

hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/DefaultS3ClientFactory.java

@@ -71,6 +71,7 @@
  */
 @InterfaceAudience.Private
 @InterfaceStability.Unstable
+@SuppressWarnings("deprecation")
 public class DefaultS3ClientFactory extends Configured
     implements S3ClientFactory {
 

hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java

@@ -133,6 +133,7 @@
 import org.apache.hadoop.fs.statistics.IOStatisticsSource;
 import org.apache.hadoop.fs.statistics.IOStatisticsContext;
 import org.apache.hadoop.fs.statistics.impl.IOStatisticsStore;
+import org.apache.hadoop.fs.store.LogExactlyOnce;
 import org.apache.hadoop.fs.store.audit.AuditEntryPoint;
 import org.apache.hadoop.fs.store.audit.ActiveThreadSpanSource;
 import org.apache.hadoop.fs.store.audit.AuditSpan;
@@ -294,6 +295,10 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
   public static final Logger LOG = LoggerFactory.getLogger(S3AFileSystem.class);
   private static final Logger PROGRESS =
       LoggerFactory.getLogger("org.apache.hadoop.fs.s3a.S3AFileSystem.Progress");
+  private static final LogExactlyOnce WARN_ON_DELEGATION_TOKENS =
+      new LogExactlyOnce(SDK_V2_UPGRADE_LOG);
+  private static final LogExactlyOnce WARN_ON_GET_S3_CLIENT =
+      new LogExactlyOnce(SDK_V2_UPGRADE_LOG);
   private LocalDirAllocator directoryAllocator;
   private CannedAccessControlList cannedACL;
 
@@ -335,6 +340,7 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
   private boolean useListV1;
   private MagicCommitIntegration committerIntegration;
 
+  @SuppressWarnings("deprecation")
   private AWSCredentialProviderList credentials;
   private SignerManager signerManager;
 
@@ -837,6 +843,7 @@ public Listing getListing() {
    * @param dtEnabled are delegation tokens enabled?
    * @throws IOException failure.
    */
+  @SuppressWarnings("deprecation")
   private void bindAWSClient(URI name, boolean dtEnabled) throws IOException {
     Configuration conf = getConf();
     credentials = null;
@@ -849,6 +856,10 @@ private void bindAWSClient(URI name, boolean dtEnabled) throws IOException {
       // with it if so.
 
       LOG.debug("Using delegation tokens");
+      WARN_ON_DELEGATION_TOKENS.warn(
+          "The credential provider interface has changed in AWS SDK V2, custom credential "
+              + "providers used in delegation tokens binding classes will need to be updated once "
+              + "S3A is upgraded to SDK V2");
       S3ADelegationTokens tokens = new S3ADelegationTokens();
       this.delegationTokens = Optional.of(tokens);
       tokens.bindToFileSystem(getCanonicalUri(),
@@ -1186,6 +1197,8 @@ AmazonS3 getAmazonS3Client() {
   @VisibleForTesting
   public AmazonS3 getAmazonS3ClientForTesting(String reason) {
     LOG.warn("Access to S3A client requested, reason {}", reason);
+    WARN_ON_GET_S3_CLIENT.warn(
+        "getAmazonS3ClientForTesting() will be removed as part of upgrading S3A to AWS SDK V2");
     return s3;
   }
 
@@ -4947,6 +4960,7 @@ public boolean hasCapability(String capability) {
    * @param purpose what is this for? This is initially for logging
    * @return a reference to shared credentials.
    */
+  @SuppressWarnings("deprecation")
   public AWSCredentialProviderList shareCredentials(final String purpose) {
     LOG.debug("Sharing credentials for: {}", purpose);
     return credentials.share();

hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java

@@ -31,6 +31,7 @@
 import com.amazonaws.services.s3.model.MultiObjectDeleteException;
 import com.amazonaws.services.s3.model.S3ObjectSummary;
 import org.apache.hadoop.classification.VisibleForTesting;
+import org.apache.hadoop.fs.store.LogExactlyOnce;
 import org.apache.hadoop.util.Preconditions;
 
 import org.apache.commons.lang3.StringUtils;
@@ -44,7 +45,6 @@
 import org.apache.hadoop.fs.RemoteIterator;
 import org.apache.hadoop.util.functional.RemoteIterators;
 import org.apache.hadoop.fs.s3a.auth.delegation.EncryptionSecrets;
-import org.apache.hadoop.fs.s3a.auth.IAMInstanceCredentialsProvider;
 import org.apache.hadoop.fs.s3a.impl.NetworkBinding;
 import org.apache.hadoop.fs.s3native.S3xLoginHelper;
 import org.apache.hadoop.net.ConnectTimeoutException;
@@ -86,6 +86,7 @@
 import static org.apache.hadoop.fs.s3a.Constants.*;
 import static org.apache.hadoop.fs.s3a.impl.ErrorTranslation.isUnknownBucket;
 import static org.apache.hadoop.fs.s3a.impl.InternalConstants.CSE_PADDING_LENGTH;
+import static org.apache.hadoop.fs.s3a.impl.InternalConstants.SDK_V2_UPGRADE_LOG_NAME;
 import static org.apache.hadoop.fs.s3a.impl.MultiObjectDeleteSupport.translateDeleteException;
 import static org.apache.hadoop.io.IOUtils.cleanupWithLogger;
 import static org.apache.hadoop.util.functional.RemoteIterators.filteringRemoteIterator;
@@ -141,6 +142,11 @@ public final class S3AUtils {
 
   private static final String BUCKET_PATTERN = FS_S3A_BUCKET_PREFIX + "%s.%s";
 
+  public static final Logger SDK_V2_UPGRADE_LOG = LoggerFactory.getLogger(SDK_V2_UPGRADE_LOG_NAME);
+
+  private static final LogExactlyOnce WARN_OF_DIRECTLY_REFERENCED_CREDENTIAL_PROVIDER =
+      new LogExactlyOnce(SDK_V2_UPGRADE_LOG);
+
   /**
    * Error message when the AWS provider list built up contains a forbidden
    * entry.
@@ -551,13 +557,14 @@ public static long dateToLong(final Date date) {
   /**
    * The standard AWS provider list for AWS connections.
    */
+  @SuppressWarnings("deprecation")
   public static final List<Class<?>>
       STANDARD_AWS_PROVIDERS = Collections.unmodifiableList(
       Arrays.asList(
           TemporaryAWSCredentialsProvider.class,
           SimpleAWSCredentialsProvider.class,
           EnvironmentVariableCredentialsProvider.class,
-          IAMInstanceCredentialsProvider.class));
+          org.apache.hadoop.fs.s3a.auth.IAMInstanceCredentialsProvider.class));
 
   /**
    * Create the AWS credentials from the providers, the URI and
@@ -568,6 +575,7 @@ public static long dateToLong(final Date date) {
    * @throws IOException Problems loading the providers (including reading
    * secrets from credential files).
    */
+  @SuppressWarnings("deprecation")
   public static AWSCredentialProviderList createAWSCredentialProviderSet(
       @Nullable URI binding,
       Configuration conf) throws IOException {
@@ -615,6 +623,7 @@ public static List<Class<?>> loadAWSProviderClasses(Configuration conf,
    * @return the list of classes, possibly empty
    * @throws IOException on a failure to load the list.
    */
+  @SuppressWarnings("deprecation")
   public static AWSCredentialProviderList buildAWSProviderList(
       @Nullable final URI binding,
       final Configuration conf,
@@ -637,6 +646,12 @@ public static AWSCredentialProviderList buildAWSProviderList(
     AWSCredentialProviderList providers = new AWSCredentialProviderList();
     for (Class<?> aClass : awsClasses) {
 
+      if (aClass.getName().contains(AWS_AUTH_CLASS_PREFIX)) {
+        WARN_OF_DIRECTLY_REFERENCED_CREDENTIAL_PROVIDER.warn(
+            "Directly referencing AWS SDK V1 credential provider {}. AWS SDK V1 credential "
+                + "providers will be removed once S3A is upgraded to SDK V2", aClass.getName());
+      }
+
       if (forbidden.contains(aClass)) {
         throw new IOException(E_FORBIDDEN_AWS_PROVIDER
             + " in option " + key + ": " + aClass);

hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ClientFactory.java

@@ -44,9 +44,12 @@
  * implementing only the deprecated method will work.
  * See https://github.com/apache/hbase-filesystem
  *
+ * @deprecated This interface will be replaced by one which uses the AWS SDK V2 S3 client as part of
+ * upgrading S3A to SDK V2. See HADOOP-18073.
  */
 @InterfaceAudience.LimitedPrivate("HBoss")
 @InterfaceStability.Evolving
+@Deprecated
 public interface S3ClientFactory {
 
   /**

hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/SharedInstanceCredentialProvider.java

@@ -20,13 +20,12 @@
 
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.fs.s3a.auth.IAMInstanceCredentialsProvider;
 import org.apache.hadoop.fs.s3a.auth.NoAwsCredentialsException;
 
 /**
  * This credential provider has jittered between existing and non-existing,
  * but it turns up in documentation enough that it has been restored.
- * It extends {@link IAMInstanceCredentialsProvider} to pick up its
+ * It extends {@link org.apache.hadoop.fs.s3a.auth.IAMInstanceCredentialsProvider} to pick up its
  * bindings, which are currently to use the
  * {@code EC2ContainerCredentialsProviderWrapper} class for IAM and container
  * authentication.
@@ -39,6 +38,7 @@
  */
 @InterfaceAudience.Public
 @InterfaceStability.Evolving
+@SuppressWarnings("deprecation")
 public final class SharedInstanceCredentialProvider extends
-    IAMInstanceCredentialsProvider {
+    org.apache.hadoop.fs.s3a.auth.IAMInstanceCredentialsProvider {
 }

hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/SimpleAWSCredentialsProvider.java

@@ -41,9 +41,13 @@
  * Please note that users may reference this class name from configuration
  * property fs.s3a.aws.credentials.provider.  Therefore, changing the class name
  * would be a backward-incompatible change.
+ *
+ * @deprecated This class will be replaced by one that implements AWS SDK V2's AwsCredentialProvider
+ * as part of upgrading S3A to SDK V2. See HADOOP-18073.
  */
 @InterfaceAudience.Public
 @InterfaceStability.Stable
+@Deprecated
 public class SimpleAWSCredentialsProvider implements AWSCredentialsProvider {
 
   public static final String NAME

hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/TemporaryAWSCredentialsProvider.java

@@ -28,7 +28,6 @@
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.s3a.auth.AbstractSessionCredentialsProvider;
 import org.apache.hadoop.fs.s3a.auth.MarshalledCredentialBinding;
 import org.apache.hadoop.fs.s3a.auth.MarshalledCredentials;
 import org.apache.hadoop.fs.s3a.auth.NoAuthWithAWSException;
@@ -43,11 +42,15 @@
  *
  * This credential provider must not fail in creation because that will
  * break a chain of credential providers.
+ *
+ * @deprecated This class will be replaced by one that implements AWS SDK V2's AwsCredentialProvider
+ * as part of upgrading S3A to SDK V2. See HADOOP-18073.
  */
 @InterfaceAudience.Public
 @InterfaceStability.Stable
+@Deprecated
 public class TemporaryAWSCredentialsProvider extends
-    AbstractSessionCredentialsProvider {
+    org.apache.hadoop.fs.s3a.auth.AbstractSessionCredentialsProvider {
 
   public static final String NAME
       = "org.apache.hadoop.fs.s3a.TemporaryAWSCredentialsProvider";

hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/AbstractAWSCredentialProvider.java

@@ -28,7 +28,11 @@
 /**
  * Base class for AWS credential providers which
  * take a URI and config in their constructor.
+ *
+ * @deprecated This class will be replaced by one that implements AWS SDK V2's AwsCredentialProvider
+ * as part of upgrading S3A to SDK V2. See HADOOP-18073.
  */
+@Deprecated
 public abstract class AbstractAWSCredentialProvider
     implements AWSCredentialsProvider {
 

hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/AbstractSessionCredentialsProvider.java

@@ -35,8 +35,12 @@
 
 /**
  * Base class for session credential support.
+ *
+ * @deprecated This class will be replaced by one that implements AWS SDK V2's AwsCredentialProvider
+ * as part of upgrading S3A to SDK V2. See HADOOP-18073.
  */
 @InterfaceAudience.Private
+@Deprecated
 public abstract class AbstractSessionCredentialsProvider
     extends AbstractAWSCredentialProvider {
 

hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/AssumedRoleCredentialProvider.java

@@ -41,13 +41,11 @@
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.s3a.AWSCredentialProviderList;
 import org.apache.hadoop.fs.s3a.CredentialInitializationException;
 import org.apache.hadoop.fs.s3a.Retries;
 import org.apache.hadoop.fs.s3a.S3AUtils;
 import org.apache.hadoop.fs.s3a.Invoker;
 import org.apache.hadoop.fs.s3a.S3ARetryPolicy;
-import org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider;
 import org.apache.hadoop.security.UserGroupInformation;
 
 import static org.apache.hadoop.fs.s3a.Constants.*;
@@ -60,9 +58,13 @@
  * unless overridden, creating a session name from the current user.
  *
  * Classname is used in configuration files; do not move.
+ *
+ * @deprecated This class will be replaced by one that implements AWS SDK V2's AwsCredentialProvider
+ * as part of upgrading S3A to SDK V2. See HADOOP-18073.
  */
 @InterfaceAudience.Public
 @InterfaceStability.Evolving
+@Deprecated
 public class AssumedRoleCredentialProvider implements AWSCredentialsProvider,
     Closeable {
 
@@ -82,7 +84,7 @@ public class AssumedRoleCredentialProvider implements AWSCredentialsProvider,
 
   private final String arn;
 
-  private final AWSCredentialProviderList credentialsToSTS;
+  private final org.apache.hadoop.fs.s3a.AWSCredentialProviderList credentialsToSTS;
 
   private final Invoker invoker;
 
@@ -108,7 +110,7 @@ public AssumedRoleCredentialProvider(@Nullable URI fsUri, Configuration conf)
     credentialsToSTS = buildAWSProviderList(fsUri, conf,
         ASSUMED_ROLE_CREDENTIALS_PROVIDER,
         Arrays.asList(
-            SimpleAWSCredentialsProvider.class,
+            org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider.class,
             EnvironmentVariableCredentialsProvider.class),
         Sets.newHashSet(this.getClass()));
     LOG.debug("Credentials to obtain role credentials: {}", credentialsToSTS);

hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/IAMInstanceCredentialsProvider.java

@@ -40,9 +40,13 @@
  * as a non-recoverable failure.
  * <p>
  * It is implicitly public; marked evolving as we can change its semantics.
+ *
+ * @deprecated This class will be replaced by one that implements AWS SDK V2's AwsCredentialProvider
+ * as part of upgrading S3A to SDK V2. See HADOOP-18073.
  */
 @InterfaceAudience.Public
 @InterfaceStability.Evolving
+@Deprecated
 public class IAMInstanceCredentialsProvider
     implements AWSCredentialsProvider, Closeable {
 

hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/MarshalledCredentialProvider.java

@@ -40,6 +40,7 @@
  */
 @InterfaceAudience.Private
 @InterfaceStability.Unstable
+@SuppressWarnings("deprecation")
 public class MarshalledCredentialProvider extends
     AbstractSessionCredentialsProvider {