diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/EncryptionS3ClientFactory.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/EncryptionS3ClientFactory.java index 7bb07933cbd22..e1744e70222d4 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/EncryptionS3ClientFactory.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/EncryptionS3ClientFactory.java @@ -77,11 +77,11 @@ private static synchronized boolean isEncryptionClientAvailable() { } /** - * Create encrypted s3 client + * Create encrypted s3 client. * @param uri S3A file system URI * @param parameters parameter object * @return encrypted s3 client - * @throws IOException + * @throws IOException IO failures */ @Override public S3Client createS3Client(URI uri, S3ClientCreationParameters parameters) @@ -98,11 +98,11 @@ public S3Client createS3Client(URI uri, S3ClientCreationParameters parameters) } /** - * Create async encrypted s3 client + * Create async encrypted s3 client. * @param uri S3A file system URI * @param parameters parameter object * @return async encrypted s3 client - * @throws IOException + * @throws IOException IO failures */ @Override public S3AsyncClient createS3AsyncClient(URI uri, S3ClientCreationParameters parameters) @@ -169,7 +169,7 @@ private S3AsyncClient createS3AsyncEncryptionClient(final CSEMaterials cseMateri } /** - * Get the custom Keyring class + * Get the custom Keyring class. * @param className * @param conf * @return diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Listing.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Listing.java index 24cb9150b1adf..2e17a57c21465 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Listing.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Listing.java @@ -63,7 +63,7 @@ import static org.apache.hadoop.fs.s3a.S3AUtils.objectRepresentsDirectory; import static org.apache.hadoop.fs.s3a.S3AUtils.stringify; import static org.apache.hadoop.fs.s3a.auth.RoleModel.pathToKey; -import static org.apache.hadoop.fs.s3a.impl.CSEUtils.isCSEIntructionFile; +import static org.apache.hadoop.fs.s3a.impl.CSEUtils.isCSEInstructionFile; import static org.apache.hadoop.fs.statistics.StoreStatisticNames.OBJECT_CONTINUE_LIST_REQUEST; import static org.apache.hadoop.fs.statistics.StoreStatisticNames.OBJECT_LIST_REQUEST; import static org.apache.hadoop.fs.statistics.impl.IOStatisticsBinding.iostatisticsStore; @@ -82,8 +82,8 @@ public class Listing extends AbstractStoreOperation { private static final Logger LOG = S3AFileSystem.LOG; private final boolean isCSEEnabled; - private S3Client s3Client; - private boolean skipCSEInstructionFile; + private final S3Client s3Client; + private final boolean skipCSEInstructionFile; static final FileStatusAcceptor ACCEPT_ALL_BUT_S3N = new AcceptAllButS3nDirs(); @@ -761,7 +761,7 @@ static class AcceptFilesOnly implements FileStatusAcceptor { public boolean accept(Path keyPath, S3Object s3Object) { return !keyPath.equals(qualifiedPath) && !s3Object.key().endsWith(S3N_FOLDER_SUFFIX) - && isCSEIntructionFile(skipCSEInstructionFile, s3Object.key()) + && isCSEInstructionFile(skipCSEInstructionFile, s3Object.key()) && !objectRepresentsDirectory(s3Object.key()); } @@ -799,17 +799,17 @@ static class AcceptAllButS3nDirs implements FileStatusAcceptor { public boolean accept(Path keyPath, S3Object s3Object) { return !s3Object.key().endsWith(S3N_FOLDER_SUFFIX) && - isCSEIntructionFile(skipCSEInstructionFile, s3Object.key()); + isCSEInstructionFile(skipCSEInstructionFile, s3Object.key()); } public boolean accept(Path keyPath, String prefix) { return !keyPath.toString().endsWith(S3N_FOLDER_SUFFIX) && - isCSEIntructionFile(skipCSEInstructionFile, keyPath.toString()); + isCSEInstructionFile(skipCSEInstructionFile, keyPath.toString()); } public boolean accept(FileStatus status) { return !status.getPath().toString().endsWith(S3N_FOLDER_SUFFIX) - && isCSEIntructionFile(skipCSEInstructionFile, status.getPath().toString()); + && isCSEInstructionFile(skipCSEInstructionFile, status.getPath().toString()); } } @@ -827,6 +827,7 @@ public static class AcceptAllButSelfAndS3nDirs implements FileStatusAcceptor { /** * Constructor. * @param qualifiedPath an already-qualified path. + * @param skipCSEInstructionFile whether to skip instruction files when cse is enabled */ public AcceptAllButSelfAndS3nDirs(Path qualifiedPath, boolean skipCSEInstructionFile) { this.qualifiedPath = qualifiedPath; @@ -845,7 +846,7 @@ public AcceptAllButSelfAndS3nDirs(Path qualifiedPath, boolean skipCSEInstruction public boolean accept(Path keyPath, S3Object s3Object) { return !keyPath.equals(qualifiedPath) && !s3Object.key().endsWith(S3N_FOLDER_SUFFIX) && - isCSEIntructionFile(skipCSEInstructionFile, s3Object.key()); + isCSEInstructionFile(skipCSEInstructionFile, s3Object.key()); } /** @@ -858,13 +859,13 @@ public boolean accept(Path keyPath, S3Object s3Object) { @Override public boolean accept(Path keyPath, String prefix) { return !keyPath.equals(qualifiedPath) && - isCSEIntructionFile(skipCSEInstructionFile, keyPath.toString()); + isCSEInstructionFile(skipCSEInstructionFile, keyPath.toString()); } @Override public boolean accept(FileStatus status) { return (status != null) && !status.getPath().equals(qualifiedPath) - && isCSEIntructionFile(skipCSEInstructionFile, status.getPath().toString()); + && isCSEInstructionFile(skipCSEInstructionFile, status.getPath().toString()); } } diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java index 6dff67a068b62..4257b6f80757f 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java @@ -465,7 +465,7 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities, private boolean isCSEEnabled; /** - * Skip Instruction File when CSE is enabled + * Skip Instruction File when CSE is enabled. */ private boolean skipCSEInstructionFile; @@ -475,7 +475,7 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities, private boolean cseRangedGetEnabled; /** - * Is reading of unencrypted objects enabled when CSE is enabled + * Is reading of unencrypted objects enabled when CSE is enabled. */ private boolean cseReadUnencryptedObjects; @@ -770,7 +770,8 @@ public void initialize(URI name, Configuration originalConf) BULK_DELETE_PAGE_SIZE_DEFAULT, 0); checkArgument(pageSize <= InternalConstants.MAX_ENTRIES_TO_DELETE, "page size out of range: %s", pageSize); - listing = new Listing(listingOperationCallbacks, createStoreContext(), s3Client, skipCSEInstructionFile); + listing = new Listing(listingOperationCallbacks, createStoreContext(), + s3Client, skipCSEInstructionFile); // now the open file logic openFileHelper = new OpenFileSupport( changeDetectionPolicy, diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java index 9936e847aacee..cb5a16f1c4f57 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java @@ -529,7 +529,11 @@ public static String stringify(AwsServiceException e) { * @param owner owner of the file * @param eTag S3 object eTag or null if unavailable * @param versionId S3 object versionId or null if unavailable + * @param s3Client s3 client object * @param isCSEEnabled is client side encryption enabled? + * @param bucket s3 bucket name + * @param cseRangedGetEnabled is ranged get enabled + * @param cseReadUnencryptedObjects is read unencrypted object enabled * @return a status entry */ public static S3AFileStatus createFileStatus(Path keyPath, diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/impl/CommitOperations.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/impl/CommitOperations.java index cce6c477a0d82..634b764028b35 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/impl/CommitOperations.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/impl/CommitOperations.java @@ -591,7 +591,8 @@ public SinglePendingCommit uploadFileToPendingCommit(File localFile, } // Read from the file input stream at current position. RequestBody body = RequestBody.fromInputStream(fileStream, size); - UploadPartResponse response = writeOperations.uploadPart(partBuilder.build(), body, statistics); + UploadPartResponse response = writeOperations.uploadPart(partBuilder.build(), + body, statistics); offset += uploadPartSize; parts.add(CompletedPart.builder() .partNumber(partNumber) diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/CSEUtils.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/CSEUtils.java index d685c2a37a6c7..bb5a246dd2a7e 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/CSEUtils.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/CSEUtils.java @@ -49,15 +49,17 @@ private CSEUtils() { } /** - * Checks if the file suffix ends with {@link S3_ENCRYPTION_CSE_SKIP_INSTRUCTION_FILE} - * when the config {@link S3_ENCRYPTION_CSE_SKIP_INSTRUCTION_FILE_DEFAULT} is enabled and - * CSE is used. + * Checks if the file suffix ends with + * {@link org.apache.hadoop.fs.s3a.Constants#S3_ENCRYPTION_CSE_INSTRUCTION_FILE_SUFFIX} + * when the config + * {@link org.apache.hadoop.fs.s3a.Constants#S3_ENCRYPTION_CSE_SKIP_INSTRUCTION_FILE_DEFAULT} + * is enabled and CSE is used. * @param skipCSEInstructionFile whether to skip checking for the filename suffix * @param key file name - * @return true if file doesn't end with {@link S3_ENCRYPTION_CSE_SKIP_INSTRUCTION_FILE} or - * if CSE or {@link S3_ENCRYPTION_CSE_SKIP_INSTRUCTION_FILE_DEFAULT} is false. + * @return true if cse is disabled or if skipping of instruction file is disabled or file name + * does not end with defined suffix */ - public static boolean isCSEIntructionFile(boolean skipCSEInstructionFile, String key) { + public static boolean isCSEInstructionFile(boolean skipCSEInstructionFile, String key) { if (!skipCSEInstructionFile) { return true; } @@ -108,8 +110,9 @@ public static boolean isObjectEncrypted(S3Client s3Client, String bucket, String } /** - * Get the unencrypted object length by either subtracting {@link CSE_PADDING_LENGTH} - * from the object size or calculating the actual size by doing S3 ranged GET operation. + * Get the unencrypted object length by either subtracting + * {@link InternalConstants#CSE_PADDING_LENGTH} from the object size or calculating the + * actual size by doing S3 ranged GET operation. * * @param s3Client S3 client * @param bucket bucket name of the s3 object @@ -119,7 +122,7 @@ public static boolean isObjectEncrypted(S3Client s3Client, String bucket, String * @param cseRangedGetEnabled is ranged get enabled * @param cseReadUnencryptedObjects is reading of une * @return unencrypted length of the object - * @throws IOException + * @throws IOException IO failures */ public static long getUnencryptedObjectLength(S3Client s3Client, String bucket, diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/encryption.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/encryption.md index 82ffe8d931965..d8c86c644acc4 100644 --- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/encryption.md +++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/encryption.md @@ -743,7 +743,6 @@ S3-CSE to work. Example for custom keyring implementation ``` public class CustomKeyring implements Keyring { - public CustomKeyring() { } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/CustomKeyring.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/CustomKeyring.java index 090c545fe884b..4d0955c33af27 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/CustomKeyring.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/CustomKeyring.java @@ -48,15 +48,12 @@ public class CustomKeyring implements Keyring { public CustomKeyring(Configuration conf) throws IOException { this.conf = conf; kmsClient = KmsClient.builder().region(Region.of(conf.get(AWS_REGION, AWS_S3_DEFAULT_REGION))) - .credentialsProvider( - new TemporaryAWSCredentialsProvider - (new Path(conf.get("test.fs.s3a.name")).toUri(), conf)) + .credentialsProvider(new TemporaryAWSCredentialsProvider( + new Path(conf.get("test.fs.s3a.name")).toUri(), conf)) .build(); - kmsKeyring = KmsKeyring.builder() .kmsClient(kmsClient) .wrappingKeyId(conf.get(Constants.S3_ENCRYPTION_KEY)).build(); - } @Override diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java index 469b3569d61a8..e6909872b30dd 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java @@ -639,9 +639,8 @@ private static void skipIfCrossRegionClient( } } - private void unsetEncryption(Configuration conf) { + private static void unsetEncryption(Configuration conf) { removeBaseAndBucketOverrides(conf, S3_ENCRYPTION_ALGORITHM); - conf.set(Constants.S3_ENCRYPTION_ALGORITHM, - S3AEncryptionMethods.NONE.getMethod()); + conf.set(Constants.S3_ENCRYPTION_ALGORITHM, S3AEncryptionMethods.NONE.getMethod()); } }