HADOOP-18757. S3A Committer only finalizes the commits in a single th…

…read (#5706) Contributed by Moditha Hewasinghage
apache · Jul 19, 2023 · b6b2590 · b6b2590
1 parent 8df1616
commit b6b2590
Show file tree

Hide file tree

Showing 2 changed files with 3 additions and 1 deletion.
diff --git a/...op-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/impl/CommitContext.java b/...op-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/impl/CommitContext.java
@@ -236,7 +236,7 @@ private ExecutorService buildThreadPool(
         .setDaemon(true)
         .setNameFormat(THREAD_PREFIX + jobId + "-%d")
         .build();
-    return new HadoopThreadPoolExecutor(0, numThreads,
+    return new HadoopThreadPoolExecutor(numThreads, numThreads,
         THREAD_KEEP_ALIVE_TIME,
         TimeUnit.SECONDS,
         new LinkedBlockingQueue<>(),

diff --git a/...rg/apache/hadoop/fs/s3a/commit/staging/integration/ITestStagingCommitProtocolFailure.java b/...rg/apache/hadoop/fs/s3a/commit/staging/integration/ITestStagingCommitProtocolFailure.java
@@ -32,6 +32,7 @@
 import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl;
 
 import static org.apache.hadoop.fs.s3a.Constants.MULTIPART_UPLOADS_ENABLED;
+import static org.apache.hadoop.fs.s3a.S3ATestUtils.disableFilesystemCaching;
 import static org.apache.hadoop.fs.s3a.S3ATestUtils.getTestBucketName;
 import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBucketOverrides;
 import static org.apache.hadoop.fs.s3a.commit.CommitConstants.FS_S3A_COMMITTER_NAME;
@@ -54,6 +55,7 @@ protected Configuration createConfiguration() {
     conf.setBoolean(MULTIPART_UPLOADS_ENABLED, false);
     conf.set(S3A_COMMITTER_FACTORY_KEY, CommitConstants.S3A_COMMITTER_FACTORY);
     conf.set(FS_S3A_COMMITTER_NAME, InternalCommitterConstants.COMMITTER_NAME_STAGING);
+    disableFilesystemCaching(conf);
     return conf;
   }