XFG-17: oap-hadoop: s3a:// -> s3://

oaplatform · Sep 14, 2023 · 46b4d47 · 46b4d47
1 parent ea8933d
commit 46b4d47
Show file tree

Hide file tree

Showing 5 changed files with 7 additions and 6 deletions.
diff --git a/oap-hadoop/src/main/java/oap/hadoop/OapFileSystemType.java b/oap-hadoop/src/main/java/oap/hadoop/OapFileSystemType.java
@@ -38,9 +38,9 @@ public Path _getPath( String name, Configuration configuration ) {
      * fs.s3a.access.key = access key
      * fs.s3a.secret.key = secret key
      * fs.s3a.bucket = bucket
-     * fs.s3a.endpoint = s3a://s3.{REGION}.amazonaws.com}
+     * fs.s3a.endpoint = s3://s3.{REGION}.amazonaws.com}
      */
-    S3A( "s3a://" ) {
+    S3A( "s3://" ) {
         @Override
         public Path _getPath( String name, Configuration configuration ) {
             String bucket = configuration.get( "fs.s3a.bucket" );

diff --git a/oap-hadoop/src/main/java/oap/hadoop/OapHadoopConfiguration.java b/oap-hadoop/src/main/java/oap/hadoop/OapHadoopConfiguration.java
@@ -29,6 +29,8 @@ public OapHadoopConfiguration( OapFileSystemType fileSystemType, Map<String, Obj
         configurationFixed.forEach( this::set );
 
         set( "fs.sftp.impl", "org.apache.hadoop.fs.sftp.SFTPFileSystem" );
+        set( "fs.s3.impl", "org.apache.hadoop.fs.s3a.S3AFileSystem" );
+        set( "fs.s3a.impl", "org.apache.hadoop.fs.s3a.S3AFileSystem" );
         set( "fs.s3a.path.style.access", "true" );
         set( "hadoop.tmp.dir", "/tmp" );
 

diff --git a/oap-hadoop/src/main/resources/META-INF/services/org.apache.hadoop.fs.FileSystem b/oap-hadoop/src/main/resources/META-INF/services/org.apache.hadoop.fs.FileSystem
diff --git a/oap-hadoop/src/test/java/oap/hadoop/FileSystemTest.java b/oap-hadoop/src/test/java/oap/hadoop/FileSystemTest.java
@@ -38,7 +38,7 @@ public void testCopy() {
         Path outFile = TestDirectoryFixture.testPath( "file-out.txt" );
         oap.io.Files.write( inFile, IoStreams.Encoding.PLAIN, "txt", ContentWriter.ofString() );
         var s3File = oapHadoopConfiguration.getPath( "folder/file.txt" );
-        assertThat( s3File ).isEqualTo( new org.apache.hadoop.fs.Path( "s3a://test-bucket/folder/file.txt" ) );
+        assertThat( s3File ).isEqualTo( new org.apache.hadoop.fs.Path( "s3://test-bucket/folder/file.txt" ) );
 
         assertThat( fileSystem.exists( s3File ) ).isFalse();
         fileSystem.copy( inFile, s3File, false );
@@ -63,6 +63,6 @@ public void testGetInputStreamFileNotFound() {
 
         FileSystem fileSystem = new FileSystem( oapHadoopConfiguration );
 
-        assertThat( fileSystem.getInputStream( new org.apache.hadoop.fs.Path( "s3a://test-bucket/testGetInputStreamFileNotFound.txt" ) ) ).isNull();
+        assertThat( fileSystem.getInputStream( new org.apache.hadoop.fs.Path( "s3://test-bucket/testGetInputStreamFileNotFound.txt" ) ) ).isNull();
     }
 }
diff --git a/oap-hadoop/src/test/java/oap/hadoop/OapHadoopConfigurationTest.java b/oap-hadoop/src/test/java/oap/hadoop/OapHadoopConfigurationTest.java
@@ -31,7 +31,7 @@ public void testGetFileSystemS3a() {
             Map.of( "fs.s3a.endpoint", "https://s3.us-east-1.awsamazon.com", "fs.s3a.bucket", "bucket1" ) );
 
         Path path = oapHadoopConfiguration.getPath( "folder/file.txt" );
-        assertThat( path ).isEqualTo( new Path( "s3a://bucket1/folder/file.txt" ) );
+        assertThat( path ).isEqualTo( new Path( "s3://bucket1/folder/file.txt" ) );
 
         assertThatCode( () -> path.getFileSystem( oapHadoopConfiguration ) ).doesNotThrowAnyException();
     }