Extract trino-filesystem module

trinodb · Aug 15, 2022 · 6310e5e · 6310e5e
1 parent 9928afe
commit 6310e5e
Show file tree

Hide file tree

Showing 52 changed files with 263 additions and 147 deletions.
diff --git a/lib/trino-filesystem/pom.xml b/lib/trino-filesystem/pom.xml
@@ -0,0 +1,74 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <parent>
+        <groupId>io.trino</groupId>
+        <artifactId>trino-root</artifactId>
+        <version>393-SNAPSHOT</version>
+        <relativePath>../../pom.xml</relativePath>
+    </parent>
+
+    <artifactId>trino-filesystem</artifactId>
+    <name>trino-filesystem</name>
+
+    <properties>
+        <air.main.basedir>${project.parent.basedir}</air.main.basedir>
+    </properties>
+
+    <dependencies>
+        <dependency>
+            <groupId>io.trino</groupId>
+            <artifactId>trino-hdfs</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>io.trino</groupId>
+            <artifactId>trino-spi</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>io.trino.hadoop</groupId>
+            <artifactId>hadoop-apache</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>io.airlift</groupId>
+            <artifactId>slice</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>com.google.guava</groupId>
+            <artifactId>guava</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>javax.inject</groupId>
+            <artifactId>javax.inject</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>org.apache.iceberg</groupId>
+            <artifactId>iceberg-api</artifactId>
+        </dependency>
+
+        <!-- for testing -->
+        <dependency>
+            <groupId>io.trino</groupId>
+            <artifactId>trino-testing-services</artifactId>
+            <scope>test</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>org.assertj</groupId>
+            <artifactId>assertj-core</artifactId>
+            <scope>test</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>org.testng</groupId>
+            <artifactId>testng</artifactId>
+            <scope>test</scope>
+        </dependency>
+    </dependencies>
+</project>
diff --git a/...io/trino/plugin/iceberg/io/FileEntry.java → ...n/java/io/trino/filesystem/FileEntry.java b/...io/trino/plugin/iceberg/io/FileEntry.java → ...n/java/io/trino/filesystem/FileEntry.java
@@ -11,7 +11,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package io.trino.plugin.iceberg.io;
+package io.trino.filesystem;
 
 import static com.google.common.base.Preconditions.checkArgument;
 import static java.util.Objects.requireNonNull;

diff --git a/...trino/plugin/iceberg/io/FileIterator.java → ...ava/io/trino/filesystem/FileIterator.java b/...trino/plugin/iceberg/io/FileIterator.java → ...ava/io/trino/filesystem/FileIterator.java
@@ -11,7 +11,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package io.trino.plugin.iceberg.io;
+package io.trino.filesystem;
 
 import java.io.IOException;
 import java.util.NoSuchElementException;

diff --git a/...no/plugin/iceberg/io/TrinoFileSystem.java → .../io/trino/filesystem/TrinoFileSystem.java b/...no/plugin/iceberg/io/TrinoFileSystem.java → .../io/trino/filesystem/TrinoFileSystem.java
@@ -11,7 +11,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package io.trino.plugin.iceberg.io;
+package io.trino.filesystem;
 
 import org.apache.iceberg.io.FileIO;
 

diff --git a/...in/iceberg/io/TrinoFileSystemFactory.java → ...no/filesystem/TrinoFileSystemFactory.java b/...in/iceberg/io/TrinoFileSystemFactory.java → ...no/filesystem/TrinoFileSystemFactory.java
@@ -11,7 +11,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package io.trino.plugin.iceberg.io;
+package io.trino.filesystem;
 
 import io.trino.spi.connector.ConnectorSession;
 import io.trino.spi.security.ConnectorIdentity;

diff --git a/...o/trino/plugin/iceberg/io/TrinoInput.java → .../java/io/trino/filesystem/TrinoInput.java b/...o/trino/plugin/iceberg/io/TrinoInput.java → .../java/io/trino/filesystem/TrinoInput.java
@@ -11,7 +11,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package io.trino.plugin.iceberg.io;
+package io.trino.filesystem;
 
 import io.airlift.slice.Slice;
 import io.airlift.slice.Slices;

diff --git a/...ino/plugin/iceberg/io/TrinoInputFile.java → ...a/io/trino/filesystem/TrinoInputFile.java b/...ino/plugin/iceberg/io/TrinoInputFile.java → ...a/io/trino/filesystem/TrinoInputFile.java
@@ -11,7 +11,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package io.trino.plugin.iceberg.io;
+package io.trino.filesystem;
 
 import java.io.IOException;
 

diff --git a/...no/plugin/iceberg/io/TrinoOutputFile.java → .../io/trino/filesystem/TrinoOutputFile.java b/...no/plugin/iceberg/io/TrinoOutputFile.java → .../io/trino/filesystem/TrinoOutputFile.java
@@ -12,7 +12,7 @@
  * limitations under the License.
  */
 
-package io.trino.plugin.iceberg.io;
+package io.trino.filesystem;
 
 import java.io.IOException;
 import java.io.OutputStream;

diff --git a/...n/iceberg/io/fileio/ForwardingFileIo.java → ...o/filesystem/fileio/ForwardingFileIo.java b/...n/iceberg/io/fileio/ForwardingFileIo.java → ...o/filesystem/fileio/ForwardingFileIo.java
@@ -11,9 +11,9 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package io.trino.plugin.iceberg.io.fileio;
+package io.trino.filesystem.fileio;
 
-import io.trino.plugin.iceberg.io.TrinoFileSystem;
+import io.trino.filesystem.TrinoFileSystem;
 import org.apache.iceberg.io.FileIO;
 import org.apache.iceberg.io.InputFile;
 import org.apache.iceberg.io.OutputFile;

diff --git a/...ceberg/io/fileio/ForwardingInputFile.java → ...ilesystem/fileio/ForwardingInputFile.java b/...ceberg/io/fileio/ForwardingInputFile.java → ...ilesystem/fileio/ForwardingInputFile.java
@@ -11,9 +11,9 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package io.trino.plugin.iceberg.io.fileio;
+package io.trino.filesystem.fileio;
 
-import io.trino.plugin.iceberg.io.TrinoInputFile;
+import io.trino.filesystem.TrinoInputFile;
 import org.apache.iceberg.exceptions.NotFoundException;
 import org.apache.iceberg.io.InputFile;
 import org.apache.iceberg.io.SeekableInputStream;

diff --git a/...eberg/io/fileio/ForwardingOutputFile.java → ...lesystem/fileio/ForwardingOutputFile.java b/...eberg/io/fileio/ForwardingOutputFile.java → ...lesystem/fileio/ForwardingOutputFile.java
@@ -11,11 +11,11 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package io.trino.plugin.iceberg.io.fileio;
+package io.trino.filesystem.fileio;
 
 import com.google.common.io.CountingOutputStream;
-import io.trino.plugin.iceberg.io.TrinoFileSystem;
-import io.trino.plugin.iceberg.io.TrinoOutputFile;
+import io.trino.filesystem.TrinoFileSystem;
+import io.trino.filesystem.TrinoOutputFile;
 import org.apache.iceberg.io.InputFile;
 import org.apache.iceberg.io.OutputFile;
 import org.apache.iceberg.io.PositionOutputStream;

diff --git a/...o/plugin/iceberg/io/hdfs/HadoopPaths.java → ...io/trino/filesystem/hdfs/HadoopPaths.java b/...o/plugin/iceberg/io/hdfs/HadoopPaths.java → ...io/trino/filesystem/hdfs/HadoopPaths.java
@@ -11,15 +11,13 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package io.trino.plugin.iceberg.io.hdfs;
+package io.trino.filesystem.hdfs;
 
-import io.trino.spi.TrinoException;
 import org.apache.hadoop.fs.Path;
 
 import java.net.URI;
 import java.net.URLEncoder;
 
-import static io.trino.plugin.iceberg.IcebergErrorCode.ICEBERG_INVALID_METADATA;
 import static java.nio.charset.StandardCharsets.UTF_8;
 
 public final class HadoopPaths
@@ -32,7 +30,7 @@ public static Path hadoopPath(String path)
         Path hadoopPath = new Path(path);
         if ("s3".equals(hadoopPath.toUri().getScheme()) && !path.equals(hadoopPath.toString())) {
             if (hadoopPath.toUri().getFragment() != null) {
-                throw new TrinoException(ICEBERG_INVALID_METADATA, "Unexpected URI fragment in path: " + path);
+                throw new IllegalArgumentException("Unexpected URI fragment in path: " + path);
             }
             URI uri = URI.create(path);
             return new Path(uri + "#" + URLEncoder.encode(uri.getPath(), UTF_8));

diff --git a/...gin/iceberg/io/hdfs/HdfsFileIterator.java → ...ino/filesystem/hdfs/HdfsFileIterator.java b/...gin/iceberg/io/hdfs/HdfsFileIterator.java → ...ino/filesystem/hdfs/HdfsFileIterator.java
@@ -11,10 +11,10 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package io.trino.plugin.iceberg.io.hdfs;
+package io.trino.filesystem.hdfs;
 
-import io.trino.plugin.iceberg.io.FileEntry;
-import io.trino.plugin.iceberg.io.FileIterator;
+import io.trino.filesystem.FileEntry;
+import io.trino.filesystem.FileIterator;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.LocatedFileStatus;
 import org.apache.hadoop.fs.Path;

diff --git a/...lugin/iceberg/io/hdfs/HdfsFileSystem.java → ...trino/filesystem/hdfs/HdfsFileSystem.java b/...lugin/iceberg/io/hdfs/HdfsFileSystem.java → ...trino/filesystem/hdfs/HdfsFileSystem.java
@@ -11,23 +11,23 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package io.trino.plugin.iceberg.io.hdfs;
+package io.trino.filesystem.hdfs;
 
+import io.trino.filesystem.FileIterator;
+import io.trino.filesystem.TrinoFileSystem;
+import io.trino.filesystem.TrinoInputFile;
+import io.trino.filesystem.TrinoOutputFile;
+import io.trino.filesystem.fileio.ForwardingFileIo;
 import io.trino.hdfs.HdfsContext;
 import io.trino.hdfs.HdfsEnvironment;
-import io.trino.plugin.iceberg.io.FileIterator;
-import io.trino.plugin.iceberg.io.TrinoFileSystem;
-import io.trino.plugin.iceberg.io.TrinoInputFile;
-import io.trino.plugin.iceberg.io.TrinoOutputFile;
-import io.trino.plugin.iceberg.io.fileio.ForwardingFileIo;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.iceberg.io.FileIO;
 
 import java.io.FileNotFoundException;
 import java.io.IOException;
 
-import static io.trino.plugin.iceberg.io.hdfs.HadoopPaths.hadoopPath;
+import static io.trino.filesystem.hdfs.HadoopPaths.hadoopPath;
 import static java.util.Objects.requireNonNull;
 
 class HdfsFileSystem

diff --git a/...ceberg/io/hdfs/HdfsFileSystemFactory.java → ...ilesystem/hdfs/HdfsFileSystemFactory.java b/...ceberg/io/hdfs/HdfsFileSystemFactory.java → ...ilesystem/hdfs/HdfsFileSystemFactory.java
@@ -11,12 +11,12 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package io.trino.plugin.iceberg.io.hdfs;
+package io.trino.filesystem.hdfs;
 
+import io.trino.filesystem.TrinoFileSystem;
+import io.trino.filesystem.TrinoFileSystemFactory;
 import io.trino.hdfs.HdfsContext;
 import io.trino.hdfs.HdfsEnvironment;
-import io.trino.plugin.iceberg.io.TrinoFileSystem;
-import io.trino.plugin.iceberg.io.TrinoFileSystemFactory;
 import io.trino.spi.security.ConnectorIdentity;
 
 import javax.inject.Inject;

diff --git a/...ino/plugin/iceberg/io/hdfs/HdfsInput.java → ...a/io/trino/filesystem/hdfs/HdfsInput.java b/...ino/plugin/iceberg/io/hdfs/HdfsInput.java → ...a/io/trino/filesystem/hdfs/HdfsInput.java
@@ -11,12 +11,12 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package io.trino.plugin.iceberg.io.hdfs;
+package io.trino.filesystem.hdfs;
 
 import io.airlift.slice.Slice;
+import io.trino.filesystem.TrinoInput;
+import io.trino.filesystem.TrinoInputFile;
 import io.trino.hdfs.FSDataInputStreamTail;
-import io.trino.plugin.iceberg.io.TrinoInput;
-import io.trino.plugin.iceberg.io.TrinoInputFile;
 import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.iceberg.io.SeekableInputStream;
 

diff --git a/...plugin/iceberg/io/hdfs/HdfsInputFile.java → .../trino/filesystem/hdfs/HdfsInputFile.java b/...plugin/iceberg/io/hdfs/HdfsInputFile.java → .../trino/filesystem/hdfs/HdfsInputFile.java
@@ -11,12 +11,12 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package io.trino.plugin.iceberg.io.hdfs;
+package io.trino.filesystem.hdfs;
 
+import io.trino.filesystem.TrinoInput;
+import io.trino.filesystem.TrinoInputFile;
 import io.trino.hdfs.HdfsContext;
 import io.trino.hdfs.HdfsEnvironment;
-import io.trino.plugin.iceberg.io.TrinoInput;
-import io.trino.plugin.iceberg.io.TrinoInputFile;
 import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
@@ -25,7 +25,7 @@
 import java.io.IOException;
 
 import static com.google.common.base.Preconditions.checkArgument;
-import static io.trino.plugin.iceberg.io.hdfs.HadoopPaths.hadoopPath;
+import static io.trino.filesystem.hdfs.HadoopPaths.hadoopPath;
 import static java.util.Objects.requireNonNull;
 
 class HdfsInputFile

diff --git a/...lugin/iceberg/io/hdfs/HdfsOutputFile.java → ...trino/filesystem/hdfs/HdfsOutputFile.java b/...lugin/iceberg/io/hdfs/HdfsOutputFile.java → ...trino/filesystem/hdfs/HdfsOutputFile.java
@@ -11,18 +11,18 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package io.trino.plugin.iceberg.io.hdfs;
+package io.trino.filesystem.hdfs;
 
+import io.trino.filesystem.TrinoOutputFile;
 import io.trino.hdfs.HdfsContext;
 import io.trino.hdfs.HdfsEnvironment;
-import io.trino.plugin.iceberg.io.TrinoOutputFile;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 
 import java.io.IOException;
 import java.io.OutputStream;
 
-import static io.trino.plugin.iceberg.io.hdfs.HadoopPaths.hadoopPath;
+import static io.trino.filesystem.hdfs.HadoopPaths.hadoopPath;
 import static java.util.Objects.requireNonNull;
 
 class HdfsOutputFile

diff --git a/...n/iceberg/io/hdfs/TestHdfsFileSystem.java → ...o/filesystem/hdfs/TestHdfsFileSystem.java b/...n/iceberg/io/hdfs/TestHdfsFileSystem.java → ...o/filesystem/hdfs/TestHdfsFileSystem.java
@@ -11,12 +11,18 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package io.trino.plugin.iceberg.io.hdfs;
+package io.trino.filesystem.hdfs;
 
 import com.google.common.collect.ImmutableList;
-import io.trino.plugin.iceberg.io.FileIterator;
-import io.trino.plugin.iceberg.io.TrinoFileSystem;
-import io.trino.plugin.iceberg.io.TrinoFileSystemFactory;
+import com.google.common.collect.ImmutableSet;
+import io.trino.filesystem.FileIterator;
+import io.trino.filesystem.TrinoFileSystem;
+import io.trino.filesystem.TrinoFileSystemFactory;
+import io.trino.hdfs.DynamicHdfsConfiguration;
+import io.trino.hdfs.HdfsConfig;
+import io.trino.hdfs.HdfsConfigurationInitializer;
+import io.trino.hdfs.HdfsEnvironment;
+import io.trino.hdfs.authentication.NoHdfsAuthentication;
 import io.trino.spi.security.ConnectorIdentity;
 import org.testng.annotations.Test;
 
@@ -26,7 +32,6 @@
 
 import static com.google.common.io.MoreFiles.deleteRecursively;
 import static com.google.common.io.RecursiveDeleteOption.ALLOW_INSECURE;
-import static io.trino.plugin.hive.HiveTestUtils.HDFS_ENVIRONMENT;
 import static java.nio.file.Files.createDirectory;
 import static java.nio.file.Files.createFile;
 import static java.nio.file.Files.createTempDirectory;
@@ -38,7 +43,11 @@ public class TestHdfsFileSystem
     public void testListing()
             throws IOException
     {
-        TrinoFileSystemFactory factory = new HdfsFileSystemFactory(HDFS_ENVIRONMENT);
+        HdfsConfig hdfsConfig = new HdfsConfig();
+        DynamicHdfsConfiguration hdfsConfiguration = new DynamicHdfsConfiguration(new HdfsConfigurationInitializer(hdfsConfig), ImmutableSet.of());
+        HdfsEnvironment hdfsEnvironment = new HdfsEnvironment(hdfsConfiguration, hdfsConfig, new NoHdfsAuthentication());
+
+        TrinoFileSystemFactory factory = new HdfsFileSystemFactory(hdfsEnvironment);
         TrinoFileSystem fileSystem = factory.create(ConnectorIdentity.ofUser("test"));
 
         Path tempDir = createTempDirectory("testListing");