Reduce Hadoop RemoteIterator usage in DirectoryLister

trinodb · May 16, 2023 · cfedd7e · cfedd7e
1 parent d834094
commit cfedd7e
Show file tree

Hide file tree

Showing 10 changed files with 28 additions and 12 deletions.
diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/fs/CachingDirectoryLister.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/fs/CachingDirectoryLister.java
@@ -28,7 +28,6 @@
 import io.trino.spi.connector.SchemaTablePrefix;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.RemoteIterator;
 import org.weakref.jmx.Managed;
 
 import javax.inject.Inject;

diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/fs/DirectoryLister.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/fs/DirectoryLister.java
@@ -17,7 +17,6 @@
 import io.trino.plugin.hive.metastore.Table;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.RemoteIterator;
 
 import java.io.IOException;
 

diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/fs/HiveFileIterator.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/fs/HiveFileIterator.java
@@ -21,7 +21,6 @@
 import io.trino.spi.TrinoException;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.RemoteIterator;
 
 import java.io.FileNotFoundException;
 import java.io.IOException;

diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/fs/RemoteIterator.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/fs/RemoteIterator.java
@@ -0,0 +1,25 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.trino.plugin.hive.fs;
+
+import java.io.IOException;
+
+public interface RemoteIterator<T>
+{
+    boolean hasNext()
+            throws IOException;
+
+    T next()
+            throws IOException;
+}
diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/fs/SimpleRemoteIterator.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/fs/SimpleRemoteIterator.java
@@ -13,8 +13,6 @@
  */
 package io.trino.plugin.hive.fs;
 
-import org.apache.hadoop.fs.RemoteIterator;
-
 import java.io.IOException;
 import java.util.Iterator;
 

diff --git a/...no-hive/src/main/java/io/trino/plugin/hive/fs/TransactionScopeCachingDirectoryLister.java b/...no-hive/src/main/java/io/trino/plugin/hive/fs/TransactionScopeCachingDirectoryLister.java
@@ -21,7 +21,6 @@
 import io.trino.plugin.hive.metastore.Table;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.RemoteIterator;
 
 import javax.annotation.Nullable;
 import javax.annotation.concurrent.GuardedBy;

diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/fs/TrinoFileStatusRemoteIterator.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/fs/TrinoFileStatusRemoteIterator.java
@@ -14,7 +14,6 @@
 package io.trino.plugin.hive.fs;
 
 import org.apache.hadoop.fs.LocatedFileStatus;
-import org.apache.hadoop.fs.RemoteIterator;
 
 import java.io.IOException;
 
@@ -23,9 +22,9 @@
 public class TrinoFileStatusRemoteIterator
         implements RemoteIterator<TrinoFileStatus>
 {
-    private final RemoteIterator<LocatedFileStatus> iterator;
+    private final org.apache.hadoop.fs.RemoteIterator<LocatedFileStatus> iterator;
 
-    public TrinoFileStatusRemoteIterator(RemoteIterator<LocatedFileStatus> iterator)
+    public TrinoFileStatusRemoteIterator(org.apache.hadoop.fs.RemoteIterator<LocatedFileStatus> iterator)
     {
         this.iterator = requireNonNull(iterator, "iterator is null");
     }

diff --git a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/AbstractTestHive.java b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/AbstractTestHive.java
@@ -38,6 +38,7 @@
 import io.trino.plugin.hive.LocationService.WriteInfo;
 import io.trino.plugin.hive.aws.athena.PartitionProjectionService;
 import io.trino.plugin.hive.fs.DirectoryLister;
+import io.trino.plugin.hive.fs.RemoteIterator;
 import io.trino.plugin.hive.fs.TransactionScopeCachingDirectoryListerFactory;
 import io.trino.plugin.hive.fs.TrinoFileStatus;
 import io.trino.plugin.hive.fs.TrinoFileStatusRemoteIterator;
@@ -142,7 +143,6 @@
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.RemoteIterator;
 import org.apache.hadoop.hive.metastore.TableType;
 import org.joda.time.DateTime;
 import org.testng.annotations.AfterClass;

diff --git a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/fs/FileSystemDirectoryLister.java b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/fs/FileSystemDirectoryLister.java
@@ -17,7 +17,6 @@
 import io.trino.plugin.hive.metastore.Table;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.RemoteIterator;
 
 import java.io.IOException;
 

diff --git a/...ive/src/test/java/io/trino/plugin/hive/fs/TestTransactionScopeCachingDirectoryLister.java b/...ive/src/test/java/io/trino/plugin/hive/fs/TestTransactionScopeCachingDirectoryLister.java
@@ -26,7 +26,6 @@
 import io.trino.plugin.hive.metastore.Table;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.RemoteIterator;
 import org.testng.annotations.Test;
 
 import java.io.IOException;