Use direct dependency on Parquet

trinodb · Jul 28, 2023 · 4bb2d7f · 4bb2d7f
1 parent 82b3ace
commit 4bb2d7f
Show file tree

Hide file tree

Showing 11 changed files with 284 additions and 41 deletions.
diff --git a/lib/trino-hive-formats/pom.xml b/lib/trino-hive-formats/pom.xml
@@ -112,12 +112,26 @@
             <scope>provided</scope>
         </dependency>
 
+        <!-- TODO: remove after making Avro use Aircompressor -->
+        <dependency>
+            <groupId>com.github.luben</groupId>
+            <artifactId>zstd-jni</artifactId>
+            <scope>runtime</scope>
+        </dependency>
+
         <dependency>
             <groupId>io.trino</groupId>
             <artifactId>trino-hadoop-toolkit</artifactId>
             <scope>runtime</scope>
         </dependency>
 
+        <!-- TODO: remove after making Avro use Aircompressor -->
+        <dependency>
+            <groupId>org.xerial.snappy</groupId>
+            <artifactId>snappy-java</artifactId>
+            <scope>runtime</scope>
+        </dependency>
+
         <dependency>
             <groupId>io.airlift</groupId>
             <artifactId>testing</artifactId>

diff --git a/lib/trino-parquet/pom.xml b/lib/trino-parquet/pom.xml
@@ -51,11 +51,6 @@
             <artifactId>trino-plugin-toolkit</artifactId>
         </dependency>
 
-        <dependency>
-            <groupId>io.trino.hive</groupId>
-            <artifactId>hive-apache</artifactId>
-        </dependency>
-
         <dependency>
             <groupId>it.unimi.dsi</groupId>
             <artifactId>fastutil</artifactId>
@@ -71,6 +66,31 @@
             <artifactId>joda-time</artifactId>
         </dependency>
 
+        <dependency>
+            <groupId>org.apache.parquet</groupId>
+            <artifactId>parquet-column</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>org.apache.parquet</groupId>
+            <artifactId>parquet-common</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>org.apache.parquet</groupId>
+            <artifactId>parquet-encoding</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>org.apache.parquet</groupId>
+            <artifactId>parquet-format-structures</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>org.apache.parquet</groupId>
+            <artifactId>parquet-hadoop</artifactId>
+        </dependency>
+
         <dependency>
             <groupId>io.trino</groupId>
             <artifactId>trino-spi</artifactId>
@@ -137,6 +157,12 @@
             <scope>test</scope>
         </dependency>
 
+        <dependency>
+            <groupId>io.trino.hive</groupId>
+            <artifactId>hive-apache</artifactId>
+            <scope>test</scope>
+        </dependency>
+
         <dependency>
             <groupId>io.trino.tpch</groupId>
             <artifactId>tpch</artifactId>

diff --git a/lib/trino-parquet/src/main/java/io/trino/parquet/reader/MetadataReader.java b/lib/trino-parquet/src/main/java/io/trino/parquet/reader/MetadataReader.java
@@ -64,7 +64,6 @@
 import static java.lang.Boolean.TRUE;
 import static java.lang.Math.min;
 import static java.lang.Math.toIntExact;
-import static org.apache.hadoop.hive.ql.io.parquet.write.DataWritableWriteSupport.WRITER_TIMEZONE;
 import static org.apache.parquet.format.Util.readFileMetaData;
 import static org.apache.parquet.format.converter.ParquetMetadataConverterUtil.getLogicalTypeAnnotation;
 
@@ -401,7 +400,7 @@ private static void validateFileMetadata(ParquetDataSourceId dataSourceId, org.a
         ParquetWriteValidation writeValidation = parquetWriteValidation.get();
         writeValidation.validateTimeZone(
                 dataSourceId,
-                Optional.ofNullable(fileMetaData.getKeyValueMetaData().get(WRITER_TIMEZONE)));
+                Optional.ofNullable(fileMetaData.getKeyValueMetaData().get("writer.time.zone")));
         writeValidation.validateColumns(dataSourceId, fileMetaData.getSchema());
     }
 }
diff --git a/lib/trino-parquet/src/main/java/io/trino/parquet/writer/ParquetWriter.java b/lib/trino-parquet/src/main/java/io/trino/parquet/writer/ParquetWriter.java
@@ -69,7 +69,6 @@
 import static java.nio.charset.StandardCharsets.US_ASCII;
 import static java.util.Collections.nCopies;
 import static java.util.Objects.requireNonNull;
-import static org.apache.hadoop.hive.ql.io.parquet.write.DataWritableWriteSupport.WRITER_TIMEZONE;
 import static org.apache.parquet.column.ParquetProperties.WriterVersion.PARQUET_1_0;
 
 public class ParquetWriter
@@ -349,7 +348,7 @@ Slice getFooter(List<RowGroup> rowGroups, MessageType messageType)
         fileMetaData.setSchema(MessageTypeConverter.toParquetSchema(messageType));
         // Added based on org.apache.hadoop.hive.ql.io.parquet.write.DataWritableWriteSupport
         parquetTimeZone.ifPresent(dateTimeZone -> fileMetaData.setKey_value_metadata(
-                ImmutableList.of(new KeyValue(WRITER_TIMEZONE).setValue(dateTimeZone.getID()))));
+                ImmutableList.of(new KeyValue("writer.time.zone").setValue(dateTimeZone.getID()))));
         long totalRows = rowGroups.stream().mapToLong(RowGroup::getNum_rows).sum();
         fileMetaData.setNum_rows(totalRows);
         fileMetaData.setRow_groups(ImmutableList.copyOf(rowGroups));

diff --git a/...rquet/src/main/java/org/apache/parquet/format/converter/ParquetMetadataConverterUtil.java b/...rquet/src/main/java/org/apache/parquet/format/converter/ParquetMetadataConverterUtil.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.parquet.format.converter;
+
+import org.apache.parquet.format.ConvertedType;
+import org.apache.parquet.format.LogicalType;
+import org.apache.parquet.format.SchemaElement;
+import org.apache.parquet.schema.LogicalTypeAnnotation;
+
+public final class ParquetMetadataConverterUtil
+{
+    private ParquetMetadataConverterUtil() {}
+
+    public static LogicalTypeAnnotation getLogicalTypeAnnotation(ParquetMetadataConverter converter, ConvertedType type, SchemaElement element)
+    {
+        return converter.getLogicalTypeAnnotation(type, element);
+    }
+
+    public static LogicalTypeAnnotation getLogicalTypeAnnotation(ParquetMetadataConverter converter, LogicalType type)
+    {
+        return converter.getLogicalTypeAnnotation(type);
+    }
+
+    public static LogicalType convertToLogicalType(ParquetMetadataConverter converter, LogicalTypeAnnotation annotation)
+    {
+        return converter.convertToLogicalType(annotation);
+    }
+}
diff --git a/plugin/trino-delta-lake/pom.xml b/plugin/trino-delta-lake/pom.xml
@@ -159,11 +159,6 @@
             <artifactId>hadoop-apache</artifactId>
         </dependency>
 
-        <dependency>
-            <groupId>io.trino.hive</groupId>
-            <artifactId>hive-apache</artifactId>
-        </dependency>
-
         <dependency>
             <groupId>jakarta.annotation</groupId>
             <artifactId>jakarta.annotation-api</artifactId>
@@ -184,6 +179,26 @@
             <artifactId>antlr4-runtime</artifactId>
         </dependency>
 
+        <dependency>
+            <groupId>org.apache.parquet</groupId>
+            <artifactId>parquet-column</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>org.apache.parquet</groupId>
+            <artifactId>parquet-common</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>org.apache.parquet</groupId>
+            <artifactId>parquet-format-structures</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>org.apache.parquet</groupId>
+            <artifactId>parquet-hadoop</artifactId>
+        </dependency>
+
         <dependency>
             <groupId>org.roaringbitmap</groupId>
             <artifactId>RoaringBitmap</artifactId>

diff --git a/plugin/trino-hive/pom.xml b/plugin/trino-hive/pom.xml
@@ -257,6 +257,26 @@
             </exclusions>
         </dependency>
 
+        <dependency>
+            <groupId>org.apache.parquet</groupId>
+            <artifactId>parquet-column</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>org.apache.parquet</groupId>
+            <artifactId>parquet-common</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>org.apache.parquet</groupId>
+            <artifactId>parquet-format-structures</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>org.apache.parquet</groupId>
+            <artifactId>parquet-hadoop</artifactId>
+        </dependency>
+
         <dependency>
             <groupId>org.apache.thrift</groupId>
             <artifactId>libthrift</artifactId>
@@ -326,12 +346,6 @@
             <scope>runtime</scope>
         </dependency>
 
-        <dependency>
-            <groupId>org.xerial.snappy</groupId>
-            <artifactId>snappy-java</artifactId>
-            <scope>runtime</scope>
-        </dependency>
-
         <dependency>
             <groupId>io.airlift</groupId>
             <artifactId>testing</artifactId>

diff --git a/plugin/trino-hudi/pom.xml b/plugin/trino-hudi/pom.xml
@@ -93,6 +93,13 @@
         <dependency>
             <groupId>io.trino</groupId>
             <artifactId>trino-hive</artifactId>
+            <exclusions>
+                <!-- TODO: remove when removed from trino-hive -->
+                <exclusion>
+                    <groupId>io.trino.hive</groupId>
+                    <artifactId>hive-apache</artifactId>
+                </exclusion>
+            </exclusions>
         </dependency>
 
         <dependency>
@@ -110,11 +117,6 @@
             <artifactId>trino-plugin-toolkit</artifactId>
         </dependency>
 
-        <dependency>
-            <groupId>io.trino.hive</groupId>
-            <artifactId>hive-apache</artifactId>
-        </dependency>
-
         <dependency>
             <groupId>jakarta.annotation</groupId>
             <artifactId>jakarta.annotation-api</artifactId>
@@ -135,6 +137,16 @@
             <artifactId>avro</artifactId>
         </dependency>
 
+        <dependency>
+            <groupId>org.apache.parquet</groupId>
+            <artifactId>parquet-column</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>org.apache.parquet</groupId>
+            <artifactId>parquet-hadoop</artifactId>
+        </dependency>
+
         <dependency>
             <groupId>org.weakref</groupId>
             <artifactId>jmxutils</artifactId>
@@ -396,5 +408,4 @@
             </plugin>
         </plugins>
     </build>
-
 </project>
diff --git a/plugin/trino-iceberg/pom.xml b/plugin/trino-iceberg/pom.xml
@@ -143,6 +143,11 @@
                     <groupId>io.airlift</groupId>
                     <artifactId>http-client</artifactId>
                 </exclusion>
+                <!-- TODO: remove when removed from trino-hive -->
+                <exclusion>
+                    <groupId>io.trino.hive</groupId>
+                    <artifactId>hive-apache</artifactId>
+                </exclusion>
             </exclusions>
         </dependency>
 
@@ -166,11 +171,6 @@
             <artifactId>trino-plugin-toolkit</artifactId>
         </dependency>
 
-        <dependency>
-            <groupId>io.trino.hive</groupId>
-            <artifactId>hive-apache</artifactId>
-        </dependency>
-
         <dependency>
             <groupId>io.trino.hive</groupId>
             <artifactId>hive-thrift</artifactId>
@@ -234,6 +234,26 @@
             <artifactId>iceberg-parquet</artifactId>
         </dependency>
 
+        <dependency>
+            <groupId>org.apache.parquet</groupId>
+            <artifactId>parquet-column</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>org.apache.parquet</groupId>
+            <artifactId>parquet-common</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>org.apache.parquet</groupId>
+            <artifactId>parquet-format-structures</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>org.apache.parquet</groupId>
+            <artifactId>parquet-hadoop</artifactId>
+        </dependency>
+
         <dependency>
             <groupId>org.jdbi</groupId>
             <artifactId>jdbi3-core</artifactId>
@@ -559,6 +579,19 @@
     </dependencies>
 
     <build>
+        <pluginManagement>
+            <plugins>
+                <plugin>
+                    <groupId>org.apache.maven.plugins</groupId>
+                    <artifactId>maven-dependency-plugin</artifactId>
+                    <configuration>
+                        <ignoredNonTestScopedDependencies>
+                            <ignoredNonTestScopedDependency>org.apache.parquet:parquet-common</ignoredNonTestScopedDependency>
+                        </ignoredNonTestScopedDependencies>
+                    </configuration>
+                </plugin>
+            </plugins>
+        </pluginManagement>
         <plugins>
             <plugin>
                 <groupId>org.antlr</groupId>