Add Parquet decryption support for Hive tables

trinodb · Dec 18, 2024 · be7a880 · be7a880
1 parent 465f48a
commit be7a880
Show file tree

Hide file tree

Showing 86 changed files with 5,545 additions and 146 deletions.
diff --git a/lib/trino-filesystem/pom.xml b/lib/trino-filesystem/pom.xml
@@ -188,4 +188,18 @@
             <scope>test</scope>
         </dependency>
     </dependencies>
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.basepom.maven</groupId>
+                <artifactId>duplicate-finder-maven-plugin</artifactId>
+                <configuration>
+                    <ignoredResourcePatterns>
+                        <ignoredResourcePattern>about.html</ignoredResourcePattern>
+                        <ignoredResourcePattern>iceberg-build.properties</ignoredResourcePattern>
+                    </ignoredResourcePatterns>
+                </configuration>
+            </plugin>
+        </plugins>
+    </build>
 </project>
diff --git a/lib/trino-orc/pom.xml b/lib/trino-orc/pom.xml
@@ -208,6 +208,16 @@
                     </excludes>
                 </configuration>
             </plugin>
+            <plugin>
+                <groupId>org.basepom.maven</groupId>
+                <artifactId>duplicate-finder-maven-plugin</artifactId>
+                <configuration>
+                    <ignoredResourcePatterns>
+                        <ignoredResourcePattern>about.html</ignoredResourcePattern>
+                        <ignoredResourcePattern>iceberg-build.properties</ignoredResourcePattern>
+                    </ignoredResourcePatterns>
+                </configuration>
+            </plugin>
         </plugins>
     </build>
 </project>
diff --git a/lib/trino-parquet/pom.xml b/lib/trino-parquet/pom.xml
@@ -13,12 +13,25 @@
     <description>Trino - Parquet file format support</description>
 
     <dependencies>
+        <dependency>
+            <groupId>com.fasterxml.jackson.core</groupId>
+            <artifactId>jackson-core</artifactId>
+            <type>jar</type>
+            <scope>compile</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>com.fasterxml.jackson.core</groupId>
+            <artifactId>jackson-databind</artifactId>
+            <type>jar</type>
+            <scope>compile</scope>
+        </dependency>
+
         <dependency>
             <groupId>com.google.errorprone</groupId>
             <artifactId>error_prone_annotations</artifactId>
             <optional>true</optional>
         </dependency>
-
         <dependency>
             <groupId>com.google.guava</groupId>
             <artifactId>guava</artifactId>
@@ -29,6 +42,11 @@
             <artifactId>aircompressor-v3</artifactId>
         </dependency>
 
+        <dependency>
+            <groupId>io.airlift</groupId>
+            <artifactId>json</artifactId>
+        </dependency>
+
         <dependency>
             <groupId>io.airlift</groupId>
             <artifactId>log</artifactId>
@@ -95,6 +113,12 @@
             </exclusions>
         </dependency>
 
+        <dependency>
+            <groupId>io.trino</groupId>
+            <artifactId>trino-filesystem</artifactId>
+            <scope>provided</scope>
+        </dependency>
+
         <dependency>
             <groupId>io.trino</groupId>
             <artifactId>trino-spi</artifactId>

diff --git a/lib/trino-parquet/src/main/java/io/trino/parquet/DataPage.java b/lib/trino-parquet/src/main/java/io/trino/parquet/DataPage.java
@@ -21,12 +21,14 @@ public abstract sealed class DataPage
 {
     protected final int valueCount;
     private final OptionalLong firstRowIndex;
+    private final int pageIndex;
 
-    public DataPage(int uncompressedSize, int valueCount, OptionalLong firstRowIndex)
+    public DataPage(int uncompressedSize, int valueCount, OptionalLong firstRowIndex, int pageIndex)
     {
         super(uncompressedSize);
         this.valueCount = valueCount;
         this.firstRowIndex = firstRowIndex;
+        this.pageIndex = pageIndex;
     }
 
     /**
@@ -41,4 +43,9 @@ public int getValueCount()
     {
         return valueCount;
     }
+
+    public int getPageIndex()
+    {
+        return pageIndex;
+    }
 }
diff --git a/lib/trino-parquet/src/main/java/io/trino/parquet/DataPageV1.java b/lib/trino-parquet/src/main/java/io/trino/parquet/DataPageV1.java
@@ -35,15 +35,17 @@ public DataPageV1(
             OptionalLong firstRowIndex,
             ParquetEncoding repetitionLevelEncoding,
             ParquetEncoding definitionLevelEncoding,
-            ParquetEncoding valuesEncoding)
+            ParquetEncoding valuesEncoding,
+            int pageIndex)
     {
-        super(uncompressedSize, valueCount, firstRowIndex);
+        super(uncompressedSize, valueCount, firstRowIndex, pageIndex);
         this.slice = requireNonNull(slice, "slice is null");
         this.repetitionLevelEncoding = repetitionLevelEncoding;
         this.definitionLevelEncoding = definitionLevelEncoding;
         this.valuesEncoding = valuesEncoding;
     }
 
+    @Override
     public Slice getSlice()
     {
         return slice;

diff --git a/lib/trino-parquet/src/main/java/io/trino/parquet/DataPageV2.java b/lib/trino-parquet/src/main/java/io/trino/parquet/DataPageV2.java
@@ -44,9 +44,10 @@ public DataPageV2(
             int uncompressedSize,
             OptionalLong firstRowIndex,
             Statistics<?> statistics,
-            boolean isCompressed)
+            boolean isCompressed,
+            int pageIndex)
     {
-        super(uncompressedSize, valueCount, firstRowIndex);
+        super(uncompressedSize, valueCount, firstRowIndex, pageIndex);
         this.rowCount = rowCount;
         this.nullCount = nullCount;
         this.repetitionLevels = requireNonNull(repetitionLevels, "repetitionLevels slice is null");
@@ -82,6 +83,7 @@ public ParquetEncoding getDataEncoding()
         return dataEncoding;
     }
 
+    @Override
     public Slice getSlice()
     {
         return slice;

diff --git a/lib/trino-parquet/src/main/java/io/trino/parquet/DictionaryPage.java b/lib/trino-parquet/src/main/java/io/trino/parquet/DictionaryPage.java
@@ -43,6 +43,7 @@ public DictionaryPage(Slice slice, int uncompressedSize, int dictionarySize, Par
                 encoding);
     }
 
+    @Override
     public Slice getSlice()
     {
         return slice;

diff --git a/lib/trino-parquet/src/main/java/io/trino/parquet/EncryptionUtils.java b/lib/trino-parquet/src/main/java/io/trino/parquet/EncryptionUtils.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.trino.parquet;
+
+import io.airlift.log.Logger;
+import io.trino.filesystem.Location;
+import io.trino.filesystem.TrinoFileSystem;
+import io.trino.parquet.crypto.FileDecryptionProperties;
+import io.trino.parquet.crypto.InternalFileDecryptor;
+import io.trino.parquet.crypto.TrinoCryptoConfigurationUtil;
+import io.trino.parquet.crypto.TrinoDecryptionPropertiesFactory;
+
+import java.lang.reflect.InvocationTargetException;
+import java.util.Optional;
+
+public class EncryptionUtils
+{
+    public static final Logger LOG = Logger.get(EncryptionUtils.class);
+
+    private EncryptionUtils() {}
+
+    public static Optional<InternalFileDecryptor> createDecryptor(ParquetReaderOptions parquetReaderOptions, Location filePath, TrinoFileSystem trinoFileSystem)
+    {
+        if (parquetReaderOptions == null || filePath == null || trinoFileSystem == null) {
+            return Optional.empty();
+        }
+
+        Optional<TrinoDecryptionPropertiesFactory> cryptoFactory = loadDecryptionPropertiesFactory(parquetReaderOptions);
+        Optional<FileDecryptionProperties> fileDecryptionProperties = cryptoFactory.map(factory -> factory.getFileDecryptionProperties(parquetReaderOptions, filePath, trinoFileSystem));
+        return fileDecryptionProperties.map(properties -> new InternalFileDecryptor(properties));
+    }
+
+    private static Optional<TrinoDecryptionPropertiesFactory> loadDecryptionPropertiesFactory(ParquetReaderOptions trinoParquetCryptoConfig)
+    {
+        if (trinoParquetCryptoConfig.getCryptoFactoryClass() == null) {
+            return Optional.empty();
+        }
+        final Class<?> foundClass = TrinoCryptoConfigurationUtil.getClassFromConfig(
+                trinoParquetCryptoConfig.getCryptoFactoryClass(), TrinoDecryptionPropertiesFactory.class);
+
+        if (foundClass == null) {
+            return Optional.empty();
+        }
+
+        try {
+            return Optional.ofNullable((TrinoDecryptionPropertiesFactory) foundClass.getConstructor().newInstance());
+        }
+        catch (InstantiationException | IllegalAccessException | NoSuchMethodException | InvocationTargetException e) {
+            LOG.warn("could not instantiate decryptionPropertiesFactoryClass class: " + foundClass, e);
+            return Optional.empty();
+        }
+    }
+}
diff --git a/lib/trino-parquet/src/main/java/io/trino/parquet/Page.java b/lib/trino-parquet/src/main/java/io/trino/parquet/Page.java
@@ -13,6 +13,8 @@
  */
 package io.trino.parquet;
 
+import io.airlift.slice.Slice;
+
 public abstract class Page
 {
     protected final int uncompressedSize;
@@ -26,4 +28,6 @@ public int getUncompressedSize()
     {
         return uncompressedSize;
     }
+
+    public abstract Slice getSlice();
 }