Control versions of indirect dependencies from hadoop (which is neede… (

#218) * Control versions of indirect dependencies from hadoop (which is needed for parquet at least for now). * Include specific version of protobuf-java for indirect dependencies.
ERDDAP · Oct 16, 2024 · 3f5be46 · 3f5be46
1 parent db58e31
commit 3f5be46
Show file tree

Hide file tree

Showing 3 changed files with 56 additions and 8 deletions.
diff --git a/WEB-INF/classes/gov/noaa/pfel/coastwatch/pointdata/Table.java b/WEB-INF/classes/gov/noaa/pfel/coastwatch/pointdata/Table.java
@@ -66,16 +66,15 @@
 import javax.sound.sampled.AudioInputStream;
 import javax.sound.sampled.AudioSystem;
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
 import org.apache.parquet.ParquetReadOptions;
 import org.apache.parquet.column.page.PageReadStore;
 import org.apache.parquet.example.data.simple.convert.GroupRecordConverter;
 import org.apache.parquet.hadoop.ParquetFileReader;
 import org.apache.parquet.hadoop.ParquetWriter;
 import org.apache.parquet.hadoop.metadata.CompressionCodecName;
-import org.apache.parquet.hadoop.util.HadoopInputFile;
 import org.apache.parquet.io.ColumnIOFactory;
 import org.apache.parquet.io.InputFile;
+import org.apache.parquet.io.LocalInputFile;
 import org.apache.parquet.io.LocalOutputFile;
 import org.apache.parquet.io.MessageColumnIO;
 import org.apache.parquet.io.RecordReader;
@@ -15956,10 +15955,7 @@ public void readParquet(
       String fullFileName, StringArray colNames, String[] colTypes, boolean simplify)
       throws Exception {
     clear();
-    Path filePath = new Path(fullFileName);
-    Configuration conf = new Configuration();
-
-    InputFile parquetFile = HadoopInputFile.fromPath(filePath, conf);
+    InputFile parquetFile = new LocalInputFile(java.nio.file.Path.of(fullFileName));
     ParquetFileReader fileReader =
         new ParquetFileReader(parquetFile, ParquetReadOptions.builder().build());
     try {

diff --git a/pom.xml b/pom.xml
@@ -521,6 +521,50 @@
                 <type>pom</type>
                 <scope>import</scope>
             </dependency>
+
+            <!-- This is needed due to indirect dependency in cdm-core -->
+            <!-- https://mvnrepository.com/artifact/com.google.protobuf/protobuf-java -->
+            <dependency>
+                <groupId>com.google.protobuf</groupId>
+                <artifactId>protobuf-java</artifactId>
+                <version>3.25.5</version>
+            </dependency>
+
+            <!--
+            The below are here to force updated dependencies of the hadoop libraries.
+            The default selections have security vulnerabilities. Ideally we wouldn't be including
+            hadoop at all, but parquet currently has dependencies on hadoop to read and write files.
+            -->
+            <!-- https://mvnrepository.com/artifact/org.apache.avro/avro -->
+            <dependency>
+                <groupId>org.apache.avro</groupId>
+                <artifactId>avro</artifactId>
+                <version>1.12.0</version>
+            </dependency>
+            <!-- https://mvnrepository.com/artifact/dnsjava/dnsjava -->
+            <dependency>
+                <groupId>dnsjava</groupId>
+                <artifactId>dnsjava</artifactId>
+                <version>3.6.2</version>
+            </dependency>
+            <!-- https://mvnrepository.com/artifact/org.bouncycastle/bcprov-jdk18on -->
+            <dependency>
+                <groupId>org.bouncycastle</groupId>
+                <artifactId>bcprov-jdk18on</artifactId>
+                <version>1.78.1</version>
+            </dependency>
+            <!-- https://mvnrepository.com/artifact/org.apache.commons/commons-configuration2 -->
+            <dependency>
+                <groupId>org.apache.commons</groupId>
+                <artifactId>commons-configuration2</artifactId>
+                <version>2.11.0</version>
+            </dependency>
+            <!-- https://mvnrepository.com/artifact/com.nimbusds/nimbus-jose-jwt -->
+            <dependency>
+                <groupId>com.nimbusds</groupId>
+                <artifactId>nimbus-jose-jwt</artifactId>
+                <version>9.41.2</version>
+            </dependency>
         </dependencies>
     </dependencyManagement>
 
@@ -932,6 +976,12 @@
             <groupId>org.apache.hadoop</groupId>
             <artifactId>hadoop-client</artifactId>
             <version>3.4.0</version>
+            <exclusions>
+                <exclusion>
+                    <groupId>org.bouncycastle</groupId>
+                    <artifactId>bcprov-jdk15on</artifactId>
+                </exclusion>
+            </exclusions> 
         </dependency>
         <dependency>
             <groupId>org.apache.parquet</groupId>

diff --git a/src/test/java/gov/noaa/pfel/coastwatch/pointdata/TableTests.java b/src/test/java/gov/noaa/pfel/coastwatch/pointdata/TableTests.java
@@ -18897,8 +18897,10 @@ void testMdb() throws Exception {
   void testReadParquet() throws Exception {
 
     String fileName =
-        TableTests.class
-            .getResource("/data/parquet/GHG_national_2012_m1_v2.0.0_a8c5929.parquet")
+        new java.io.File(
+                TableTests.class
+                    .getResource("/data/parquet/GHG_national_2012_m1_v2.0.0_a8c5929.parquet")
+                    .getFile())
             .getPath();
     // don't simplify
     Table table = new Table();