Use C++ to parse and filter parquet footers.

build-libcudf.xml

@@ -42,6 +42,7 @@
       <arg value="-DCMAKE_INSTALL_PREFIX=${libcudf.install.path}"/>
       <arg value="-DCUDA_STATIC_RUNTIME=ON"/>
       <arg value="-DCUDF_ENABLE_ARROW_S3=OFF"/>
+      <arg value="-DCUDF_ENABLE_ARROW_PARQUET=OF"/>
       <arg value="-DCUDF_USE_ARROW_STATIC=ON"/>
       <arg value="-DPER_THREAD_DEFAULT_STREAM=${PER_THREAD_DEFAULT_STREAM}" />
       <arg value="-DRMM_LOGGING_LEVEL=${RMM_LOGGING_LEVEL}" />

build/Dockerfile.centos7

@@ -0,0 +1,36 @@
+#
+# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+###
+# Build the image for cudf development environment.
+#
+# Arguments: CUDA_VERSION=11.5.0
+#
+###
+ARG CUDA_VERSION=11.5.0
+# use rapids gpuci/cuda images until nvidia/cuda cuda 11.5+ images are available in docker hub
+# FROM nvidia/cuda:$CUDA_VERSION-devel-centos7
+FROM cudf-build:$CUDA_VERSION-devel-centos7
+
+## install a version of boost that is needed for arrow/parquet to work
+RUN cd /usr/local && wget https://boostorg.jfrog.io/artifactory/main/release/1.79.0/source/boost_1_79_0.tar.gz && \
+  tar -xzf boost_1_79_0.tar.gz && \
+  rm boost_1_79_0.tar.gz && \
+  cd boost_1_79_0 && \
+  ./bootstrap.sh --prefix=/usr/local && \
+  ./b2 install --prefix=/usr/local --with-filesystem --with-system && \
+   cd /usr/local && \
+   rm -rf boost_1_79_0

build/build-in-docker

@@ -31,7 +31,8 @@ LOCAL_MAVEN_REPO=${LOCAL_MAVEN_REPO:-"$HOME/.m2/repository"}
 PER_THREAD_DEFAULT_STREAM=${PER_THREAD_DEFAULT_STREAM:-ON}
 USE_GDS=${USE_GDS:-ON}
 
-IMAGE_NAME="cudf-build:${CUDA_VERSION}-devel-centos7"
+CUDF_IMAGE_NAME="cudf-build:${CUDA_VERSION}-devel-centos7"
+SPARK_IMAGE_NAME="spark-rapids-jni-build:${CUDA_VERSION}-devel-centos7"
 WORKSPACE_DIR=/rapids
 WORKSPACE_REPODIR="$WORKSPACE_DIR/spark-rapids-jni"
 WORKSPACE_MAVEN_REPODIR="$WORKSPACE_DIR/.m2/repository"
@@ -43,9 +44,14 @@ fi
 
 $DOCKER_CMD build -f $REPODIR/thirdparty/cudf/java/ci/Dockerfile.centos7 \
   --build-arg CUDA_VERSION=$CUDA_VERSION \
-  -t $IMAGE_NAME \
+  -t $CUDF_IMAGE_NAME \
   $REPODIR/thirdparty/cudf/java/ci
 
+$DOCKER_CMD build -f $REPODIR/build/Dockerfile.centos7 \
+  --build-arg CUDA_VERSION=$CUDA_VERSION \
+  -t $SPARK_IMAGE_NAME \
+  $REPODIR/build
+
 $DOCKER_CMD run -it -u $(id -u):$(id -g) --rm \
   -v "/etc/group:/etc/group:ro" \
   -v "/etc/passwd:/etc/passwd:ro" \
@@ -58,7 +64,7 @@ $DOCKER_CMD run -it -u $(id -u):$(id -g) --rm \
   -e CUDA_VISIBLE_DEVICES \
   -e PARALLEL_LEVEL \
   -e VERBOSE \
-  $IMAGE_NAME \
+  $SPARK_IMAGE_NAME \
   scl enable devtoolset-9 "mvn \
     -Dmaven.repo.local=$WORKSPACE_MAVEN_REPODIR \
     -DPER_THREAD_DEFAULT_STREAM=$PER_THREAD_DEFAULT_STREAM \

ci/Dockerfile

@@ -27,7 +27,6 @@ FROM gpuci/cuda:$CUDA_VERSION-devel-centos7
 RUN yum install -y centos-release-scl
 RUN yum install -y devtoolset-9 rh-python38 epel-release
 RUN yum install -y zlib-devel maven tar wget patch ninja-build
-RUN yum -y install https://packages.endpoint.com/rhel/7/os/x86_64/endpoint-repo-1.7-1.x86_64.rpm && yum install -y git
 RUN scl enable rh-python38 "pip install requests"
 
 ## pre-create the CMAKE_INSTALL_PREFIX folder, set writable by any user for Jenkins
@@ -38,3 +37,13 @@ RUN cd /usr/local/ && wget --quiet https://github.com/Kitware/CMake/releases/dow
    tar zxf cmake-${CMAKE_VERSION}-linux-x86_64.tar.gz && \
    rm cmake-${CMAKE_VERSION}-linux-x86_64.tar.gz
 ENV PATH /usr/local/cmake-${CMAKE_VERSION}-linux-x86_64/bin:$PATH
+
+## install a version of boost that is needed for arrow/parquet to work
+RUN cd /usr/local && wget https://boostorg.jfrog.io/artifactory/main/release/1.79.0/source/boost_1_79_0.tar.gz && \
+  tar -xzf boost_1_79_0.tar.gz && \
+  rm boost_1_79_0.tar.gz && \
+  cd boost_1_79_0 && \
+  ./bootstrap.sh --prefix=/usr/local && \
+  ./b2 install --prefix=/usr/local --with-filesystem --with-system && \
+   cd /usr/local && \
+   rm -rf boost_1_79_0

src/main/cpp/CMakeLists.txt

@@ -102,9 +102,27 @@ find_library(CUDFJNI_LIB "libcudfjni.a" REQUIRED NO_DEFAULT_PATH
   HINTS "${PROJECT_BINARY_DIR}/../libcudfjni"
 )
 
+# parquet
+find_library(PARQUET_LIB "libparquet.a" REQUIRED NO_DEFAULT_PATH
+  HINTS "${PROJECT_BINARY_DIR}/../libcudf-install/lib64"
+)
+
+# Internal parquet headers
+set (GENERATED_PARQUET_INCLUDE
+    "${CUDF_DIR}/cpp/build/_deps/arrow-src/cpp/src/"
+    CACHE STRING "generated parquet thrift headers"
+)
+
+# thrift
+find_library(THRIFT_LIB "libthrift.a" REQUIRED NO_DEFAULT_PATH
+    HINTS "${CUDF_DIR}/cpp/build/_deps/arrow-build/thrift_ep-install/lib/"
+)
+
 set(CUDFJNI_INCLUDE_DIRS
   "${CUDF_DIR}/java/src/main/native/include"
   "${CUDF_DIR}/java/src/main/native/src"
+  "${GENERATED_PARQUET_INCLUDE}"
+  "${CUDF_DIR}/cpp/build/_deps/arrow-build/thrift_ep-install/include/"
 )
 
 # ##################################################################################################
@@ -113,6 +131,7 @@ set(CUDFJNI_INCLUDE_DIRS
 add_library(
   spark_rapids_jni SHARED
   src/RowConversionJni.cpp
+  src/NativeParquetJni.cpp
   src/row_conversion.cu
 )
 
@@ -159,6 +178,8 @@ target_link_libraries(
   -Wl,--whole-archive
     ${CUDFJNI_LIB}
     cudf::cudf
+    ${PARQUET_LIB}
+    ${THRIFT_LIB}
   -Wl,--no-whole-archive
   cudf::cudf
 )