Use pre-built nvcomp 2.3 binaries by default

build.sh

@@ -17,7 +17,7 @@ ARGS=$*
 # script, and that this script resides in the repo dir!
 REPODIR=$(cd $(dirname $0); pwd)
 
-VALIDARGS="clean libcudf cudf cudfjar dask_cudf benchmarks tests libcudf_kafka cudf_kafka custreamz -v -g -n -l --allgpuarch --disable_nvtx --show_depr_warn --ptds -h --build_metrics --incl_cache_stats"
+VALIDARGS="clean libcudf cudf cudfjar dask_cudf benchmarks tests libcudf_kafka cudf_kafka custreamz -v -g -n -l --allgpuarch --disable_nvtx --opensource_nvcomp  --show_depr_warn --ptds -h --build_metrics --incl_cache_stats"
 HELP="$0 [clean] [libcudf] [cudf] [cudfjar] [dask_cudf] [benchmarks] [tests] [libcudf_kafka] [cudf_kafka] [custreamz] [-v] [-g] [-n] [-h] [--cmake-args=\\\"<args>\\\"]
    clean                         - remove all existing build artifacts and configuration (start
                                    over)
@@ -35,6 +35,7 @@ HELP="$0 [clean] [libcudf] [cudf] [cudfjar] [dask_cudf] [benchmarks] [tests] [li
    -n                            - no install step
    --allgpuarch                  - build for all supported GPU architectures
    --disable_nvtx                - disable inserting NVTX profiling ranges
+   --opensource_nvcomp           - disable use of proprietary nvcomp extensions
    --show_depr_warn              - show cmake deprecation warnings
    --ptds                        - enable per-thread default stream
    --build_metrics               - generate build metrics report for libcudf
@@ -67,6 +68,7 @@ BUILD_DISABLE_DEPRECATION_WARNING=ON
 BUILD_PER_THREAD_DEFAULT_STREAM=OFF
 BUILD_REPORT_METRICS=OFF
 BUILD_REPORT_INCL_CACHE_STATS=OFF
+USE_PROPRIETARY_NVCOMP=ON
 
 # Set defaults for vars that may not have been defined externally
 #  FIXME: if INSTALL_PREFIX is not set, check PREFIX, then check
@@ -144,6 +146,7 @@ function buildLibCudfJniInDocker {
                 -DCMAKE_CUDA_ARCHITECTURES=${CUDF_CMAKE_CUDA_ARCHITECTURES} \
                 -DCMAKE_INSTALL_PREFIX=/usr/local/rapids \
                 -DUSE_NVTX=ON \
+                -DCMAKE_USE_PROPRIETARY_NVCOMP=ON \
                 -DCUDF_USE_ARROW_STATIC=ON \
                 -DCUDF_ENABLE_ARROW_S3=OFF \
                 -DBUILD_TESTS=OFF \
@@ -209,6 +212,9 @@ fi
 if hasArg --disable_nvtx; then
     BUILD_NVTX="OFF"
 fi
+if hasArg --opensource_nvcomp; then
+    USE_PROPRIETARY_NVCOMP="OFF"
+fi
 if hasArg --show_depr_warn; then
     BUILD_DISABLE_DEPRECATION_WARNING=OFF
 fi
@@ -271,6 +277,7 @@ if buildAll || hasArg libcudf; then
           -DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX} \
           -DCMAKE_CUDA_ARCHITECTURES=${CUDF_CMAKE_CUDA_ARCHITECTURES} \
           -DUSE_NVTX=${BUILD_NVTX} \
+          -DCUDF_USE_PROPRIETARY_NVCOMP=${USE_PROPRIETARY_NVCOMP} \
           -DBUILD_TESTS=${BUILD_TESTS} \
           -DBUILD_BENCHMARKS=${BUILD_BENCHMARKS} \
           -DDISABLE_DEPRECATION_WARNING=${BUILD_DISABLE_DEPRECATION_WARNING} \

conda/recipes/libcudf/meta.yaml

@@ -179,6 +179,7 @@ outputs:
         - test -f $PREFIX/include/cudf/lists/contains.hpp
         - test -f $PREFIX/include/cudf/lists/gather.hpp
         - test -f $PREFIX/include/cudf/lists/lists_column_view.hpp
+        - test -f $PREFIX/include/cudf/lists/list_view.hpp
         - test -f $PREFIX/include/cudf/lists/sorting.hpp
         - test -f $PREFIX/include/cudf/lists/stream_compaction.hpp
         - test -f $PREFIX/include/cudf/merge.hpp
@@ -280,6 +281,8 @@ outputs:
       license_family: Apache
       license_file: LICENSE
       summary: libcudf library
+      prelink_message:
+        - nvcomp.txt
   - name: libcudf_kafka
     version: {{ version }}
     script: install_libcudf_kafka.sh

conda/recipes/libcudf/nvcomp.txt

@@ -0,0 +1,3 @@
+By downloading and using the libcudf conda package, you accept the terms
+and conditions of the NVIDIA NVCOMP Software License Agreement:
+  https://developer.download.nvidia.com/compute/nvcomp/2.3/LICENSE.txt

conda/recipes/libcudf/post-link.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+# Copyright (c) 2022, NVIDIA CORPORATION.
+# Only add the license notice to libcudf and not our examples / tests
+if [[ "$PKG_NAME" == "libcudf" ]]; then
+  cat ./nvlink.txt>> $PREFIX/.messages.txt
+fi

cpp/CMakeLists.txt

@@ -42,6 +42,7 @@ option(BUILD_TESTS "Configure CMake to build tests" ON)
 option(BUILD_BENCHMARKS "Configure CMake to build (google & nvbench) benchmarks" OFF)
 option(BUILD_SHARED_LIBS "Build cuDF shared libraries" ON)
 option(JITIFY_USE_CACHE "Use a file cache for JIT compiled kernels" ON)
+option(CUDF_USE_PROPRIETARY_NVCOMP "Download and use NVCOMP with proprietary extensions" ON)
 option(CUDF_USE_ARROW_STATIC "Build and statically link Arrow libraries" OFF)
 option(CUDF_ENABLE_ARROW_ORC "Build the Arrow ORC adapter" OFF)
 option(CUDF_ENABLE_ARROW_PYTHON "Find (or build) Arrow with Python support" OFF)

cpp/cmake/thirdparty/get_nvcomp.cmake

@@ -13,33 +13,19 @@
 # =============================================================================
 
 # This function finds nvcomp and sets any additional necessary environment variables.
-function(find_and_configure_nvcomp VERSION_MIN VERSION_MAX)
-  # Search for latest version of nvComp
-  rapids_find_package(nvcomp ${VERSION_MAX} QUIET)
-  # If latest isn't found, fall back to building oldest support from source
-  rapids_cpm_find(
-    nvcomp ${VERSION_MIN}
-    GLOBAL_TARGETS nvcomp::nvcomp
-    CPM_ARGS GITHUB_REPOSITORY NVIDIA/nvcomp
-    GIT_TAG v${VERSION_MIN}
-    OPTIONS "BUILD_STATIC ON" "BUILD_TESTS OFF" "BUILD_BENCHMARKS OFF" "BUILD_EXAMPLES OFF"
-  )
-
-  if(nvcomp_BINARY_DIR)
-    include("${rapids-cmake-dir}/export/find_package_root.cmake")
-    rapids_export_find_package_root(BUILD nvcomp "${nvcomp_BINARY_DIR}" cudf-exports)
-  endif()
+function(find_and_configure_nvcomp)
 
-  if(NOT TARGET nvcomp::nvcomp)
-    add_library(nvcomp::nvcomp ALIAS nvcomp)
-  endif()
+  include(${rapids-cmake-dir}/cpm/nvcomp.cmake)
+  rapids_cpm_nvcomp(
+    BUILD_EXPORT_SET cudf-exports
+    INSTALL_EXPORT_SET cudf-exports
+    USE_PROPRIETARY_BINARY ${CUDF_USE_PROPRIETARY_NVCOMP}
+  )
 
-  # Per-thread default stream
+  # If we are building nvcomp use enable per-thread default stream
   if(TARGET nvcomp AND PER_THREAD_DEFAULT_STREAM)
     target_compile_definitions(nvcomp PRIVATE CUDA_API_PER_THREAD_DEFAULT_STREAM)
   endif()
 endfunction()
 
-set(CUDF_MIN_VERSION_nvCOMP 2.2.0)
-set(CUDF_MAX_VERSION_nvCOMP 2.3.0)
-find_and_configure_nvcomp(${CUDF_MIN_VERSION_nvCOMP} ${CUDF_MAX_VERSION_nvCOMP})
+find_and_configure_nvcomp()

fetch_rapids.cmake

@@ -11,6 +11,7 @@
 # or implied. See the License for the specific language governing permissions and limitations under
 # the License.
 # =============================================================================
+include(FetchContent)
 file(DOWNLOAD https://raw.githubusercontent.com/rapidsai/rapids-cmake/branch-22.06/RAPIDS.cmake
      ${CMAKE_BINARY_DIR}/RAPIDS.cmake
 )

java/pom.xml

@@ -507,6 +507,9 @@
                                     <includes>
                                         <include>libcudfjni.so</include>
                                         <include>libcufilejni.so</include>
+                                        <include>libnvcomp.so</include>
+                                        <include>libnvcomp_gdeflate.so</include>
+                                        <include>libnvcomp_bitcomp.so</include>
                                     </includes>
                                 </resource>
                                 <resource>

java/src/main/java/ai/rapids/cudf/NativeDepsLoader.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019-2021, NVIDIA CORPORATION.
+ * Copyright (c) 2019-2022, NVIDIA CORPORATION.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -54,6 +54,12 @@ public class NativeDepsLoader {
    * subsequent stages are loaded.
    */
   private static final String[][] loadOrder = new String[][]{
+      new String[]{
+          "nvcomp_bitcomp", "nvcomp_gdeflate"
+      },
+      new String[]{
+          "nvcomp"
+      },
       new String[]{
           "cudf"
       },

java/src/main/native/CMakeLists.txt

@@ -90,18 +90,20 @@ endif()
 #   Set a default build type if none was specified
 rapids_cmake_build_type("Release")
 
-# ##################################################################################################
-# * nvcomp------------------------------------------------------------------------------------------
-
-set(nvcomp_DIR "${CUDF_CPP_BUILD_DIR}/_deps/nvcomp-build")
-rapids_find_package(nvcomp REQUIRED)
-
 # ##################################################################################################
 # * CUDF ------------------------------------------------------------------------------------------
 
 set(cudf_ROOT "${CUDF_CPP_BUILD_DIR}")
 rapids_find_package(cudf REQUIRED)
 
+# ##################################################################################################
+# * nvcomp------------------------------------------------------------------------------------------
+
+if(NOT DEFINED nvcomp_DIR)
+  set(nvcomp_DIR "${CUDF_CPP_BUILD_DIR}/_deps/nvcomp-build")
+endif()
+rapids_find_package(nvcomp REQUIRED)
+
 # ##################################################################################################
 # * find JNI -------------------------------------------------------------------------------------
 find_package(JNI REQUIRED)
@@ -240,3 +242,16 @@ else()
   # Tell CMake what CUDA language runtime to use
   set_target_properties(cudfjni PROPERTIES CUDA_RUNTIME_LIBRARY Shared)
 endif()
+
+# ##################################################################################################
+# * install shared libraries ----------------------------------------------------------------------
+if(TARGET nvcomp::nvcomp)
+  add_custom_command(
+    TARGET cudfjni
+    PRE_LINK
+    COMMAND
+      ${CMAKE_COMMAND} -E copy $<TARGET_FILE:nvcomp::nvcomp> $<TARGET_FILE:nvcomp::nvcomp_gdeflate>
+      $<TARGET_FILE:nvcomp::nvcomp_bitcomp> "${PROJECT_BINARY_DIR}"
+    COMMENT "Copying nvcomp libraries to ${PROJECT_BINARY_DIR}"
+  )
+endif()