From 168af0500c01c41b9a37f9779c68416ad8f84169 Mon Sep 17 00:00:00 2001 From: Jim Brennan Date: Tue, 10 May 2022 08:41:57 -0500 Subject: [PATCH 01/19] Add support for nvcomp-2.3 in java build --- java/pom.xml | 3 +++ .../java/ai/rapids/cudf/NativeDepsLoader.java | 6 ++++++ java/src/main/native/CMakeLists.txt | 15 +++++++++++++++ 3 files changed, 24 insertions(+) diff --git a/java/pom.xml b/java/pom.xml index 31a79ec9801..4e915b0f906 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -507,6 +507,9 @@ libcudfjni.so libcufilejni.so + libnvcomp.so + libnvcomp_gdeflate.so + libnvcomp_bitcomp.so diff --git a/java/src/main/java/ai/rapids/cudf/NativeDepsLoader.java b/java/src/main/java/ai/rapids/cudf/NativeDepsLoader.java index 9663fbcafb4..9f07e6d4520 100755 --- a/java/src/main/java/ai/rapids/cudf/NativeDepsLoader.java +++ b/java/src/main/java/ai/rapids/cudf/NativeDepsLoader.java @@ -54,6 +54,12 @@ public class NativeDepsLoader { * subsequent stages are loaded. */ private static final String[][] loadOrder = new String[][]{ + new String[]{ + "nvcomp_bitcomp", "nvcomp_gdeflate" + }, + new String[]{ + "nvcomp" + }, new String[]{ "cudf" }, diff --git a/java/src/main/native/CMakeLists.txt b/java/src/main/native/CMakeLists.txt index 3a375412bbd..e1631c327e8 100755 --- a/java/src/main/native/CMakeLists.txt +++ b/java/src/main/native/CMakeLists.txt @@ -240,3 +240,18 @@ else() # Tell CMake what CUDA language runtime to use set_target_properties(cudfjni PROPERTIES CUDA_RUNTIME_LIBRARY Shared) endif() + +# ################################################################################################## +# * install shared libraries ---------------------------------------------------------------------- +if(TARGET nvcomp::nvcomp) + get_target_property(NVCOMP_LOCATION nvcomp::nvcomp LOCATION) + get_filename_component(NVCOMP_LIB_PATH "${NVCOMP_LOCATION}" DIRECTORY) + + add_custom_command( + TARGET cudfjni PRE_LINK COMMAND ${CMAKE_COMMAND} -E copy + "${NVCOMP_LIB_PATH}/libnvcomp.so" + "${NVCOMP_LIB_PATH}/libnvcomp_gdeflate.so" + "${NVCOMP_LIB_PATH}/libnvcomp_bitcomp.so" + "${PROJECT_BINARY_DIR}" + ) +endif() From f2f9ad93f46c9c06cdef7d007174b93dd4b92dcd Mon Sep 17 00:00:00 2001 From: Jim Brennan Date: Thu, 12 May 2022 09:32:03 -0500 Subject: [PATCH 02/19] Update copyrights. --- java/src/main/java/ai/rapids/cudf/NativeDepsLoader.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java/src/main/java/ai/rapids/cudf/NativeDepsLoader.java b/java/src/main/java/ai/rapids/cudf/NativeDepsLoader.java index 9f07e6d4520..27322cca436 100755 --- a/java/src/main/java/ai/rapids/cudf/NativeDepsLoader.java +++ b/java/src/main/java/ai/rapids/cudf/NativeDepsLoader.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2021, NVIDIA CORPORATION. + * Copyright (c) 2019-2022, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. From 7f5d453de9f56eed3b7c09e7a933fd663aba8815 Mon Sep 17 00:00:00 2001 From: Jim Brennan Date: Thu, 12 May 2022 09:49:15 -0500 Subject: [PATCH 03/19] Use $ to get the full path --- java/src/main/native/CMakeLists.txt | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/java/src/main/native/CMakeLists.txt b/java/src/main/native/CMakeLists.txt index e1631c327e8..44b63a5b4f2 100755 --- a/java/src/main/native/CMakeLists.txt +++ b/java/src/main/native/CMakeLists.txt @@ -244,14 +244,11 @@ endif() # ################################################################################################## # * install shared libraries ---------------------------------------------------------------------- if(TARGET nvcomp::nvcomp) - get_target_property(NVCOMP_LOCATION nvcomp::nvcomp LOCATION) - get_filename_component(NVCOMP_LIB_PATH "${NVCOMP_LOCATION}" DIRECTORY) - add_custom_command( TARGET cudfjni PRE_LINK COMMAND ${CMAKE_COMMAND} -E copy - "${NVCOMP_LIB_PATH}/libnvcomp.so" - "${NVCOMP_LIB_PATH}/libnvcomp_gdeflate.so" - "${NVCOMP_LIB_PATH}/libnvcomp_bitcomp.so" + $ + $ + $ "${PROJECT_BINARY_DIR}" ) endif() From d8b8c89b632ec990cbeb12bf0e35094fe2f8bc57 Mon Sep 17 00:00:00 2001 From: Jim Brennan Date: Thu, 12 May 2022 14:49:45 -0500 Subject: [PATCH 04/19] Add COMMENT to custom command to fix checkstyle issue --- java/src/main/native/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/java/src/main/native/CMakeLists.txt b/java/src/main/native/CMakeLists.txt index 44b63a5b4f2..a465d48f5c1 100755 --- a/java/src/main/native/CMakeLists.txt +++ b/java/src/main/native/CMakeLists.txt @@ -250,5 +250,6 @@ if(TARGET nvcomp::nvcomp) $ $ "${PROJECT_BINARY_DIR}" + COMMENT "Copying nvcomp libraries to ${PROJECT_BINARY_DIR}" ) endif() From eb43115027ad23471d7534b11bc9fa69e2750edd Mon Sep 17 00:00:00 2001 From: Robert Maynard Date: Fri, 13 May 2022 12:22:22 -0400 Subject: [PATCH 05/19] Use pre-built nvcomp 2.3 binaries by default When building cudf on x86_64 hardware we can enable the new pre-built nvcomp 2.3 binaries to leverage new compression codec support. --- cpp/cmake/thirdparty/get_nvcomp.cmake | 30 +++++++-------------------- fetch_rapids.cmake | 7 +++++++ 2 files changed, 14 insertions(+), 23 deletions(-) diff --git a/cpp/cmake/thirdparty/get_nvcomp.cmake b/cpp/cmake/thirdparty/get_nvcomp.cmake index d0007f93628..cafc78ad546 100644 --- a/cpp/cmake/thirdparty/get_nvcomp.cmake +++ b/cpp/cmake/thirdparty/get_nvcomp.cmake @@ -13,33 +13,17 @@ # ============================================================================= # This function finds nvcomp and sets any additional necessary environment variables. -function(find_and_configure_nvcomp VERSION_MIN VERSION_MAX) - # Search for latest version of nvComp - rapids_find_package(nvcomp ${VERSION_MAX} QUIET) - # If latest isn't found, fall back to building oldest support from source - rapids_cpm_find( - nvcomp ${VERSION_MIN} - GLOBAL_TARGETS nvcomp::nvcomp - CPM_ARGS GITHUB_REPOSITORY NVIDIA/nvcomp - GIT_TAG v${VERSION_MIN} - OPTIONS "BUILD_STATIC ON" "BUILD_TESTS OFF" "BUILD_BENCHMARKS OFF" "BUILD_EXAMPLES OFF" - ) +function(find_and_configure_nvcomp ) - if(nvcomp_BINARY_DIR) - include("${rapids-cmake-dir}/export/find_package_root.cmake") - rapids_export_find_package_root(BUILD nvcomp "${nvcomp_BINARY_DIR}" cudf-exports) - endif() - - if(NOT TARGET nvcomp::nvcomp) - add_library(nvcomp::nvcomp ALIAS nvcomp) - endif() + include(${rapids-cmake-dir}/cpm/nvcomp.cmake) + rapids_cpm_nvcomp(BUILD_EXPORT_SET cudf-exports + INSTALL_EXPORT_SET cudf-exports + USE_PROPRIETARY_BINARY ON) - # Per-thread default stream + # If we are building nvcomp use enable per-thread default stream if(TARGET nvcomp AND PER_THREAD_DEFAULT_STREAM) target_compile_definitions(nvcomp PRIVATE CUDA_API_PER_THREAD_DEFAULT_STREAM) endif() endfunction() -set(CUDF_MIN_VERSION_nvCOMP 2.2.0) -set(CUDF_MAX_VERSION_nvCOMP 2.3.0) -find_and_configure_nvcomp(${CUDF_MIN_VERSION_nvCOMP} ${CUDF_MAX_VERSION_nvCOMP}) +find_and_configure_nvcomp() diff --git a/fetch_rapids.cmake b/fetch_rapids.cmake index 2382abe38a9..6f0d355c945 100644 --- a/fetch_rapids.cmake +++ b/fetch_rapids.cmake @@ -11,6 +11,13 @@ # or implied. See the License for the specific language governing permissions and limitations under # the License. # ============================================================================= +include(FetchContent) +FetchContent_Declare( + rapids-cmake + GIT_REPOSITORY https://github.com/robertmaynard/rapids-cmake.git + GIT_TAG add_cpm_nvcomp +) + file(DOWNLOAD https://raw.githubusercontent.com/rapidsai/rapids-cmake/branch-22.06/RAPIDS.cmake ${CMAKE_BINARY_DIR}/RAPIDS.cmake ) From a5b47611dbeb49c317926826e32ec65c8a98aecf Mon Sep 17 00:00:00 2001 From: Robert Maynard Date: Fri, 13 May 2022 13:06:54 -0400 Subject: [PATCH 06/19] fix style issues found by ci --- cpp/cmake/thirdparty/get_nvcomp.cmake | 8 ++++---- fetch_rapids.cmake | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/cpp/cmake/thirdparty/get_nvcomp.cmake b/cpp/cmake/thirdparty/get_nvcomp.cmake index cafc78ad546..8c4494b0534 100644 --- a/cpp/cmake/thirdparty/get_nvcomp.cmake +++ b/cpp/cmake/thirdparty/get_nvcomp.cmake @@ -13,12 +13,12 @@ # ============================================================================= # This function finds nvcomp and sets any additional necessary environment variables. -function(find_and_configure_nvcomp ) +function(find_and_configure_nvcomp) include(${rapids-cmake-dir}/cpm/nvcomp.cmake) - rapids_cpm_nvcomp(BUILD_EXPORT_SET cudf-exports - INSTALL_EXPORT_SET cudf-exports - USE_PROPRIETARY_BINARY ON) + rapids_cpm_nvcomp( + BUILD_EXPORT_SET cudf-exports INSTALL_EXPORT_SET cudf-exports USE_PROPRIETARY_BINARY ON + ) # If we are building nvcomp use enable per-thread default stream if(TARGET nvcomp AND PER_THREAD_DEFAULT_STREAM) diff --git a/fetch_rapids.cmake b/fetch_rapids.cmake index 6f0d355c945..da52361fb0c 100644 --- a/fetch_rapids.cmake +++ b/fetch_rapids.cmake @@ -13,9 +13,9 @@ # ============================================================================= include(FetchContent) FetchContent_Declare( - rapids-cmake - GIT_REPOSITORY https://github.com/robertmaynard/rapids-cmake.git - GIT_TAG add_cpm_nvcomp + rapids-cmake + GIT_REPOSITORY https://github.com/robertmaynard/rapids-cmake.git + GIT_TAG add_cpm_nvcomp ) file(DOWNLOAD https://raw.githubusercontent.com/rapidsai/rapids-cmake/branch-22.06/RAPIDS.cmake From c1e5275c93b4ed6ef8423f840d44f5053ec2a2f2 Mon Sep 17 00:00:00 2001 From: Robert Maynard Date: Fri, 13 May 2022 13:29:30 -0400 Subject: [PATCH 07/19] Update java side to use nvcomp from cudf --- java/src/main/native/CMakeLists.txt | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/java/src/main/native/CMakeLists.txt b/java/src/main/native/CMakeLists.txt index 3a375412bbd..5b5e58f7361 100755 --- a/java/src/main/native/CMakeLists.txt +++ b/java/src/main/native/CMakeLists.txt @@ -90,18 +90,20 @@ endif() # Set a default build type if none was specified rapids_cmake_build_type("Release") -# ################################################################################################## -# * nvcomp------------------------------------------------------------------------------------------ - -set(nvcomp_DIR "${CUDF_CPP_BUILD_DIR}/_deps/nvcomp-build") -rapids_find_package(nvcomp REQUIRED) - # ################################################################################################## # * CUDF ------------------------------------------------------------------------------------------ set(cudf_ROOT "${CUDF_CPP_BUILD_DIR}") rapids_find_package(cudf REQUIRED) +# ################################################################################################## +# * nvcomp------------------------------------------------------------------------------------------ + +if(NOT DEFINED nvcomp_DIR) + set(nvcomp_DIR "${CUDF_CPP_BUILD_DIR}/_deps/nvcomp-build") +endif() +rapids_find_package(nvcomp REQUIRED) + # ################################################################################################## # * find JNI ------------------------------------------------------------------------------------- find_package(JNI REQUIRED) From d1ad96b8862f0700d4f4c5323c8c406743dd6290 Mon Sep 17 00:00:00 2001 From: Robert Maynard Date: Mon, 16 May 2022 11:45:49 -0400 Subject: [PATCH 08/19] Correct style issues found by CI --- java/src/main/native/CMakeLists.txt | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/java/src/main/native/CMakeLists.txt b/java/src/main/native/CMakeLists.txt index d7f2c60321a..0c8cf417582 100755 --- a/java/src/main/native/CMakeLists.txt +++ b/java/src/main/native/CMakeLists.txt @@ -247,11 +247,11 @@ endif() # * install shared libraries ---------------------------------------------------------------------- if(TARGET nvcomp::nvcomp) add_custom_command( - TARGET cudfjni PRE_LINK COMMAND ${CMAKE_COMMAND} -E copy - $ - $ - $ - "${PROJECT_BINARY_DIR}" + TARGET cudfjni + PRE_LINK + COMMAND + ${CMAKE_COMMAND} -E copy $ $ + $ "${PROJECT_BINARY_DIR}" COMMENT "Copying nvcomp libraries to ${PROJECT_BINARY_DIR}" ) endif() From f0667a81de1bddcfb6340e92f4c6d6fb77b285a6 Mon Sep 17 00:00:00 2001 From: Robert Maynard Date: Mon, 16 May 2022 13:19:53 -0400 Subject: [PATCH 09/19] Add build.sh option to control using proprietary nvcomp binaries --- build.sh | 9 ++++++++- cpp/CMakeLists.txt | 1 + cpp/cmake/thirdparty/get_nvcomp.cmake | 2 +- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/build.sh b/build.sh index ab3bd0e7a89..8f40aaee8ca 100755 --- a/build.sh +++ b/build.sh @@ -17,7 +17,7 @@ ARGS=$* # script, and that this script resides in the repo dir! REPODIR=$(cd $(dirname $0); pwd) -VALIDARGS="clean libcudf cudf cudfjar dask_cudf benchmarks tests libcudf_kafka cudf_kafka custreamz -v -g -n -l --allgpuarch --disable_nvtx --show_depr_warn --ptds -h --build_metrics --incl_cache_stats" +VALIDARGS="clean libcudf cudf cudfjar dask_cudf benchmarks tests libcudf_kafka cudf_kafka custreamz -v -g -n -l --allgpuarch --disable_nvtx --opensource_nvcomp --show_depr_warn --ptds -h --build_metrics --incl_cache_stats" HELP="$0 [clean] [libcudf] [cudf] [cudfjar] [dask_cudf] [benchmarks] [tests] [libcudf_kafka] [cudf_kafka] [custreamz] [-v] [-g] [-n] [-h] [--cmake-args=\\\"\\\"] clean - remove all existing build artifacts and configuration (start over) @@ -35,6 +35,7 @@ HELP="$0 [clean] [libcudf] [cudf] [cudfjar] [dask_cudf] [benchmarks] [tests] [li -n - no install step --allgpuarch - build for all supported GPU architectures --disable_nvtx - disable inserting NVTX profiling ranges + --opensource_nvcomp - disable use of propretary nvcomp extensions --show_depr_warn - show cmake deprecation warnings --ptds - enable per-thread default stream --build_metrics - generate build metrics report for libcudf @@ -67,6 +68,7 @@ BUILD_DISABLE_DEPRECATION_WARNING=ON BUILD_PER_THREAD_DEFAULT_STREAM=OFF BUILD_REPORT_METRICS=OFF BUILD_REPORT_INCL_CACHE_STATS=OFF +USE_PROPRIETARY_NVCOMP=ON # Set defaults for vars that may not have been defined externally # FIXME: if INSTALL_PREFIX is not set, check PREFIX, then check @@ -144,6 +146,7 @@ function buildLibCudfJniInDocker { -DCMAKE_CUDA_ARCHITECTURES=${CUDF_CMAKE_CUDA_ARCHITECTURES} \ -DCMAKE_INSTALL_PREFIX=/usr/local/rapids \ -DUSE_NVTX=ON \ + -DCMAKE_USE_PROPRIETARY_NVCOMP=ON \ -DCUDF_USE_ARROW_STATIC=ON \ -DCUDF_ENABLE_ARROW_S3=OFF \ -DBUILD_TESTS=OFF \ @@ -209,6 +212,9 @@ fi if hasArg --disable_nvtx; then BUILD_NVTX="OFF" fi +if hasArg --opensource_nvcomp; then + USE_PROPRIETARY_NVCOMP="OFF" +fi if hasArg --show_depr_warn; then BUILD_DISABLE_DEPRECATION_WARNING=OFF fi @@ -271,6 +277,7 @@ if buildAll || hasArg libcudf; then -DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX} \ -DCMAKE_CUDA_ARCHITECTURES=${CUDF_CMAKE_CUDA_ARCHITECTURES} \ -DUSE_NVTX=${BUILD_NVTX} \ + -DCUDF_USE_PROPRIETARY_NVCOMP=${USE_PROPRIETARY_NVCOMP} \ -DBUILD_TESTS=${BUILD_TESTS} \ -DBUILD_BENCHMARKS=${BUILD_BENCHMARKS} \ -DDISABLE_DEPRECATION_WARNING=${BUILD_DISABLE_DEPRECATION_WARNING} \ diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 6a08637dc11..76cf3bdf210 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -42,6 +42,7 @@ option(BUILD_TESTS "Configure CMake to build tests" ON) option(BUILD_BENCHMARKS "Configure CMake to build (google & nvbench) benchmarks" OFF) option(BUILD_SHARED_LIBS "Build cuDF shared libraries" ON) option(JITIFY_USE_CACHE "Use a file cache for JIT compiled kernels" ON) +option(CUDF_USE_PROPRIETARY_NVCOMP "Download and use NVCOMP with proprietary extensions" ON) option(CUDF_USE_ARROW_STATIC "Build and statically link Arrow libraries" OFF) option(CUDF_ENABLE_ARROW_ORC "Build the Arrow ORC adapter" OFF) option(CUDF_ENABLE_ARROW_PYTHON "Find (or build) Arrow with Python support" OFF) diff --git a/cpp/cmake/thirdparty/get_nvcomp.cmake b/cpp/cmake/thirdparty/get_nvcomp.cmake index 8c4494b0534..d15cb721673 100644 --- a/cpp/cmake/thirdparty/get_nvcomp.cmake +++ b/cpp/cmake/thirdparty/get_nvcomp.cmake @@ -17,7 +17,7 @@ function(find_and_configure_nvcomp) include(${rapids-cmake-dir}/cpm/nvcomp.cmake) rapids_cpm_nvcomp( - BUILD_EXPORT_SET cudf-exports INSTALL_EXPORT_SET cudf-exports USE_PROPRIETARY_BINARY ON + BUILD_EXPORT_SET cudf-exports INSTALL_EXPORT_SET cudf-exports USE_PROPRIETARY_BINARY ${CUDF_USE_PROPRIETARY_NVCOMP} ) # If we are building nvcomp use enable per-thread default stream From c64d4d9306b0ac40f063d840f10dee176abafb29 Mon Sep 17 00:00:00 2001 From: Robert Maynard Date: Mon, 16 May 2022 13:42:25 -0400 Subject: [PATCH 10/19] Add conda license agreement --- conda/recipes/libcudf/meta.yaml | 2 ++ conda/recipes/libcudf/nvlink.txt | 3 +++ 2 files changed, 5 insertions(+) create mode 100644 conda/recipes/libcudf/nvlink.txt diff --git a/conda/recipes/libcudf/meta.yaml b/conda/recipes/libcudf/meta.yaml index c8b5a9d373b..d8e6d8918ac 100644 --- a/conda/recipes/libcudf/meta.yaml +++ b/conda/recipes/libcudf/meta.yaml @@ -280,6 +280,8 @@ outputs: license_family: Apache license_file: LICENSE summary: libcudf library + prelink_message: + - nvcomp.txt - name: libcudf_kafka version: {{ version }} script: install_libcudf_kafka.sh diff --git a/conda/recipes/libcudf/nvlink.txt b/conda/recipes/libcudf/nvlink.txt new file mode 100644 index 00000000000..9a0047e71fa --- /dev/null +++ b/conda/recipes/libcudf/nvlink.txt @@ -0,0 +1,3 @@ +By downloading and using the libcudf conda package, you accept the terms +and conditions of the NVIDIA NVCOMP Software License Agreement: + https://developer.download.nvidia.com/compute/nvcomp/2.3/LICENSE.txt From db7a4376c9c5e94ad0cd365d177ff643feab71c6 Mon Sep 17 00:00:00 2001 From: Robert Maynard Date: Mon, 16 May 2022 13:42:25 -0400 Subject: [PATCH 11/19] Add conda license agreement --- conda/recipes/libcudf/post-link.sh | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 conda/recipes/libcudf/post-link.sh diff --git a/conda/recipes/libcudf/post-link.sh b/conda/recipes/libcudf/post-link.sh new file mode 100644 index 00000000000..49849a206a6 --- /dev/null +++ b/conda/recipes/libcudf/post-link.sh @@ -0,0 +1,6 @@ +#!/bin/bash +# Copyright (c) 2022, NVIDIA CORPORATION. +# Only add the license notice to libcudf and not our examples / tests +if [[ "$PKG_NAME" == "libcudf" ]]; then + cat ./nvlink.txt>> $PREFIX/.messages.txt +fi From 0bb4bba4e09e17a1f0b334c3404e1f8dc226eff3 Mon Sep 17 00:00:00 2001 From: Robert Maynard Date: Mon, 16 May 2022 15:56:01 -0400 Subject: [PATCH 12/19] Update build.sh Co-authored-by: Jason Lowe --- build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sh b/build.sh index 8f40aaee8ca..90e7eba9483 100755 --- a/build.sh +++ b/build.sh @@ -35,7 +35,7 @@ HELP="$0 [clean] [libcudf] [cudf] [cudfjar] [dask_cudf] [benchmarks] [tests] [li -n - no install step --allgpuarch - build for all supported GPU architectures --disable_nvtx - disable inserting NVTX profiling ranges - --opensource_nvcomp - disable use of propretary nvcomp extensions + --opensource_nvcomp - disable use of proprietary nvcomp extensions --show_depr_warn - show cmake deprecation warnings --ptds - enable per-thread default stream --build_metrics - generate build metrics report for libcudf From cfd4e4b0c96ceeada17e0a4a6e32d9962a12a7a0 Mon Sep 17 00:00:00 2001 From: Robert Maynard Date: Mon, 16 May 2022 16:19:21 -0400 Subject: [PATCH 13/19] Use rapids-cmake 22.06 --- fetch_rapids.cmake | 6 ------ 1 file changed, 6 deletions(-) diff --git a/fetch_rapids.cmake b/fetch_rapids.cmake index da52361fb0c..e7653b946ca 100644 --- a/fetch_rapids.cmake +++ b/fetch_rapids.cmake @@ -12,12 +12,6 @@ # the License. # ============================================================================= include(FetchContent) -FetchContent_Declare( - rapids-cmake - GIT_REPOSITORY https://github.com/robertmaynard/rapids-cmake.git - GIT_TAG add_cpm_nvcomp -) - file(DOWNLOAD https://raw.githubusercontent.com/rapidsai/rapids-cmake/branch-22.06/RAPIDS.cmake ${CMAKE_BINARY_DIR}/RAPIDS.cmake ) From 6ac5a9cf330a161c4cca055dfa6db4cbf5d71110 Mon Sep 17 00:00:00 2001 From: Robert Maynard Date: Mon, 16 May 2022 16:31:50 -0400 Subject: [PATCH 14/19] Fix style issues found by ci --- cpp/cmake/thirdparty/get_nvcomp.cmake | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cpp/cmake/thirdparty/get_nvcomp.cmake b/cpp/cmake/thirdparty/get_nvcomp.cmake index d15cb721673..ddb1d7d1ffe 100644 --- a/cpp/cmake/thirdparty/get_nvcomp.cmake +++ b/cpp/cmake/thirdparty/get_nvcomp.cmake @@ -17,7 +17,8 @@ function(find_and_configure_nvcomp) include(${rapids-cmake-dir}/cpm/nvcomp.cmake) rapids_cpm_nvcomp( - BUILD_EXPORT_SET cudf-exports INSTALL_EXPORT_SET cudf-exports USE_PROPRIETARY_BINARY ${CUDF_USE_PROPRIETARY_NVCOMP} + BUILD_EXPORT_SET cudf-exports INSTALL_EXPORT_SET cudf-exports USE_PROPRIETARY_BINARY + ${CUDF_USE_PROPRIETARY_NVCOMP} ) # If we are building nvcomp use enable per-thread default stream From 437ba413b1c45d619101dac040baf21f264e51aa Mon Sep 17 00:00:00 2001 From: Robert Maynard Date: Mon, 16 May 2022 17:21:12 -0400 Subject: [PATCH 15/19] Adding missing entry to meta.yml --- conda/recipes/libcudf/meta.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/conda/recipes/libcudf/meta.yaml b/conda/recipes/libcudf/meta.yaml index d8e6d8918ac..143d7b96b63 100644 --- a/conda/recipes/libcudf/meta.yaml +++ b/conda/recipes/libcudf/meta.yaml @@ -179,6 +179,7 @@ outputs: - test -f $PREFIX/include/cudf/lists/contains.hpp - test -f $PREFIX/include/cudf/lists/gather.hpp - test -f $PREFIX/include/cudf/lists/lists_column_view.hpp + - test -f $PREFIX/include/cudf/lists/lists_view.hpp - test -f $PREFIX/include/cudf/lists/sorting.hpp - test -f $PREFIX/include/cudf/lists/stream_compaction.hpp - test -f $PREFIX/include/cudf/merge.hpp From 7f514bfcf0c0cec9ecb160a44bf41b511aa43557 Mon Sep 17 00:00:00 2001 From: Robert Maynard Date: Mon, 16 May 2022 17:27:32 -0400 Subject: [PATCH 16/19] Adding missing entry to meta.yml --- conda/recipes/libcudf/meta.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda/recipes/libcudf/meta.yaml b/conda/recipes/libcudf/meta.yaml index 143d7b96b63..d27d27ce22d 100644 --- a/conda/recipes/libcudf/meta.yaml +++ b/conda/recipes/libcudf/meta.yaml @@ -179,7 +179,7 @@ outputs: - test -f $PREFIX/include/cudf/lists/contains.hpp - test -f $PREFIX/include/cudf/lists/gather.hpp - test -f $PREFIX/include/cudf/lists/lists_column_view.hpp - - test -f $PREFIX/include/cudf/lists/lists_view.hpp + - test -f $PREFIX/include/cudf/lists/list_view.hpp - test -f $PREFIX/include/cudf/lists/sorting.hpp - test -f $PREFIX/include/cudf/lists/stream_compaction.hpp - test -f $PREFIX/include/cudf/merge.hpp From 0403c065a9652e8079bea09c7dca6816604d0786 Mon Sep 17 00:00:00 2001 From: Robert Maynard Date: Mon, 16 May 2022 17:36:47 -0400 Subject: [PATCH 17/19] Fix style issues found by ci --- cpp/cmake/thirdparty/get_nvcomp.cmake | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/cpp/cmake/thirdparty/get_nvcomp.cmake b/cpp/cmake/thirdparty/get_nvcomp.cmake index ddb1d7d1ffe..a29e9bf25e3 100644 --- a/cpp/cmake/thirdparty/get_nvcomp.cmake +++ b/cpp/cmake/thirdparty/get_nvcomp.cmake @@ -17,8 +17,9 @@ function(find_and_configure_nvcomp) include(${rapids-cmake-dir}/cpm/nvcomp.cmake) rapids_cpm_nvcomp( - BUILD_EXPORT_SET cudf-exports INSTALL_EXPORT_SET cudf-exports USE_PROPRIETARY_BINARY - ${CUDF_USE_PROPRIETARY_NVCOMP} + BUILD_EXPORT_SET cudf-exports + INSTALL_EXPORT_SET cudf-exports + USE_PROPRIETARY_BINARY ${CUDF_USE_PROPRIETARY_NVCOMP} ) # If we are building nvcomp use enable per-thread default stream From 618d750d9b8f5c6f5c53c763c95233ff833c6c9b Mon Sep 17 00:00:00 2001 From: Robert Maynard Date: Wed, 18 May 2022 09:51:55 -0400 Subject: [PATCH 18/19] properly name nvcomp license file --- conda/recipes/libcudf/{nvlink.txt => nvcomp.txt} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename conda/recipes/libcudf/{nvlink.txt => nvcomp.txt} (100%) diff --git a/conda/recipes/libcudf/nvlink.txt b/conda/recipes/libcudf/nvcomp.txt similarity index 100% rename from conda/recipes/libcudf/nvlink.txt rename to conda/recipes/libcudf/nvcomp.txt From f531c11cc2a2ce6d93de51042ab1bdde6ae15c04 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Wed, 18 May 2022 11:30:29 -0700 Subject: [PATCH 19/19] Apply suggestions from code review --- build.sh | 2 +- conda/recipes/libcudf/post-link.sh | 2 +- fetch_rapids.cmake | 1 - 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/build.sh b/build.sh index 90e7eba9483..23e94e5a609 100755 --- a/build.sh +++ b/build.sh @@ -146,7 +146,7 @@ function buildLibCudfJniInDocker { -DCMAKE_CUDA_ARCHITECTURES=${CUDF_CMAKE_CUDA_ARCHITECTURES} \ -DCMAKE_INSTALL_PREFIX=/usr/local/rapids \ -DUSE_NVTX=ON \ - -DCMAKE_USE_PROPRIETARY_NVCOMP=ON \ + -DCUDF_USE_PROPRIETARY_NVCOMP=ON \ -DCUDF_USE_ARROW_STATIC=ON \ -DCUDF_ENABLE_ARROW_S3=OFF \ -DBUILD_TESTS=OFF \ diff --git a/conda/recipes/libcudf/post-link.sh b/conda/recipes/libcudf/post-link.sh index 49849a206a6..64e0b1ad305 100644 --- a/conda/recipes/libcudf/post-link.sh +++ b/conda/recipes/libcudf/post-link.sh @@ -2,5 +2,5 @@ # Copyright (c) 2022, NVIDIA CORPORATION. # Only add the license notice to libcudf and not our examples / tests if [[ "$PKG_NAME" == "libcudf" ]]; then - cat ./nvlink.txt>> $PREFIX/.messages.txt + cat ./nvlink.txt >> $PREFIX/.messages.txt fi diff --git a/fetch_rapids.cmake b/fetch_rapids.cmake index e7653b946ca..2382abe38a9 100644 --- a/fetch_rapids.cmake +++ b/fetch_rapids.cmake @@ -11,7 +11,6 @@ # or implied. See the License for the specific language governing permissions and limitations under # the License. # ============================================================================= -include(FetchContent) file(DOWNLOAD https://raw.githubusercontent.com/rapidsai/rapids-cmake/branch-22.06/RAPIDS.cmake ${CMAKE_BINARY_DIR}/RAPIDS.cmake )