From 9409559433ae55b9c44d68ef52d13b79885a8fde Mon Sep 17 00:00:00 2001 From: Jason Lowe Date: Mon, 18 Apr 2022 09:25:59 -0500 Subject: [PATCH] Rework JNI CMake to leverage rapids_find_package (#10649) The JNI CMakeLists.txt has been fragile, looking for specific .a or .so libraries and header file locations, and will break again when libcudf moves to a pre-installed nvcomp 2.3 package since it expects to find nvcomp in a very specific location today. This refactors the JNI CMakeLists.txt to leverage `rapids_find_package` to reuse the work performed in the libcudf build and also has the nice side-effect of avoiding redundant pulls and builds of the Thrust and RMM repositories that is happening today. Another side-effect is that the JNI will now automatically pull in the same RMM compile definitions that are used for libcudf, meaning the separate RMM logging flag for the JNI build is no longer necessary. Similarly it's no longer necessary to explicitly specify to the JNI build which type of Arrow library to use (i.e,.: static or dynamic), it will automatically use whichever Arrow library was built by libcudf. Authors: - Jason Lowe (https://github.com/jlowe) Approvers: - Robert (Bobby) Evans (https://github.com/revans2) - Gera Shegalov (https://github.com/gerashegalov) URL: https://github.com/rapidsai/cudf/pull/10649 --- build.sh | 2 - java/README.md | 10 --- java/ci/build-in-docker.sh | 1 - java/pom.xml | 4 - java/src/main/native/CMakeLists.txt | 132 ++++------------------------ 5 files changed, 17 insertions(+), 132 deletions(-) diff --git a/build.sh b/build.sh index e1d6df016dd..48182ca1a6f 100755 --- a/build.sh +++ b/build.sh @@ -148,10 +148,8 @@ function buildLibCudfJniInDocker { -DCUDF_CPP_BUILD_DIR=$workspaceRepoDir/java/target/libcudf-cmake-build \ -DCUDA_STATIC_RUNTIME=ON \ -DPER_THREAD_DEFAULT_STREAM=ON \ - -DRMM_LOGGING_LEVEL=OFF \ -DUSE_GDS=ON \ -DGPU_ARCHS=${CUDF_CMAKE_CUDA_ARCHITECTURES} \ - -DCUDF_JNI_ARROW_STATIC=ON \ -DCUDF_JNI_LIBCUDF_STATIC=ON \ -Dtest=*,!CuFileTest" } diff --git a/java/README.md b/java/README.md index afd69df11ef..ea1b9e3e4e4 100644 --- a/java/README.md +++ b/java/README.md @@ -75,16 +75,6 @@ If you decide to build without Docker and the build script, examining the cmake settings in the [Java CI build script](ci/build-in-docker.sh) can be helpful if you are encountering difficulties during the build. -## Dynamically Linking Arrow - -Since libcudf builds by default with a dynamically linked Arrow dependency, it may be -desirable to build the Java bindings without requiring a statically-linked Arrow to avoid -rebuilding an already built libcudf.so. To do so, specify the additional command-line flag -`-DCUDF_JNI_ARROW_STATIC=OFF` when building the Java bindings with Maven. However this will -result in a jar that requires the correct Arrow version to be available in the runtime -environment, and therefore is not recommended unless you are only performing local testing -within the libcudf build environment. - ## Statically Linking the CUDA Runtime If you use the default cmake options libcudart will be dynamically linked to libcudf and libcudfjni. diff --git a/java/ci/build-in-docker.sh b/java/ci/build-in-docker.sh index d6a193fbeaf..d21010ba30e 100755 --- a/java/ci/build-in-docker.sh +++ b/java/ci/build-in-docker.sh @@ -78,7 +78,6 @@ BUILD_ARG="-Dmaven.repo.local=\"$WORKSPACE/.m2\"\ -DPER_THREAD_DEFAULT_STREAM=$ENABLE_PTDS\ -DCUDA_STATIC_RUNTIME=$ENABLE_CUDA_STATIC_RUNTIME\ -DCUDF_JNI_LIBCUDF_STATIC=ON\ - -DRMM_LOGGING_LEVEL=$RMM_LOGGING_LEVEL\ -DUSE_GDS=$ENABLE_GDS -Dtest=*,!CuFileTest" if [ "$SIGN_FILE" == true ]; then diff --git a/java/pom.xml b/java/pom.xml index e2efed19636..50b6ca59440 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -165,10 +165,8 @@ OFF OFF OFF - INFO OFF ALL - ON OFF ${project.build.directory}/cmake-build 1.7.30 @@ -386,13 +384,11 @@ - - diff --git a/java/src/main/native/CMakeLists.txt b/java/src/main/native/CMakeLists.txt index 9851102d011..3a375412bbd 100755 --- a/java/src/main/native/CMakeLists.txt +++ b/java/src/main/native/CMakeLists.txt @@ -41,7 +41,6 @@ option(BUILD_TESTS "Configure CMake to build tests" ON) option(PER_THREAD_DEFAULT_STREAM "Build with per-thread default stream" OFF) option(CUDA_STATIC_RUNTIME "Statically link the CUDA runtime" OFF) option(USE_GDS "Build with GPUDirect Storage (GDS)/cuFile support" OFF) -option(CUDF_JNI_ARROW_STATIC "Statically link Arrow" ON) option(CUDF_JNI_LIBCUDF_STATIC "Link with libcudf.a" OFF) message(VERBOSE "CUDF_JNI: Build with NVTX support: ${USE_NVTX}") @@ -50,7 +49,6 @@ message(VERBOSE "CUDF_JNI: Configure CMake to build tests: ${BUILD_TESTS}") message(VERBOSE "CUDF_JNI: Build with per-thread default stream: ${PER_THREAD_DEFAULT_STREAM}") message(VERBOSE "CUDF_JNI: Statically link the CUDA runtime: ${CUDA_STATIC_RUNTIME}") message(VERBOSE "CUDF_JNI: Build with GPUDirect Storage support: ${USE_GDS}") -message(VERBOSE "CUDF_JNI: Build with static Arrow library: ${CUDF_JNI_ARROW_STATIC}") message(VERBOSE "CUDF_JNI: Link with libcudf statically: ${CUDF_JNI_LIBCUDF_STATIC}") set(CUDF_SOURCE_DIR "${PROJECT_SOURCE_DIR}/../../../../cpp") @@ -93,67 +91,16 @@ endif() rapids_cmake_build_type("Release") # ################################################################################################## -# * Thrust/CUB -# ------------------------------------------------------------------------------------ -include(${CUDF_SOURCE_DIR}/cmake/thirdparty/get_thrust.cmake) +# * nvcomp------------------------------------------------------------------------------------------ -# ################################################################################################## -# * CUDF ------------------------------------------------------------------------------------------ - -set(CUDF_INCLUDE "${PROJECT_SOURCE_DIR}/../../../../cpp/include" - "${PROJECT_SOURCE_DIR}/../../../../cpp/src/" -) - -set(CUDF_LIB_HINTS HINTS "$ENV{CUDF_ROOT}" "$ENV{CUDF_ROOT}/lib" "$ENV{CONDA_PREFIX}/lib" - "${CUDF_CPP_BUILD_DIR}" -) - -find_library(CUDF_LIB "cudf" REQUIRED HINTS ${CUDF_LIB_HINTS}) - -# ################################################################################################## -# * ZLIB ------------------------------------------------------------------------------------------ - -# find zlib -rapids_find_package(ZLIB REQUIRED) +set(nvcomp_DIR "${CUDF_CPP_BUILD_DIR}/_deps/nvcomp-build") +rapids_find_package(nvcomp REQUIRED) # ################################################################################################## -# * RMM ------------------------------------------------------------------------------------------- +# * CUDF ------------------------------------------------------------------------------------------ -include(${CUDF_SOURCE_DIR}/cmake/thirdparty/get_rmm.cmake) - -# ################################################################################################## -# * ARROW ----------------------------------------------------------------------------------------- - -find_path(ARROW_INCLUDE "arrow" HINTS "$ENV{ARROW_ROOT}/include" - "${CUDF_CPP_BUILD_DIR}/_deps/arrow-src/cpp/src" -) - -message(STATUS "ARROW: ARROW_INCLUDE set to ${ARROW_INCLUDE}") - -if(CUDF_JNI_ARROW_STATIC) - # Find static version of Arrow lib - set(CUDF_JNI_ARROW_LIBNAME "libarrow.a") -else() - set(CUDF_JNI_ARROW_LIBNAME "arrow") -endif() - -find_library( - ARROW_LIBRARY ${CUDF_JNI_ARROW_LIBNAME} REQUIRED - HINTS "$ENV{ARROW_ROOT}/lib" "${CUDF_CPP_BUILD_DIR}/_deps/arrow-build/release" - "${CUDF_CPP_BUILD_DIR}/_deps/arrow-build/debug" -) - -if(NOT ARROW_LIBRARY) - if(CUDF_JNI_ARROW_STATIC) - message( - FATAL_ERROR "Arrow static library not found. Was libcudf built with CUDF_USE_ARROW_STATIC=ON?" - ) - else() - message(FATAL_ERROR "Arrow dynamic library not found.") - endif() -else() - message(STATUS "ARROW: ARROW_LIBRARY set to ${ARROW_LIBRARY}") -endif() +set(cudf_ROOT "${CUDF_CPP_BUILD_DIR}") +rapids_find_package(cudf REQUIRED) # ################################################################################################## # * find JNI ------------------------------------------------------------------------------------- @@ -164,27 +111,6 @@ else() message(FATAL_ERROR "JDK with JNI not found, please check your settings.") endif() -# ################################################################################################## -# * nvcomp ---------------------------------------------------------------------------------------- - -find_path(NVCOMP_INCLUDE "nvcomp" HINTS "${CUDF_CPP_BUILD_DIR}/_deps/nvcomp-src/include" - "$ENV{CONDA_PREFIX}/include" -) - -message(STATUS "NVCOMP: NVCOMP_INCLUDE set to ${NVCOMP_INCLUDE}") - -set(CUDF_JNI_NVCOMP_LIBNAME "libnvcomp.a") -find_library( - NVCOMP_LIBRARY ${CUDF_JNI_NVCOMP_LIBNAME} REQUIRED HINTS "${CUDF_CPP_BUILD_DIR}/lib" - "$ENV{CONDA_PREFIX}/lib" -) - -if(NOT NVCOMP_LIBRARY) - message(FATAL_ERROR "nvcomp static library not found.") -else() - message(STATUS "NVCOMP: NVCOMP_LIBRARY set to ${NVCOMP_LIBRARY}") -endif() - # ################################################################################################## # * GDS/cufile ------------------------------------------------------------------------------------ @@ -238,17 +164,8 @@ endif() # * include paths --------------------------------------------------------------------------------- target_include_directories( - cudfjni - PUBLIC cudf::Thrust - "${LIBCUDACXX_INCLUDE}" - "${CUDAToolkit_INCLUDE_DIRS}" - "${NVCOMP_INCLUDE}" - "${CMAKE_BINARY_DIR}/include" - "${CMAKE_SOURCE_DIR}/include" - "${CMAKE_SOURCE_DIR}/src" - "${JNI_INCLUDE_DIRS}" - "${CUDF_INCLUDE}" - "${ARROW_INCLUDE}" + cudfjni PUBLIC "${CMAKE_BINARY_DIR}/include" "${CMAKE_SOURCE_DIR}/include" + "${CMAKE_SOURCE_DIR}/src" "${JNI_INCLUDE_DIRS}" ) # ################################################################################################## @@ -291,39 +208,24 @@ if(USE_GDS) POSITION_INDEPENDENT_CODE ON INTERFACE_POSITION_INDEPENDENT_CODE ON ) - target_include_directories( - cufilejni - PUBLIC "${LIBCUDACXX_INCLUDE}" "${CUDF_INCLUDE}" - PRIVATE "${cuFile_INCLUDE_DIRS}" - ) - target_link_libraries(cufilejni PRIVATE cudfjni rmm::rmm "${cuFile_LIBRARIES}") + target_include_directories(cufilejni PRIVATE "${cuFile_INCLUDE_DIRS}") + target_link_libraries(cufilejni PRIVATE cudfjni "${cuFile_LIBRARIES}") endif() -# ################################################################################################## -# * rmm logging level ----------------------------------------------------------------------------- - -set(RMM_LOGGING_LEVEL - "INFO" - CACHE STRING "Choose the logging level." -) -# Set the possible values of build type for cmake-gui -set_property( - CACHE RMM_LOGGING_LEVEL PROPERTY STRINGS "TRACE" "DEBUG" "INFO" "WARN" "ERROR" "CRITICAL" "OFF" -) -message(STATUS "RMM_LOGGING_LEVEL = '${RMM_LOGGING_LEVEL}'.") - -target_compile_definitions(cudfjni PUBLIC SPDLOG_ACTIVE_LEVEL=SPDLOG_LEVEL_${RMM_LOGGING_LEVEL}) - # ################################################################################################## # * link libraries -------------------------------------------------------------------------------- -set(CUDF_LINK ${CUDF_LIB}) +set(CUDF_LINK PUBLIC cudf::cudf) if(CUDF_JNI_LIBCUDF_STATIC) - set(CUDF_LINK -Wl,--whole-archive ${CUDF_LIB} -Wl,--no-whole-archive ZLIB::ZLIB) + # Whole-link libcudf.a into the shared library but not its dependencies + set(CUDF_LINK PRIVATE -Wl,--whole-archive cudf::cudf -Wl,--no-whole-archive PUBLIC cudf::cudf) endif() +# When nvcomp is installed we need to use nvcomp::nvcomp but from the cudf build directory it will +# just be nvcomp. target_link_libraries( - cudfjni PRIVATE ${CUDF_LINK} ${NVCOMP_LIBRARY} ${ARROW_LIBRARY} rmm::rmm CUDA::cuda_driver + cudfjni ${CUDF_LINK} PRIVATE $ + $ ) # ##################################################################################################