Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix JNI build after removal of libcudf sub-libraries [skip ci] #7486

Merged
merged 2 commits into from
Mar 2, 2021
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 1 addition & 12 deletions java/pom.xml
Original file line number Diff line number Diff line change
@@ -518,18 +518,7 @@
<!--Set by groovy script-->
<directory>${native.cudf.path}</directory>
<includes>
<include>libcudf_ast.so</include>
<include>libcudf_base.so</include>
<include>libcudf_comms.so</include>
<include>libcudf_hash.so</include>
<include>libcudf_interop.so</include>
<include>libcudf_io.so</include>
<include>libcudf_join.so</include>
<include>libcudf_merge.so</include>
<include>libcudf_partitioning.so</include>
<include>libcudf_reductions.so</include>
<include>libcudf_replace.so</include>
<include>libcudf_rolling.so</include>
jlowe marked this conversation as resolved.
Show resolved Hide resolved
<include>libcudf.so</include>
</includes>
</resource>
<resource>
17 changes: 2 additions & 15 deletions java/src/main/java/ai/rapids/cudf/NativeDepsLoader.java
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2019-2020, NVIDIA CORPORATION.
* Copyright (c) 2019-2021, NVIDIA CORPORATION.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -56,20 +56,7 @@ public class NativeDepsLoader {
private static final String[][] loadOrder = new String[][]{
new String[]{
"nvcomp",
"cudf_base"
},
new String[]{
"cudf_ast",
"cudf_comms",
"cudf_hash",
"cudf_interop",
"cudf_io",
"cudf_join",
"cudf_merge",
"cudf_partitioning",
"cudf_reductions",
"cudf_replace",
"cudf_rolling"
"cudf"
},
new String[]{
"cudfjni"
140 changes: 64 additions & 76 deletions java/src/main/native/CMakeLists.txt
Original file line number Diff line number Diff line change
@@ -13,10 +13,11 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#=============================================================================
cmake_minimum_required(VERSION 3.12 FATAL_ERROR)
cmake_minimum_required(VERSION 3.18 FATAL_ERROR)

# Use GPU_ARCHS if CMAKE_CUDA_ARCHITECTURES is not defined
if(NOT DEFINED CMAKE_CUDA_ARCHITECTURES AND DEFINED GPU_ARCHS)
# Use GPU_ARCHS if it is defined
if(DEFINED GPU_ARCHS)
unset(CMAKE_CUDA_ARCHITECTURES CACHE)
if(NOT "${GPU_ARCHS}" STREQUAL "ALL")
set(CMAKE_CUDA_ARCHITECTURES "${GPU_ARCHS}")
endif()
@@ -29,22 +30,61 @@ endif()
# This needs to be run before enabling the CUDA language due to the default initialization behavior
# of `CMAKE_CUDA_ARCHITECTURES`, https://gitlab.kitware.com/cmake/cmake/-/issues/21302
if(NOT DEFINED CMAKE_CUDA_ARCHITECTURES)
set(CUDF_JNI_BUILD_FOR_ALL_ARCHS TRUE)
set(CUDF_BUILD_FOR_ALL_ARCHS TRUE)
elseif(CMAKE_CUDA_ARCHITECTURES STREQUAL "")
unset(CMAKE_CUDA_ARCHITECTURES CACHE)
set(CUDF_JNI_BUILD_FOR_DETECTED_ARCHS TRUE)
set(CUDF_BUILD_FOR_DETECTED_ARCHS TRUE)
endif()

project(CUDF_JNI VERSION 0.7.0 LANGUAGES C CXX CUDA)
project(CUDF_JNI VERSION 0.19 LANGUAGES C CXX)

set(CUDA_DATAFRAME_SOURCE_DIR "${PROJECT_SOURCE_DIR}/../../../../cpp")
set(CUDF_CPP_BUILD_DIR "${CUDA_DATAFRAME_SOURCE_DIR}/build")
###################################################################################################
# - build options ---------------------------------------------------------------------------------

option(USE_NVTX "Build with NVTX support" ON)
option(BUILD_TESTS "Configure CMake to build tests" ON)
option(PER_THREAD_DEFAULT_STREAM "Build with per-thread default stream" OFF)
option(CUDA_STATIC_RUNTIME "Statically link the CUDA runtime" OFF)
option(USE_GDS "Build with GPUDirect Storage (GDS)/cuFile support" OFF)

message(VERBOSE "CUDF_JNI: Build with NVTX support: ${USE_NVTX}")
message(VERBOSE "CUDF_JNI: Configure CMake to build tests: ${BUILD_TESTS}")
message(VERBOSE "CUDF_JNI: Build with per-thread default stream: ${PER_THREAD_DEFAULT_STREAM}")
message(VERBOSE "CUDF_JNI: Statically link the CUDA runtime: ${CUDA_STATIC_RUNTIME}")
message(VERBOSE "CUDF_JNI: Build with GPUDirect Storage support: ${USE_GDS}")

set(CUDF_SOURCE_DIR "${PROJECT_SOURCE_DIR}/../../../../cpp")
set(CUDF_CPP_BUILD_DIR "${CUDF_SOURCE_DIR}/build")

set(CMAKE_MODULE_PATH
"${CMAKE_CURRENT_SOURCE_DIR}/cmake/Modules/"
"${CUDA_DATAFRAME_SOURCE_DIR}/cmake/Modules/"
"${CUDF_SOURCE_DIR}/cmake/Modules/"
${CMAKE_MODULE_PATH})

###################################################################################################
# - compiler options ------------------------------------------------------------------------------

set(CUDF_CXX_FLAGS "")
set(CUDF_CUDA_FLAGS "")
set(CUDF_CXX_DEFINITIONS "")
set(CUDF_CUDA_DEFINITIONS "")

# * find CUDAToolkit package
# * determine GPU architectures
# * enable the CMake CUDA language
# * set other CUDA compilation flags
include(ConfigureCUDA)

# Disable NVTX if necessary
if(NOT USE_NVTX)
target_compile_definitions(cudfjni PUBLIC NVTX_DISABLE)
endif()

if(PER_THREAD_DEFAULT_STREAM)
message(STATUS "Using per-thread default stream")
add_compile_definitions(CUDA_API_PER_THREAD_DEFAULT_STREAM)
endif(PER_THREAD_DEFAULT_STREAM)

###################################################################################################
# - build type ------------------------------------------------------------------------------------

@@ -60,41 +100,17 @@ if(NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES)
"Debug" "Release" "MinSizeRel" "RelWithDebInfo")
endif()

###################################################################################################
# - compiler options ------------------------------------------------------------------------------

set(CMAKE_CXX_STANDARD 14)
set(CMAKE_C_COMPILER $ENV{CC})
set(CMAKE_CXX_COMPILER $ENV{CXX})
set(CMAKE_CXX_STANDARD_REQUIRED ON)

set(CMAKE_CUDA_STANDARD 14)
set(CMAKE_CUDA_STANDARD_REQUIRED ON)

option(BUILD_TESTS "Configure CMake to build tests"
ON)

###################################################################################################
# - cudart options --------------------------------------------------------------------------------
# cudart can be statically linked or dynamically linked. The python ecosystem wants dynamic linking

option(CUDA_STATIC_RUNTIME "Statically link the CUDA runtime" OFF)

if(CUDA_STATIC_RUNTIME)
message(STATUS "Enabling static linking of cudart")
set(CUDART_LIBRARY "cudart_static")
else()
set(CUDART_LIBRARY "cudart")
endif(CUDA_STATIC_RUNTIME)

###################################################################################################
# - cmake modules ---------------------------------------------------------------------------------

include(ConfigureCUDA)
include(FeatureSummary)
include(CheckIncludeFiles)
include(CheckLibraryExists)

###################################################################################################
# - Thrust/CUB/libcudacxx ------------------------------------------------------------------------------------

@@ -123,33 +139,13 @@ find_path(SPDLOG_INCLUDE "spdlog"
set(CUDF_INCLUDE "${PROJECT_SOURCE_DIR}/../../../../cpp/include"
"${PROJECT_SOURCE_DIR}/../../../../cpp/src/")

set(CUDF_LIBS_HINTS
set(CUDF_LIB_HINTS
HINTS "$ENV{CUDF_ROOT}"
"$ENV{CUDF_ROOT}/lib"
"$ENV{CONDA_PREFIX}/lib"
"${CUDF_CPP_BUILD_DIR}")

# This list of cudf libraries needs to be kept in sync with the lists
# in pom.xml and NativeDepsLoader.java
set(CUDF_LIB_BASES
"cudf_ast"
"cudf_base"
"cudf_comms"
"cudf_hash"
"cudf_interop"
"cudf_io"
"cudf_join"
"cudf_merge"
"cudf_partitioning"
"cudf_reductions"
"cudf_replace"
"cudf_rolling")

set(CUDF_LIBS "")
foreach(CUDF_LIB_BASE ${CUDF_LIB_BASES})
find_library(LIB_${CUDF_LIB_BASE} ${CUDF_LIB_BASE} REQUIRED HINTS ${CUDF_LIBS_HINTS})
list(APPEND CUDF_LIBS ${LIB_${CUDF_LIB_BASE}})
endforeach()
find_library(CUDF_LIB "cudf" REQUIRED HINTS ${CUDF_LIB_HINTS})

###################################################################################################
# - RMM -------------------------------------------------------------------------------------------
@@ -168,15 +164,15 @@ message(STATUS "RMM: RMM_INCLUDE set to ${RMM_INCLUDE}")
find_path(ARROW_INCLUDE "arrow"
HINTS "$ENV{ARROW_ROOT}/include"
"$ENV{CONDA_PREFIX}/include"
"${CUDF_CPP_BUILD_DIR}/arrow/install/include")
"${CUDF_CPP_BUILD_DIR}/_deps/arrow-src/cpp/src")

message(STATUS "ARROW: ARROW_INCLUDE set to ${ARROW_INCLUDE}")

# Find static version of Arrow lib
find_library(ARROW_LIBRARY libarrow.a
HINTS "$ENV{ARROW_ROOT}/lib"
"$ENV{CONDA_PREFIX}/lib"
"${CUDF_CPP_BUILD_DIR}/arrow/install/lib")
"${CUDF_CPP_BUILD_DIR}/_deps/arrow-build/release")

if(NOT ARROW_LIBRARY)
message(FATAL_ERROR "Arrow static libs not found. Was libcudf built with ARROW_STATIC_LIB=ON?")
@@ -208,7 +204,6 @@ add_library(nvcomp STATIC IMPORTED ${NVCOMP_LIB})
###################################################################################################
# - GDS/cufile ------------------------------------------------------------------------------------

option(USE_GDS "Build with GPUDirect Storage (GDS)/cuFile support" OFF)
if(USE_GDS)
message(STATUS "Building with GPUDirect Storage (GDS)/cuFile support")
find_package(cuFile REQUIRED)
@@ -261,29 +256,22 @@ add_library(cudfjni SHARED ${SOURCE_FILES})
#Override RPATH for cudfjni
SET_TARGET_PROPERTIES(cudfjni PROPERTIES BUILD_RPATH "\$ORIGIN")

target_compile_options(cudfjni
PRIVATE "$<$<COMPILE_LANGUAGE:CXX>:${CUDF_CXX_FLAGS}>"
"$<$<COMPILE_LANGUAGE:CUDA>:${CUDF_CUDA_FLAGS}>"
)

target_compile_definitions(cudfjni
PUBLIC "$<$<COMPILE_LANGUAGE:CXX>:${CUDF_CXX_DEFINITIONS}>"
"$<$<COMPILE_LANGUAGE:CUDA>:${CUDF_CUDA_DEFINITIONS}>"
)

if(USE_GDS)
add_library(cufilejni SHARED "src/CuFileJni.cpp")
target_include_directories(cufilejni PRIVATE "${cuFile_INCLUDE_DIRS}")
target_link_libraries(cufilejni PRIVATE "${cuFile_LIBRARIES}")
endif(USE_GDS)

###################################################################################################
# - build options ---------------------------------------------------------------------------------

option(USE_NVTX "Build with NVTX support" ON)
if(USE_NVTX)
message(STATUS "Using Nvidia Tools Extension")
find_library(NVTX_LIBRARY nvToolsExt PATH ${CMAKE_CUDA_IMPLICIT_LINK_DIRECTORIES})
target_link_libraries(cudfjni ${NVTX_LIBRARY})
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DUSE_NVTX")
endif(USE_NVTX)

option(PER_THREAD_DEFAULT_STREAM "Build with per-thread default stream" OFF)
if(PER_THREAD_DEFAULT_STREAM)
message(STATUS "Using per-thread default stream")
add_compile_definitions(CUDA_API_PER_THREAD_DEFAULT_STREAM)
endif(PER_THREAD_DEFAULT_STREAM)

###################################################################################################
# - rmm logging level -----------------------------------------------------------------------------

@@ -298,4 +286,4 @@ target_compile_definitions(cudfjni PUBLIC SPDLOG_ACTIVE_LEVEL=SPDLOG_LEVEL_${RMM
###################################################################################################
# - link libraries --------------------------------------------------------------------------------

target_link_libraries(cudfjni ${CUDF_LIBS} ${ARROW_LIBRARY} ${NVCOMP_LIB} ${CUDART_LIBRARY} cuda nvrtc)
target_link_libraries(cudfjni ${CUDF_LIB} ${ARROW_LIBRARY} ${NVCOMP_LIB} ${CUDART_LIBRARY} cuda nvrtc)