Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[REVIEW] Use FetchContent to get jitify and libcudacxx #5398

Merged
merged 6 commits into from
Jun 9, 2020
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 0 additions & 8 deletions .gitmodules

This file was deleted.

3 changes: 2 additions & 1 deletion CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,8 @@

- PR #5403 Update required CMake version to 3.14 in contribution guide
- PR #5245 Add column reduction benchmark
- PR #5315 Use CMake `FetchContent` to obtain latest release of `cub` and `thrust`
- PR #5315 Use CMake `FetchContent` to obtain `cub` and `thrust`
- PR #5398 Use CMake `FetchContent` to obtain `jitify` and `libcudacxx`
- PR #5268 Rely on NumPy arrays for out-of-band pickling
- PR #5288 Drop `auto_pickle` decorator #5288
- PR #5231 Type `Buffer` as `uint8`
Expand Down
83 changes: 39 additions & 44 deletions cpp/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -195,18 +195,13 @@ else()
endif(ARROW_FOUND)

###################################################################################################
# - clone libcu++ ------------------------------------------------------------------------------------

message(STATUS "Installing libcu++")
execute_process(WORKING_DIRECTORY ${CMAKE_BINARY_DIR}
COMMAND git clone --branch cudf --recurse-submodules
https://github.com/rapidsai/thirdparty-freestanding.git libcudacxx)
# - copy libcu++ ------------------------------------------------------------------------------------

file(MAKE_DIRECTORY "${CMAKE_BINARY_DIR}/include/libcudacxx")
file(MAKE_DIRECTORY "${CMAKE_BINARY_DIR}/include/libcxx/include")

file(INSTALL ${CMAKE_BINARY_DIR}/libcudacxx/include/ DESTINATION include/libcudacxx)
file(INSTALL ${CMAKE_BINARY_DIR}/libcudacxx/libcxx/include/ DESTINATION include/libcxx/include)
file(INSTALL ${LIBCUDACXX_INCLUDE_DIR} DESTINATION ${CMAKE_BINARY_DIR}/include/libcudacxx/)
file(INSTALL ${LIBCXX_INCLUDE_DIR} DESTINATION ${CMAKE_BINARY_DIR}/include/libcxx/include/)
harrism marked this conversation as resolved.
Show resolved Hide resolved

###################################################################################################
# - find zlib -------------------------------------------------------------------------------------
Expand Down Expand Up @@ -332,7 +327,9 @@ message(STATUS "BENCHMARK_LIST set to: ${BENCHMARK_LIST}")
# - include paths ---------------------------------------------------------------------------------

include_directories("${CUB_INCLUDE_DIR}"
"${THRUST_INCLUDE_DIR}")
"${THRUST_INCLUDE_DIR}"
"${JITIFY_INCLUDE_DIR}"
"${LIBCUDACXX_INCLUDE_DIR}")

if(CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES)
include_directories("${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES}")
Expand All @@ -342,8 +339,6 @@ include_directories("${CMAKE_BINARY_DIR}/include"
"${CMAKE_BINARY_DIR}/include/jit"
"${CMAKE_SOURCE_DIR}/include"
"${CMAKE_SOURCE_DIR}/src"
"${CMAKE_SOURCE_DIR}/thirdparty/jitify"
"${CMAKE_SOURCE_DIR}/thirdparty/libcudacxx/include"
"${ARROW_INCLUDE_DIR}"
"${FLATBUFFERS_INCLUDE_DIR}"
"${ZLIB_INCLUDE_DIRS}"
Expand Down Expand Up @@ -574,7 +569,7 @@ set_target_properties(cudf PROPERTIES BUILD_RPATH "\$ORIGIN")
# - jitify ----------------------------------------------------------------------------------------

# Creates executable stringify and uses it to convert types.h to c-str for use in JIT code
add_executable(stringify "${CMAKE_SOURCE_DIR}/thirdparty/jitify/stringify.cpp")
add_executable(stringify "${JITIFY_INCLUDE_DIR}/stringify.cpp")
execute_process(WORKING_DIRECTORY ${CMAKE_BINARY_DIR}
COMMAND ${CMAKE_COMMAND} -E make_directory
${CMAKE_BINARY_DIR}/include/jit
Expand Down Expand Up @@ -609,45 +604,45 @@ add_custom_command(WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/include
${CMAKE_CURRENT_SOURCE_DIR}/include/cudf/types.hpp
${CMAKE_CURRENT_SOURCE_DIR}/include/cudf/utilities/bit.hpp
${CMAKE_CURRENT_SOURCE_DIR}/include/cudf/wrappers/timestamps.hpp
${CMAKE_SOURCE_DIR}/thirdparty/libcudacxx/include/details/__config
${CMAKE_SOURCE_DIR}/thirdparty/libcudacxx/include/simt/limits
${CMAKE_SOURCE_DIR}/thirdparty/libcudacxx/include/simt/cfloat
${CMAKE_SOURCE_DIR}/thirdparty/libcudacxx/include/simt/chrono
${CMAKE_SOURCE_DIR}/thirdparty/libcudacxx/include/simt/ctime
${CMAKE_SOURCE_DIR}/thirdparty/libcudacxx/include/simt/ratio
${CMAKE_SOURCE_DIR}/thirdparty/libcudacxx/include/simt/type_traits
${CMAKE_SOURCE_DIR}/thirdparty/libcudacxx/include/simt/version
${CMAKE_SOURCE_DIR}/thirdparty/libcudacxx/libcxx/include/__config
${CMAKE_SOURCE_DIR}/thirdparty/libcudacxx/libcxx/include/__undef_macros
${CMAKE_SOURCE_DIR}/thirdparty/libcudacxx/libcxx/include/cfloat
${CMAKE_SOURCE_DIR}/thirdparty/libcudacxx/libcxx/include/chrono
${CMAKE_SOURCE_DIR}/thirdparty/libcudacxx/libcxx/include/ctime
${CMAKE_SOURCE_DIR}/thirdparty/libcudacxx/libcxx/include/limits
${CMAKE_SOURCE_DIR}/thirdparty/libcudacxx/libcxx/include/ratio
${CMAKE_SOURCE_DIR}/thirdparty/libcudacxx/libcxx/include/type_traits
${LIBCUDACXX_INCLUDE_DIR}/details/__config
${LIBCUDACXX_INCLUDE_DIR}/simt/limits
${LIBCUDACXX_INCLUDE_DIR}/simt/cfloat
${LIBCUDACXX_INCLUDE_DIR}/simt/chrono
${LIBCUDACXX_INCLUDE_DIR}/simt/ctime
${LIBCUDACXX_INCLUDE_DIR}/simt/ratio
${LIBCUDACXX_INCLUDE_DIR}/simt/type_traits
${LIBCUDACXX_INCLUDE_DIR}/simt/version
${LIBCXX_INCLUDE_DIR}/__config
${LIBCXX_INCLUDE_DIR}/__undef_macros
${LIBCXX_INCLUDE_DIR}/cfloat
${LIBCXX_INCLUDE_DIR}/chrono
${LIBCXX_INCLUDE_DIR}/ctime
${LIBCXX_INCLUDE_DIR}/limits
${LIBCXX_INCLUDE_DIR}/ratio
${LIBCXX_INCLUDE_DIR}/type_traits

# stringified headers are placed underneath the bin include jit directory and end in ".jit"
COMMAND ${CMAKE_BINARY_DIR}/stringify cudf/types.h > ${CMAKE_BINARY_DIR}/include/jit/types.h.jit
COMMAND ${CMAKE_BINARY_DIR}/stringify cudf/types.hpp > ${CMAKE_BINARY_DIR}/include/jit/types.hpp.jit
COMMAND ${CMAKE_BINARY_DIR}/stringify cudf/utilities/bit.hpp > ${CMAKE_BINARY_DIR}/include/bit.hpp.jit
COMMAND ${CMAKE_BINARY_DIR}/stringify ../src/rolling/rolling_jit_detail.hpp > ${CMAKE_BINARY_DIR}/include/rolling_jit_detail.hpp.jit
COMMAND ${CMAKE_BINARY_DIR}/stringify cudf/wrappers/timestamps.hpp > ${CMAKE_BINARY_DIR}/include/jit/timestamps.hpp.jit
COMMAND ${CMAKE_BINARY_DIR}/stringify ../../thirdparty/libcudacxx/include/details/__config libcudacxx_details_config > ${CMAKE_BINARY_DIR}/include/jit/libcudacxx/details/__config.jit
COMMAND ${CMAKE_BINARY_DIR}/stringify ../../thirdparty/libcudacxx/include/simt/limits libcudacxx_simt_limits > ${CMAKE_BINARY_DIR}/include/jit/libcudacxx/simt/limits.jit
COMMAND ${CMAKE_BINARY_DIR}/stringify ../../thirdparty/libcudacxx/include/simt/cfloat libcudacxx_simt_cfloat > ${CMAKE_BINARY_DIR}/include/jit/libcudacxx/simt/cfloat.jit
COMMAND ${CMAKE_BINARY_DIR}/stringify ../../thirdparty/libcudacxx/include/simt/chrono libcudacxx_simt_chrono > ${CMAKE_BINARY_DIR}/include/jit/libcudacxx/simt/chrono.jit
COMMAND ${CMAKE_BINARY_DIR}/stringify ../../thirdparty/libcudacxx/include/simt/ctime libcudacxx_simt_ctime > ${CMAKE_BINARY_DIR}/include/jit/libcudacxx/simt/ctime.jit
COMMAND ${CMAKE_BINARY_DIR}/stringify ../../thirdparty/libcudacxx/include/simt/ratio libcudacxx_simt_ratio > ${CMAKE_BINARY_DIR}/include/jit/libcudacxx/simt/ratio.jit
COMMAND ${CMAKE_BINARY_DIR}/stringify ../../thirdparty/libcudacxx/include/simt/type_traits libcudacxx_simt_type_traits > ${CMAKE_BINARY_DIR}/include/jit/libcudacxx/simt/type_traits.jit
COMMAND ${CMAKE_BINARY_DIR}/stringify ../../thirdparty/libcudacxx/include/version libcudacxx_simt_version > ${CMAKE_BINARY_DIR}/include/jit/libcudacxx/simt/version.jit
COMMAND ${CMAKE_BINARY_DIR}/stringify ../../thirdparty/libcudacxx/libcxx/include/__config libcxx_config > ${CMAKE_BINARY_DIR}/include/jit/libcudacxx/libcxx/include/__config.jit
COMMAND ${CMAKE_BINARY_DIR}/stringify ../../thirdparty/libcudacxx/libcxx/include/__undef_macros libcxx_undef_macros > ${CMAKE_BINARY_DIR}/include/jit/libcudacxx/libcxx/include/__undef_macros.jit
COMMAND ${CMAKE_BINARY_DIR}/stringify ../../thirdparty/libcudacxx/libcxx/include/cfloat libcxx_cfloat > ${CMAKE_BINARY_DIR}/include/jit/libcudacxx/libcxx/include/cfloat.jit
COMMAND ${CMAKE_BINARY_DIR}/stringify ../../thirdparty/libcudacxx/libcxx/include/chrono libcxx_chrono > ${CMAKE_BINARY_DIR}/include/jit/libcudacxx/libcxx/include/chrono.jit
COMMAND ${CMAKE_BINARY_DIR}/stringify ../../thirdparty/libcudacxx/libcxx/include/ctime libcxx_ctime > ${CMAKE_BINARY_DIR}/include/jit/libcudacxx/libcxx/include/ctime.jit
COMMAND ${CMAKE_BINARY_DIR}/stringify ../../thirdparty/libcudacxx/libcxx/include/limits libcxx_limits > ${CMAKE_BINARY_DIR}/include/jit/libcudacxx/libcxx/include/limits.jit
COMMAND ${CMAKE_BINARY_DIR}/stringify ../../thirdparty/libcudacxx/libcxx/include/ratio libcxx_ratio > ${CMAKE_BINARY_DIR}/include/jit/libcudacxx/libcxx/include/ratio.jit
COMMAND ${CMAKE_BINARY_DIR}/stringify ../../thirdparty/libcudacxx/libcxx/include/type_traits libcxx_type_traits > ${CMAKE_BINARY_DIR}/include/jit/libcudacxx/libcxx/include/type_traits.jit
COMMAND ${CMAKE_BINARY_DIR}/stringify ${LIBCUDACXX_INCLUDE_DIR}/details/__config libcudacxx_details_config > ${CMAKE_BINARY_DIR}/include/jit/libcudacxx/details/__config.jit
COMMAND ${CMAKE_BINARY_DIR}/stringify ${LIBCUDACXX_INCLUDE_DIR}/simt/limits libcudacxx_simt_limits > ${CMAKE_BINARY_DIR}/include/jit/libcudacxx/simt/limits.jit
COMMAND ${CMAKE_BINARY_DIR}/stringify ${LIBCUDACXX_INCLUDE_DIR}/simt/cfloat libcudacxx_simt_cfloat > ${CMAKE_BINARY_DIR}/include/jit/libcudacxx/simt/cfloat.jit
COMMAND ${CMAKE_BINARY_DIR}/stringify ${LIBCUDACXX_INCLUDE_DIR}/simt/chrono libcudacxx_simt_chrono > ${CMAKE_BINARY_DIR}/include/jit/libcudacxx/simt/chrono.jit
COMMAND ${CMAKE_BINARY_DIR}/stringify ${LIBCUDACXX_INCLUDE_DIR}/simt/ctime libcudacxx_simt_ctime > ${CMAKE_BINARY_DIR}/include/jit/libcudacxx/simt/ctime.jit
COMMAND ${CMAKE_BINARY_DIR}/stringify ${LIBCUDACXX_INCLUDE_DIR}/simt/ratio libcudacxx_simt_ratio > ${CMAKE_BINARY_DIR}/include/jit/libcudacxx/simt/ratio.jit
COMMAND ${CMAKE_BINARY_DIR}/stringify ${LIBCUDACXX_INCLUDE_DIR}/simt/type_traits libcudacxx_simt_type_traits > ${CMAKE_BINARY_DIR}/include/jit/libcudacxx/simt/type_traits.jit
COMMAND ${CMAKE_BINARY_DIR}/stringify ${LIBCUDACXX_INCLUDE_DIR}/version libcudacxx_simt_version > ${CMAKE_BINARY_DIR}/include/jit/libcudacxx/simt/version.jit
COMMAND ${CMAKE_BINARY_DIR}/stringify ${LIBCXX_INCLUDE_DIR}/__config libcxx_config > ${CMAKE_BINARY_DIR}/include/jit/libcudacxx/libcxx/include/__config.jit
COMMAND ${CMAKE_BINARY_DIR}/stringify ${LIBCXX_INCLUDE_DIR}/__undef_macros libcxx_undef_macros > ${CMAKE_BINARY_DIR}/include/jit/libcudacxx/libcxx/include/__undef_macros.jit
COMMAND ${CMAKE_BINARY_DIR}/stringify ${LIBCXX_INCLUDE_DIR}/cfloat libcxx_cfloat > ${CMAKE_BINARY_DIR}/include/jit/libcudacxx/libcxx/include/cfloat.jit
COMMAND ${CMAKE_BINARY_DIR}/stringify ${LIBCXX_INCLUDE_DIR}/chrono libcxx_chrono > ${CMAKE_BINARY_DIR}/include/jit/libcudacxx/libcxx/include/chrono.jit
COMMAND ${CMAKE_BINARY_DIR}/stringify ${LIBCXX_INCLUDE_DIR}/ctime libcxx_ctime > ${CMAKE_BINARY_DIR}/include/jit/libcudacxx/libcxx/include/ctime.jit
COMMAND ${CMAKE_BINARY_DIR}/stringify ${LIBCXX_INCLUDE_DIR}/limits libcxx_limits > ${CMAKE_BINARY_DIR}/include/jit/libcudacxx/libcxx/include/limits.jit
COMMAND ${CMAKE_BINARY_DIR}/stringify ${LIBCXX_INCLUDE_DIR}/ratio libcxx_ratio > ${CMAKE_BINARY_DIR}/include/jit/libcudacxx/libcxx/include/ratio.jit
COMMAND ${CMAKE_BINARY_DIR}/stringify ${LIBCXX_INCLUDE_DIR}/type_traits libcxx_type_traits > ${CMAKE_BINARY_DIR}/include/jit/libcudacxx/libcxx/include/type_traits.jit
)

add_custom_target(stringify_run DEPENDS
Expand Down
12 changes: 8 additions & 4 deletions cpp/benchmarks/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -24,14 +24,18 @@ endfunction(ConfigureBench)

include_directories("${CUB_INCLUDE_DIR}"
"${THRUST_INCLUDE_DIR}"
"${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES}"
"${CMAKE_BINARY_DIR}/include"
"${JITIFY_INCLUDE_DIR}"
"${LIBCUDACXX_INCLUDE_DIR}")

if(CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES)
include_directories("${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES}")
endif(CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES)

include_directories("${CMAKE_BINARY_DIR}/include"
"${CMAKE_SOURCE_DIR}/include"
"${CMAKE_SOURCE_DIR}"
"${CMAKE_SOURCE_DIR}/src"
"${CMAKE_SOURCE_DIR}/thirdparty/dlpack/include"
"${CMAKE_SOURCE_DIR}/thirdparty/jitify"
"${CMAKE_SOURCE_DIR}/thirdparty/libcudacxx/include"
"${GTEST_INCLUDE_DIR}"
"${GBENCH_INCLUDE_DIR}"
"${ARROW_INCLUDE_DIR}"
Expand Down
6 changes: 3 additions & 3 deletions cpp/tests/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,9 @@ endfunction(ConfigureTest)
# - include paths ---------------------------------------------------------------------------------

include_directories("${CUB_INCLUDE_DIR}"
"${THRUST_INCLUDE_DIR}")
"${THRUST_INCLUDE_DIR}"
"${JITIFY_INCLUDE_DIR}"
"${LIBCUDACXX_INCLUDE_DIR}")

if(CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES)
include_directories("${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES}")
Expand All @@ -69,9 +71,7 @@ include_directories("${CMAKE_BINARY_DIR}/include"
"${CMAKE_SOURCE_DIR}/include"
"${CMAKE_SOURCE_DIR}"
"${CMAKE_SOURCE_DIR}/src"
"${CMAKE_SOURCE_DIR}/thirdparty/jitify"
"${CMAKE_SOURCE_DIR}/thirdparty/dlpack/include"
"${CMAKE_SOURCE_DIR}/thirdparty/libcudacxx/include"
"${GTEST_INCLUDE_DIR}"
"${ARROW_INCLUDE_DIR}"
"${FLATBUFFERS_INCLUDE_DIR}"
Expand Down
12 changes: 8 additions & 4 deletions python/cudf/setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,8 @@

cuda_include_dir = os.path.join(CUDA_HOME, "include")

CUDF_ROOT = os.environ.get("CUDF_ROOT", "../../cpp/build/")

try:
nthreads = int(os.environ.get("PARALLEL_LEVEL", "0") or "0")
except Exception:
Expand All @@ -43,10 +45,12 @@
include_dirs=[
"../../cpp/include/cudf",
"../../cpp/include",
"../../cpp/build/include",
"../../cpp/build/_deps/cub-src",
"../../cpp/build/_deps/thrust-src",
"../../thirdparty/libcudacxx/include",
os.path.join(CUDF_ROOT, "include"),
os.path.join(CUDF_ROOT, "_deps/libcudacxx-src/include"),
os.path.join(
os.path.dirname(sysconfig.get_path("include")),
"libcudf/libcudacxx",
),
os.path.dirname(sysconfig.get_path("include")),
np.get_include(),
cuda_include_dir,
Expand Down
45 changes: 45 additions & 0 deletions thirdparty/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@ include(FetchContent)
###################################################################################################
# - cub -------------------------------------------------------------------------------------------

message("Fetching CUB")

FetchContent_Declare(
cub
GIT_REPOSITORY https://github.com/thrust/cub.git
Expand All @@ -23,6 +25,8 @@ set(CUB_INCLUDE_DIR "${cub_SOURCE_DIR}" PARENT_SCOPE)
###################################################################################################
# - thrust ----------------------------------------------------------------------------------------

message("Fetching Thrust")

FetchContent_Declare(
thrust
GIT_REPOSITORY https://github.com/thrust/thrust.git
Expand All @@ -36,3 +40,44 @@ if(NOT thrust_POPULATED)
# We are not using the thrust CMake targets, so no need to call `add_subdirectory()`.
endif()
set(THRUST_INCLUDE_DIR "${thrust_SOURCE_DIR}" PARENT_SCOPE)

###################################################################################################
# - jitify ----------------------------------------------------------------------------------------

message("Fetching Jitify")

FetchContent_Declare(
jitify
GIT_REPOSITORY https://github.com/rapidsai/jitify.git
GIT_TAG cudf_0.15
GIT_SHALLOW true
)

FetchContent_GetProperties(jitify)
if(NOT jitify_POPULATED)
FetchContent_Populate(jitify)
# We are not using the jitify CMake targets, so no need to call `add_subdirectory()`.
endif()
set(JITIFY_INCLUDE_DIR "${jitify_SOURCE_DIR}" PARENT_SCOPE)

###################################################################################################
# - libcudacxx ------------------------------------------------------------------------------------

message("Fetching libcudacxx")

FetchContent_Declare(
libcudacxx
GIT_REPOSITORY https://github.com/rapidsai/thirdparty-freestanding.git
GIT_TAG cudf
GIT_SHALLOW true
)

FetchContent_GetProperties(libcudacxx)
if(NOT libcudacxx_POPULATED)
FetchContent_Populate(libcudacxx)
# libcudacxx has no CMake targets, so no need to call `add_subdirectory()`.
endif()
set(LIBCUDACXX_DIR "${libcudacxx_SOURCE_DIR}" PARENT_SCOPE)
set(LIBCUDACXX_INCLUDE_DIR "${libcudacxx_SOURCE_DIR}/include/" PARENT_SCOPE)
set(LIBCXX_DIR "${libcudacxx_SOURCE_DIR}/libcxx/" PARENT_SCOPE)
set(LIBCXX_INCLUDE_DIR "${libcudacxx_SOURCE_DIR}/libcxx/include/" PARENT_SCOPE)
1 change: 0 additions & 1 deletion thirdparty/jitify
Submodule jitify deleted from e3f867
1 change: 0 additions & 1 deletion thirdparty/libcudacxx
Submodule libcudacxx deleted from cdcda4