diff --git a/.gitmodules b/.gitmodules deleted file mode 100644 index a01c0310e72..00000000000 --- a/.gitmodules +++ /dev/null @@ -1,8 +0,0 @@ -[submodule "thirdparty/jitify"] - path = thirdparty/jitify - url = https://github.com/rapidsai/jitify.git - branch = cudf_0.14 -[submodule "thirdparty/libcudacxx"] - path = thirdparty/libcudacxx - url = https://github.com/rapidsai/thirdparty-freestanding.git - branch = cudf diff --git a/CHANGELOG.md b/CHANGELOG.md index fe0b762b7a8..4c556052082 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,7 +17,8 @@ - PR #5403 Update required CMake version to 3.14 in contribution guide - PR #5245 Add column reduction benchmark -- PR #5315 Use CMake `FetchContent` to obtain latest release of `cub` and `thrust` +- PR #5315 Use CMake `FetchContent` to obtain `cub` and `thrust` +- PR #5398 Use CMake `FetchContent` to obtain `jitify` and `libcudacxx` - PR #5268 Rely on NumPy arrays for out-of-band pickling - PR #5288 Drop `auto_pickle` decorator #5288 - PR #5231 Type `Buffer` as `uint8` diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 50d060e7a56..7fa242fa11f 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -195,18 +195,13 @@ else() endif(ARROW_FOUND) ################################################################################################### -# - clone libcu++ ------------------------------------------------------------------------------------ - -message(STATUS "Installing libcu++") -execute_process(WORKING_DIRECTORY ${CMAKE_BINARY_DIR} - COMMAND git clone --branch cudf --recurse-submodules - https://github.com/rapidsai/thirdparty-freestanding.git libcudacxx) +# - copy libcu++ ------------------------------------------------------------------------------------ file(MAKE_DIRECTORY "${CMAKE_BINARY_DIR}/include/libcudacxx") file(MAKE_DIRECTORY "${CMAKE_BINARY_DIR}/include/libcxx/include") -file(INSTALL ${CMAKE_BINARY_DIR}/libcudacxx/include/ DESTINATION include/libcudacxx) -file(INSTALL ${CMAKE_BINARY_DIR}/libcudacxx/libcxx/include/ DESTINATION include/libcxx/include) +file(INSTALL ${LIBCUDACXX_INCLUDE_DIR} DESTINATION ${CMAKE_BINARY_DIR}/include/libcudacxx/) +file(INSTALL ${LIBCXX_INCLUDE_DIR} DESTINATION ${CMAKE_BINARY_DIR}/include/libcxx/include/) ################################################################################################### # - find zlib ------------------------------------------------------------------------------------- @@ -332,7 +327,9 @@ message(STATUS "BENCHMARK_LIST set to: ${BENCHMARK_LIST}") # - include paths --------------------------------------------------------------------------------- include_directories("${CUB_INCLUDE_DIR}" - "${THRUST_INCLUDE_DIR}") + "${THRUST_INCLUDE_DIR}" + "${JITIFY_INCLUDE_DIR}" + "${LIBCUDACXX_INCLUDE_DIR}") if(CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES) include_directories("${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES}") @@ -342,8 +339,6 @@ include_directories("${CMAKE_BINARY_DIR}/include" "${CMAKE_BINARY_DIR}/include/jit" "${CMAKE_SOURCE_DIR}/include" "${CMAKE_SOURCE_DIR}/src" - "${CMAKE_SOURCE_DIR}/thirdparty/jitify" - "${CMAKE_SOURCE_DIR}/thirdparty/libcudacxx/include" "${ARROW_INCLUDE_DIR}" "${FLATBUFFERS_INCLUDE_DIR}" "${ZLIB_INCLUDE_DIRS}" @@ -574,7 +569,7 @@ set_target_properties(cudf PROPERTIES BUILD_RPATH "\$ORIGIN") # - jitify ---------------------------------------------------------------------------------------- # Creates executable stringify and uses it to convert types.h to c-str for use in JIT code -add_executable(stringify "${CMAKE_SOURCE_DIR}/thirdparty/jitify/stringify.cpp") +add_executable(stringify "${JITIFY_INCLUDE_DIR}/stringify.cpp") execute_process(WORKING_DIRECTORY ${CMAKE_BINARY_DIR} COMMAND ${CMAKE_COMMAND} -E make_directory ${CMAKE_BINARY_DIR}/include/jit @@ -609,22 +604,22 @@ add_custom_command(WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/include ${CMAKE_CURRENT_SOURCE_DIR}/include/cudf/types.hpp ${CMAKE_CURRENT_SOURCE_DIR}/include/cudf/utilities/bit.hpp ${CMAKE_CURRENT_SOURCE_DIR}/include/cudf/wrappers/timestamps.hpp - ${CMAKE_SOURCE_DIR}/thirdparty/libcudacxx/include/details/__config - ${CMAKE_SOURCE_DIR}/thirdparty/libcudacxx/include/simt/limits - ${CMAKE_SOURCE_DIR}/thirdparty/libcudacxx/include/simt/cfloat - ${CMAKE_SOURCE_DIR}/thirdparty/libcudacxx/include/simt/chrono - ${CMAKE_SOURCE_DIR}/thirdparty/libcudacxx/include/simt/ctime - ${CMAKE_SOURCE_DIR}/thirdparty/libcudacxx/include/simt/ratio - ${CMAKE_SOURCE_DIR}/thirdparty/libcudacxx/include/simt/type_traits - ${CMAKE_SOURCE_DIR}/thirdparty/libcudacxx/include/simt/version - ${CMAKE_SOURCE_DIR}/thirdparty/libcudacxx/libcxx/include/__config - ${CMAKE_SOURCE_DIR}/thirdparty/libcudacxx/libcxx/include/__undef_macros - ${CMAKE_SOURCE_DIR}/thirdparty/libcudacxx/libcxx/include/cfloat - ${CMAKE_SOURCE_DIR}/thirdparty/libcudacxx/libcxx/include/chrono - ${CMAKE_SOURCE_DIR}/thirdparty/libcudacxx/libcxx/include/ctime - ${CMAKE_SOURCE_DIR}/thirdparty/libcudacxx/libcxx/include/limits - ${CMAKE_SOURCE_DIR}/thirdparty/libcudacxx/libcxx/include/ratio - ${CMAKE_SOURCE_DIR}/thirdparty/libcudacxx/libcxx/include/type_traits + ${LIBCUDACXX_INCLUDE_DIR}/details/__config + ${LIBCUDACXX_INCLUDE_DIR}/simt/limits + ${LIBCUDACXX_INCLUDE_DIR}/simt/cfloat + ${LIBCUDACXX_INCLUDE_DIR}/simt/chrono + ${LIBCUDACXX_INCLUDE_DIR}/simt/ctime + ${LIBCUDACXX_INCLUDE_DIR}/simt/ratio + ${LIBCUDACXX_INCLUDE_DIR}/simt/type_traits + ${LIBCUDACXX_INCLUDE_DIR}/simt/version + ${LIBCXX_INCLUDE_DIR}/__config + ${LIBCXX_INCLUDE_DIR}/__undef_macros + ${LIBCXX_INCLUDE_DIR}/cfloat + ${LIBCXX_INCLUDE_DIR}/chrono + ${LIBCXX_INCLUDE_DIR}/ctime + ${LIBCXX_INCLUDE_DIR}/limits + ${LIBCXX_INCLUDE_DIR}/ratio + ${LIBCXX_INCLUDE_DIR}/type_traits # stringified headers are placed underneath the bin include jit directory and end in ".jit" COMMAND ${CMAKE_BINARY_DIR}/stringify cudf/types.h > ${CMAKE_BINARY_DIR}/include/jit/types.h.jit @@ -632,22 +627,22 @@ add_custom_command(WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/include COMMAND ${CMAKE_BINARY_DIR}/stringify cudf/utilities/bit.hpp > ${CMAKE_BINARY_DIR}/include/bit.hpp.jit COMMAND ${CMAKE_BINARY_DIR}/stringify ../src/rolling/rolling_jit_detail.hpp > ${CMAKE_BINARY_DIR}/include/rolling_jit_detail.hpp.jit COMMAND ${CMAKE_BINARY_DIR}/stringify cudf/wrappers/timestamps.hpp > ${CMAKE_BINARY_DIR}/include/jit/timestamps.hpp.jit - COMMAND ${CMAKE_BINARY_DIR}/stringify ../../thirdparty/libcudacxx/include/details/__config libcudacxx_details_config > ${CMAKE_BINARY_DIR}/include/jit/libcudacxx/details/__config.jit - COMMAND ${CMAKE_BINARY_DIR}/stringify ../../thirdparty/libcudacxx/include/simt/limits libcudacxx_simt_limits > ${CMAKE_BINARY_DIR}/include/jit/libcudacxx/simt/limits.jit - COMMAND ${CMAKE_BINARY_DIR}/stringify ../../thirdparty/libcudacxx/include/simt/cfloat libcudacxx_simt_cfloat > ${CMAKE_BINARY_DIR}/include/jit/libcudacxx/simt/cfloat.jit - COMMAND ${CMAKE_BINARY_DIR}/stringify ../../thirdparty/libcudacxx/include/simt/chrono libcudacxx_simt_chrono > ${CMAKE_BINARY_DIR}/include/jit/libcudacxx/simt/chrono.jit - COMMAND ${CMAKE_BINARY_DIR}/stringify ../../thirdparty/libcudacxx/include/simt/ctime libcudacxx_simt_ctime > ${CMAKE_BINARY_DIR}/include/jit/libcudacxx/simt/ctime.jit - COMMAND ${CMAKE_BINARY_DIR}/stringify ../../thirdparty/libcudacxx/include/simt/ratio libcudacxx_simt_ratio > ${CMAKE_BINARY_DIR}/include/jit/libcudacxx/simt/ratio.jit - COMMAND ${CMAKE_BINARY_DIR}/stringify ../../thirdparty/libcudacxx/include/simt/type_traits libcudacxx_simt_type_traits > ${CMAKE_BINARY_DIR}/include/jit/libcudacxx/simt/type_traits.jit - COMMAND ${CMAKE_BINARY_DIR}/stringify ../../thirdparty/libcudacxx/include/version libcudacxx_simt_version > ${CMAKE_BINARY_DIR}/include/jit/libcudacxx/simt/version.jit - COMMAND ${CMAKE_BINARY_DIR}/stringify ../../thirdparty/libcudacxx/libcxx/include/__config libcxx_config > ${CMAKE_BINARY_DIR}/include/jit/libcudacxx/libcxx/include/__config.jit - COMMAND ${CMAKE_BINARY_DIR}/stringify ../../thirdparty/libcudacxx/libcxx/include/__undef_macros libcxx_undef_macros > ${CMAKE_BINARY_DIR}/include/jit/libcudacxx/libcxx/include/__undef_macros.jit - COMMAND ${CMAKE_BINARY_DIR}/stringify ../../thirdparty/libcudacxx/libcxx/include/cfloat libcxx_cfloat > ${CMAKE_BINARY_DIR}/include/jit/libcudacxx/libcxx/include/cfloat.jit - COMMAND ${CMAKE_BINARY_DIR}/stringify ../../thirdparty/libcudacxx/libcxx/include/chrono libcxx_chrono > ${CMAKE_BINARY_DIR}/include/jit/libcudacxx/libcxx/include/chrono.jit - COMMAND ${CMAKE_BINARY_DIR}/stringify ../../thirdparty/libcudacxx/libcxx/include/ctime libcxx_ctime > ${CMAKE_BINARY_DIR}/include/jit/libcudacxx/libcxx/include/ctime.jit - COMMAND ${CMAKE_BINARY_DIR}/stringify ../../thirdparty/libcudacxx/libcxx/include/limits libcxx_limits > ${CMAKE_BINARY_DIR}/include/jit/libcudacxx/libcxx/include/limits.jit - COMMAND ${CMAKE_BINARY_DIR}/stringify ../../thirdparty/libcudacxx/libcxx/include/ratio libcxx_ratio > ${CMAKE_BINARY_DIR}/include/jit/libcudacxx/libcxx/include/ratio.jit - COMMAND ${CMAKE_BINARY_DIR}/stringify ../../thirdparty/libcudacxx/libcxx/include/type_traits libcxx_type_traits > ${CMAKE_BINARY_DIR}/include/jit/libcudacxx/libcxx/include/type_traits.jit + COMMAND ${CMAKE_BINARY_DIR}/stringify ${LIBCUDACXX_INCLUDE_DIR}/details/__config libcudacxx_details_config > ${CMAKE_BINARY_DIR}/include/jit/libcudacxx/details/__config.jit + COMMAND ${CMAKE_BINARY_DIR}/stringify ${LIBCUDACXX_INCLUDE_DIR}/simt/limits libcudacxx_simt_limits > ${CMAKE_BINARY_DIR}/include/jit/libcudacxx/simt/limits.jit + COMMAND ${CMAKE_BINARY_DIR}/stringify ${LIBCUDACXX_INCLUDE_DIR}/simt/cfloat libcudacxx_simt_cfloat > ${CMAKE_BINARY_DIR}/include/jit/libcudacxx/simt/cfloat.jit + COMMAND ${CMAKE_BINARY_DIR}/stringify ${LIBCUDACXX_INCLUDE_DIR}/simt/chrono libcudacxx_simt_chrono > ${CMAKE_BINARY_DIR}/include/jit/libcudacxx/simt/chrono.jit + COMMAND ${CMAKE_BINARY_DIR}/stringify ${LIBCUDACXX_INCLUDE_DIR}/simt/ctime libcudacxx_simt_ctime > ${CMAKE_BINARY_DIR}/include/jit/libcudacxx/simt/ctime.jit + COMMAND ${CMAKE_BINARY_DIR}/stringify ${LIBCUDACXX_INCLUDE_DIR}/simt/ratio libcudacxx_simt_ratio > ${CMAKE_BINARY_DIR}/include/jit/libcudacxx/simt/ratio.jit + COMMAND ${CMAKE_BINARY_DIR}/stringify ${LIBCUDACXX_INCLUDE_DIR}/simt/type_traits libcudacxx_simt_type_traits > ${CMAKE_BINARY_DIR}/include/jit/libcudacxx/simt/type_traits.jit + COMMAND ${CMAKE_BINARY_DIR}/stringify ${LIBCUDACXX_INCLUDE_DIR}/version libcudacxx_simt_version > ${CMAKE_BINARY_DIR}/include/jit/libcudacxx/simt/version.jit + COMMAND ${CMAKE_BINARY_DIR}/stringify ${LIBCXX_INCLUDE_DIR}/__config libcxx_config > ${CMAKE_BINARY_DIR}/include/jit/libcudacxx/libcxx/include/__config.jit + COMMAND ${CMAKE_BINARY_DIR}/stringify ${LIBCXX_INCLUDE_DIR}/__undef_macros libcxx_undef_macros > ${CMAKE_BINARY_DIR}/include/jit/libcudacxx/libcxx/include/__undef_macros.jit + COMMAND ${CMAKE_BINARY_DIR}/stringify ${LIBCXX_INCLUDE_DIR}/cfloat libcxx_cfloat > ${CMAKE_BINARY_DIR}/include/jit/libcudacxx/libcxx/include/cfloat.jit + COMMAND ${CMAKE_BINARY_DIR}/stringify ${LIBCXX_INCLUDE_DIR}/chrono libcxx_chrono > ${CMAKE_BINARY_DIR}/include/jit/libcudacxx/libcxx/include/chrono.jit + COMMAND ${CMAKE_BINARY_DIR}/stringify ${LIBCXX_INCLUDE_DIR}/ctime libcxx_ctime > ${CMAKE_BINARY_DIR}/include/jit/libcudacxx/libcxx/include/ctime.jit + COMMAND ${CMAKE_BINARY_DIR}/stringify ${LIBCXX_INCLUDE_DIR}/limits libcxx_limits > ${CMAKE_BINARY_DIR}/include/jit/libcudacxx/libcxx/include/limits.jit + COMMAND ${CMAKE_BINARY_DIR}/stringify ${LIBCXX_INCLUDE_DIR}/ratio libcxx_ratio > ${CMAKE_BINARY_DIR}/include/jit/libcudacxx/libcxx/include/ratio.jit + COMMAND ${CMAKE_BINARY_DIR}/stringify ${LIBCXX_INCLUDE_DIR}/type_traits libcxx_type_traits > ${CMAKE_BINARY_DIR}/include/jit/libcudacxx/libcxx/include/type_traits.jit ) add_custom_target(stringify_run DEPENDS diff --git a/cpp/benchmarks/CMakeLists.txt b/cpp/benchmarks/CMakeLists.txt index c31d8a49779..3a7674677a3 100644 --- a/cpp/benchmarks/CMakeLists.txt +++ b/cpp/benchmarks/CMakeLists.txt @@ -24,14 +24,18 @@ endfunction(ConfigureBench) include_directories("${CUB_INCLUDE_DIR}" "${THRUST_INCLUDE_DIR}" - "${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES}" - "${CMAKE_BINARY_DIR}/include" + "${JITIFY_INCLUDE_DIR}" + "${LIBCUDACXX_INCLUDE_DIR}") + +if(CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES) + include_directories("${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES}") +endif(CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES) + +include_directories("${CMAKE_BINARY_DIR}/include" "${CMAKE_SOURCE_DIR}/include" "${CMAKE_SOURCE_DIR}" "${CMAKE_SOURCE_DIR}/src" "${CMAKE_SOURCE_DIR}/thirdparty/dlpack/include" - "${CMAKE_SOURCE_DIR}/thirdparty/jitify" - "${CMAKE_SOURCE_DIR}/thirdparty/libcudacxx/include" "${GTEST_INCLUDE_DIR}" "${GBENCH_INCLUDE_DIR}" "${ARROW_INCLUDE_DIR}" diff --git a/cpp/tests/CMakeLists.txt b/cpp/tests/CMakeLists.txt index 98e26ab3cfa..f8ca33b8432 100644 --- a/cpp/tests/CMakeLists.txt +++ b/cpp/tests/CMakeLists.txt @@ -59,7 +59,9 @@ endfunction(ConfigureTest) # - include paths --------------------------------------------------------------------------------- include_directories("${CUB_INCLUDE_DIR}" - "${THRUST_INCLUDE_DIR}") + "${THRUST_INCLUDE_DIR}" + "${JITIFY_INCLUDE_DIR}" + "${LIBCUDACXX_INCLUDE_DIR}") if(CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES) include_directories("${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES}") @@ -69,9 +71,7 @@ include_directories("${CMAKE_BINARY_DIR}/include" "${CMAKE_SOURCE_DIR}/include" "${CMAKE_SOURCE_DIR}" "${CMAKE_SOURCE_DIR}/src" - "${CMAKE_SOURCE_DIR}/thirdparty/jitify" "${CMAKE_SOURCE_DIR}/thirdparty/dlpack/include" - "${CMAKE_SOURCE_DIR}/thirdparty/libcudacxx/include" "${GTEST_INCLUDE_DIR}" "${ARROW_INCLUDE_DIR}" "${FLATBUFFERS_INCLUDE_DIR}" diff --git a/python/cudf/setup.py b/python/cudf/setup.py index 8fcf45d7c0d..cad620952d2 100644 --- a/python/cudf/setup.py +++ b/python/cudf/setup.py @@ -31,6 +31,8 @@ cuda_include_dir = os.path.join(CUDA_HOME, "include") +CUDF_ROOT = os.environ.get("CUDF_ROOT", "../../cpp/build/") + try: nthreads = int(os.environ.get("PARALLEL_LEVEL", "0") or "0") except Exception: @@ -43,10 +45,12 @@ include_dirs=[ "../../cpp/include/cudf", "../../cpp/include", - "../../cpp/build/include", - "../../cpp/build/_deps/cub-src", - "../../cpp/build/_deps/thrust-src", - "../../thirdparty/libcudacxx/include", + os.path.join(CUDF_ROOT, "include"), + os.path.join(CUDF_ROOT, "_deps/libcudacxx-src/include"), + os.path.join( + os.path.dirname(sysconfig.get_path("include")), + "libcudf/libcudacxx", + ), os.path.dirname(sysconfig.get_path("include")), np.get_include(), cuda_include_dir, diff --git a/thirdparty/CMakeLists.txt b/thirdparty/CMakeLists.txt index 7f16a1df397..671dec44ae4 100644 --- a/thirdparty/CMakeLists.txt +++ b/thirdparty/CMakeLists.txt @@ -3,6 +3,8 @@ include(FetchContent) ################################################################################################### # - cub ------------------------------------------------------------------------------------------- +message("Fetching CUB") + FetchContent_Declare( cub GIT_REPOSITORY https://github.com/thrust/cub.git @@ -23,6 +25,8 @@ set(CUB_INCLUDE_DIR "${cub_SOURCE_DIR}" PARENT_SCOPE) ################################################################################################### # - thrust ---------------------------------------------------------------------------------------- +message("Fetching Thrust") + FetchContent_Declare( thrust GIT_REPOSITORY https://github.com/thrust/thrust.git @@ -36,3 +40,44 @@ if(NOT thrust_POPULATED) # We are not using the thrust CMake targets, so no need to call `add_subdirectory()`. endif() set(THRUST_INCLUDE_DIR "${thrust_SOURCE_DIR}" PARENT_SCOPE) + +################################################################################################### +# - jitify ---------------------------------------------------------------------------------------- + +message("Fetching Jitify") + +FetchContent_Declare( + jitify + GIT_REPOSITORY https://github.com/rapidsai/jitify.git + GIT_TAG cudf_0.15 + GIT_SHALLOW true +) + +FetchContent_GetProperties(jitify) +if(NOT jitify_POPULATED) + FetchContent_Populate(jitify) + # We are not using the jitify CMake targets, so no need to call `add_subdirectory()`. +endif() +set(JITIFY_INCLUDE_DIR "${jitify_SOURCE_DIR}" PARENT_SCOPE) + +################################################################################################### +# - libcudacxx ------------------------------------------------------------------------------------ + +message("Fetching libcudacxx") + +FetchContent_Declare( + libcudacxx + GIT_REPOSITORY https://github.com/rapidsai/thirdparty-freestanding.git + GIT_TAG cudf + GIT_SHALLOW true +) + +FetchContent_GetProperties(libcudacxx) +if(NOT libcudacxx_POPULATED) + FetchContent_Populate(libcudacxx) + # libcudacxx has no CMake targets, so no need to call `add_subdirectory()`. +endif() +set(LIBCUDACXX_DIR "${libcudacxx_SOURCE_DIR}" PARENT_SCOPE) +set(LIBCUDACXX_INCLUDE_DIR "${libcudacxx_SOURCE_DIR}/include/" PARENT_SCOPE) +set(LIBCXX_DIR "${libcudacxx_SOURCE_DIR}/libcxx/" PARENT_SCOPE) +set(LIBCXX_INCLUDE_DIR "${libcudacxx_SOURCE_DIR}/libcxx/include/" PARENT_SCOPE) diff --git a/thirdparty/jitify b/thirdparty/jitify deleted file mode 160000 index e3f867027c1..00000000000 --- a/thirdparty/jitify +++ /dev/null @@ -1 +0,0 @@ -Subproject commit e3f867027c1d9603b5a677795900465b9fac9cb8 diff --git a/thirdparty/libcudacxx b/thirdparty/libcudacxx deleted file mode 160000 index cdcda484d0c..00000000000 --- a/thirdparty/libcudacxx +++ /dev/null @@ -1 +0,0 @@ -Subproject commit cdcda484d0c7db114ea29c3b33429de5756ecfd8