Update cugraph python build (rapidsai#2378)

This PR updates cugraph python build to use scikit-build instead of setuptools. 
Scikit-build leverages cmake to build the python extension

closes rapidsai#2333

Authors:
  - Joseph Nke (https://github.com/jnke2016)
  - Chuck Hastings (https://github.com/ChuckHastings)
  - Vyas Ramasubramani (https://github.com/vyasr)

Approvers:
  - Brad Rees (https://github.com/BradReesWork)
  - Sevag H (https://github.com/sevagh)
  - Rick Ratzel (https://github.com/rlratzel)
  - Vyas Ramasubramani (https://github.com/vyasr)

URL: rapidsai#2378

.gitignore

@@ -29,11 +29,15 @@ junit-cugraph.xml
 test-results
 
 ## Python build directories & artifacts
+dask-worker-space/
 htmlcov
 dist/
 cugraph.egg-info/
 python/build
 python/cugraph/bindings/*.cpp
+wheels/
+_skbuild/
+cufile.log
 
 ## pylibcugraph build directories & artifacts
 python/pylibcugraph/pylibcugraph.egg-info

build.sh

@@ -159,6 +159,8 @@ if hasArg clean; then
         pushd ${REPODIR}/python > /dev/null
         rm -rf dist dask-worker-space cugraph/raft *.egg-info
         find . -name "__pycache__" -type d -exec rm -rf {} \; > /dev/null 2>&1
+        find . -type d -name _skbuild -exec rm -rf {} \; > /dev/null 2>&1
+        find . -type d -name dist -exec rm -rf {} \; > /dev/null 2>&1
         find . -name "*.cpp" -type f -delete
         find . -name "*.cpython*.so" -type f -delete
         find . -type d -name _external_repositories -exec rm -rf {} \; > /dev/null 2>&1
@@ -230,7 +232,8 @@ if buildAll || hasArg pylibcugraph; then
     # setup.py references an env var CUGRAPH_BUILD_PATH to find the libcugraph
     # build. If not set by the user, set it to LIBCUGRAPH_BUILD_DIR
     CUGRAPH_BUILD_PATH=${CUGRAPH_BUILD_PATH:=${LIBCUGRAPH_BUILD_DIR}}
-    env CUGRAPH_BUILD_PATH=${CUGRAPH_BUILD_PATH} python setup.py build_ext --inplace --library-dir=${LIBCUGRAPH_BUILD_DIR}
+    python setup.py build_ext --inplace -- -DFIND_CUGRAPH_CPP=ON \
+           -Dcugraph_ROOT=${LIBCUGRAPH_BUILD_DIR} -- -j${PARALLEL_LEVEL:-1}
     if [[ ${INSTALL_TARGET} != "" ]]; then
 	env CUGRAPH_BUILD_PATH=${CUGRAPH_BUILD_PATH} python setup.py install
     fi
@@ -243,7 +246,8 @@ if buildAll || hasArg cugraph; then
     # setup.py references an env var CUGRAPH_BUILD_PATH to find the libcugraph
     # build. If not set by the user, set it to LIBCUGRAPH_BUILD_DIR
     CUGRAPH_BUILD_PATH=${CUGRAPH_BUILD_PATH:=${LIBCUGRAPH_BUILD_DIR}}
-    env CUGRAPH_BUILD_PATH=${CUGRAPH_BUILD_PATH} python setup.py build_ext --inplace --library-dir=${LIBCUGRAPH_BUILD_DIR}
+    python setup.py build_ext --inplace -- -DFIND_CUGRAPH_CPP=ON \
+           -Dcugraph_ROOT=${LIBCUGRAPH_BUILD_DIR} -- -j${PARALLEL_LEVEL:-1}
     if [[ ${INSTALL_TARGET} != "" ]]; then
 	env CUGRAPH_BUILD_PATH=${CUGRAPH_BUILD_PATH} python setup.py install
     fi

ci/release/update-version.sh

@@ -39,6 +39,9 @@ function sed_runner() {
     sed -i.bak ''"$1"'' $2 && rm -f ${2}.bak
 }
 
+# rapids-cmake version
+sed_runner 's/'"branch-.*\/RAPIDS.cmake"'/'"branch-${NEXT_SHORT_TAG}\/RAPIDS.cmake"'/g' fetch_rapids.cmake
+
 # CMakeLists update
 sed_runner 's/'"CUGRAPH VERSION .* LANGUAGES C CXX CUDA)"'/'"CUGRAPH VERSION ${NEXT_FULL_TAG} LANGUAGES C CXX CUDA)"'/g' cpp/CMakeLists.txt
 sed_runner 's|'"branch-.*/RAPIDS.cmake"'|'"branch-${NEXT_SHORT_TAG}/RAPIDS.cmake"'|g' cpp/CMakeLists.txt

conda/environments/cugraph_dev_cuda11.2.yml

@@ -28,6 +28,7 @@ dependencies:
 - clang=11.1.0
 - clang-tools=11.1.0
 - cmake>=3.20.1,!=3.23.0
+- scikit-build>=0.13.1
 - python>=3.8,<3.10
 - notebook>=0.5.0
 - boost

conda/environments/cugraph_dev_cuda11.4.yml

@@ -28,6 +28,7 @@ dependencies:
 - clang=11.1.0
 - clang-tools=11.1.0
 - cmake>=3.20.1,!=3.23.0
+- scikit-build>=0.13.1
 - python>=3.8,<3.10
 - notebook>=0.5.0
 - boost

conda/environments/cugraph_dev_cuda11.5.yml

@@ -28,6 +28,7 @@ dependencies:
 - clang=11.1.0
 - clang-tools=11.1.0
 - cmake>=3.20.1,!=3.23.0
+- scikit-build>=0.13.1
 - python>=3.8,<3.10
 - notebook>=0.5.0
 - boost

conda/recipes/cugraph/conda_build_config.yaml

@@ -7,5 +7,8 @@ cxx_compiler_version:
 cuda_compiler:
   - nvcc
 
+cmake_version:
+  - ">=3.20.1,!=3.23.0"
+
 sysroot_version:
   - "2.17"

conda/recipes/cugraph/meta.yaml

@@ -25,13 +25,15 @@ build:
 
 requirements:
   build:
+    - cmake {{ cmake_version }}
     - {{ compiler('c') }}
     - {{ compiler('cxx') }}
     - {{ compiler('cuda') }} {{ cuda_version }}
     - sysroot_{{ target_platform }} {{ sysroot_version }}
   host:
     - python x.x
     - cython>=0.29,<0.30
+    - scikit-build>=0.13.1
     - libcugraph={{ version }}
     - libraft-headers {{ minor_version }}
     - pyraft {{ minor_version }}

conda/recipes/pylibcugraph/conda_build_config.yaml

@@ -7,5 +7,8 @@ cxx_compiler_version:
 cuda_compiler:
   - nvcc
 
+cmake_version:
+  - ">=3.20.1,!=3.23.0"
+
 sysroot_version:
   - "2.17"

conda/recipes/pylibcugraph/meta.yaml

@@ -25,13 +25,15 @@ build:
 
 requirements:
   build:
+    - cmake {{ cmake_version }}
     - {{ compiler('c') }}
     - {{ compiler('cxx') }}
     - {{ compiler('cuda') }} {{ cuda_version }}
     - sysroot_{{ target_platform }} {{ sysroot_version }}
   host:
     - python x.x
     - cython>=0.29,<0.30
+    - scikit-build>=0.13.1
     - libcugraph={{ version }}
     - ucx-py {{ ucx_py_version }}
     - ucx-proc=*=gpu

cpp/CMakeLists.txt

@@ -15,10 +15,8 @@
 #=============================================================================
 
 cmake_minimum_required(VERSION 3.20.1 FATAL_ERROR)
-file(DOWNLOAD https://raw.githubusercontent.com/rapidsai/rapids-cmake/branch-22.08/RAPIDS.cmake
-    ${CMAKE_BINARY_DIR}/RAPIDS.cmake)
-include(${CMAKE_BINARY_DIR}/RAPIDS.cmake)
 
+include(../fetch_rapids.cmake)
 include(rapids-cmake)
 include(rapids-cpm)
 include(rapids-cuda)

cpp/cmake/thirdparty/get_libcudacxx.cmake

@@ -16,8 +16,7 @@
 function(find_and_configure_libcudacxx)
   include(${rapids-cmake-dir}/cpm/libcudacxx.cmake)
 
-  rapids_cpm_libcudacxx(BUILD_EXPORT_SET cugraph-exports
-                        INSTALL_EXPORT_SET cugraph-exports)
+  rapids_cpm_libcudacxx(BUILD_EXPORT_SET cugraph-exports)
 
 endfunction()
 

fetch_rapids.cmake

@@ -0,0 +1,17 @@
+# =============================================================================
+# Copyright (c) 2022, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
+# in compliance with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+# or implied. See the License for the specific language governing permissions and limitations under
+# the License.
+# =============================================================================
+file(DOWNLOAD https://raw.githubusercontent.com/rapidsai/rapids-cmake/branch-22.08/RAPIDS.cmake
+     ${CMAKE_BINARY_DIR}/RAPIDS.cmake
+)
+include(${CMAKE_BINARY_DIR}/RAPIDS.cmake)

python/cugraph/CMakeLists.txt

@@ -0,0 +1,84 @@
+# =============================================================================
+# Copyright (c) 2022, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
+# in compliance with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+# or implied. See the License for the specific language governing permissions and limitations under
+# the License.
+# =============================================================================
+
+cmake_minimum_required(VERSION 3.20.1 FATAL_ERROR)
+
+set(cugraph_version 22.08.00)
+
+include(../../fetch_rapids.cmake)
+
+project(
+  cugraph-python
+  VERSION ${cugraph_version}
+  LANGUAGES # TODO: Building Python extension modules via the python_extension_module requires the C
+            # language to be enabled here. The test project that is built in scikit-build to verify
+            # various linking options for the python library is hardcoded to build with C, so until
+            # that is fixed we need to keep C.
+            C CXX
+)
+
+################################################################################
+# - User Options  --------------------------------------------------------------
+option(FIND_CUGRAPH_CPP "Search for existing CUGRAPH C++ installations before defaulting to local files"
+       OFF
+)
+
+# If the user requested it,  we attempt to find CUGRAPH.
+if(FIND_CUGRAPH_CPP)
+  find_package(cugraph ${cugraph_version} REQUIRED)
+else()
+  set(cugraph_FOUND OFF)
+endif()
+
+if(NOT cugraph_FOUND)
+  # TODO: This will not be necessary once we upgrade to CMake 3.22, which will pull in the required
+  # languages for the C++ project even if this project does not require those languges.
+  include(rapids-cuda)
+  rapids_cuda_init_architectures(CUGRAPH)
+  enable_language(CUDA)
+
+  # Since cugraph only enables CUDA optionally, we need to manually include the file that
+  # rapids_cuda_init_architectures relies on `project` including.
+
+  include("${CMAKE_PROJECT_cugraph-python_INCLUDE}")
+
+  add_subdirectory(../../cpp cugraph-cpp)
+
+  install(TARGETS cugraph DESTINATION cugraph/library)
+endif()
+
+
+include(rapids-cython)
+rapids_cython_init()
+
+add_subdirectory(cugraph/centrality)
+add_subdirectory(cugraph/community)
+add_subdirectory(cugraph/components)
+add_subdirectory(cugraph/cores)
+add_subdirectory(cugraph/dask/centrality)
+add_subdirectory(cugraph/dask/comms)
+add_subdirectory(cugraph/dask/community)
+add_subdirectory(cugraph/dask/components)
+add_subdirectory(cugraph/dask/link_analysis)
+add_subdirectory(cugraph/dask/structure)
+add_subdirectory(cugraph/generators)
+add_subdirectory(cugraph/internals)
+add_subdirectory(cugraph/layout)
+add_subdirectory(cugraph/linear_assignment)
+add_subdirectory(cugraph/link_analysis)
+add_subdirectory(cugraph/link_prediction)
+add_subdirectory(cugraph/sampling)
+add_subdirectory(cugraph/structure)
+add_subdirectory(cugraph/tree)
+add_subdirectory(cugraph/utilities)

python/cugraph/cugraph/centrality/CMakeLists.txt

@@ -0,0 +1,29 @@
+# =============================================================================
+# Copyright (c) 2022, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
+# in compliance with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+# or implied. See the License for the specific language governing permissions and limitations under
+# the License.
+# =============================================================================
+
+set(cython_sources
+    betweenness_centrality_wrapper.pyx
+    edge_betweenness_centrality_wrapper.pyx
+)
+set(linked_libraries cugraph::cugraph)
+
+rapids_cython_create_modules(
+  CXX
+  SOURCE_FILES "${cython_sources}"
+  LINKED_LIBRARIES "${linked_libraries}" MODULE_PREFIX centrality_
+)
+
+foreach(cython_module IN LISTS RAPIDS_CYTHON_CREATED_TARGETS)
+  set_target_properties(${cython_module} PROPERTIES INSTALL_RPATH "\$ORIGIN;\$ORIGIN/../library")
+endforeach()

python/cugraph/cugraph/community/CMakeLists.txt

@@ -0,0 +1,34 @@
+# =============================================================================
+# Copyright (c) 2022, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
+# in compliance with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+# or implied. See the License for the specific language governing permissions and limitations under
+# the License.
+# =============================================================================
+
+set(cython_sources
+    ecg_wrapper.pyx egonet_wrapper.pyx
+    ktruss_subgraph_wrapper.pyx
+    leiden_wrapper.pyx
+    louvain_wrapper.pyx
+    spectral_clustering_wrapper.pyx
+    subgraph_extraction_wrapper.pyx
+    triangle_count_wrapper.pyx
+)
+
+set(linked_libraries cugraph::cugraph)
+rapids_cython_create_modules(
+  CXX
+  SOURCE_FILES "${cython_sources}"
+  LINKED_LIBRARIES "${linked_libraries}" MODULE_PREFIX community_
+)
+
+foreach(cython_module IN LISTS RAPIDS_CYTHON_CREATED_TARGETS)
+  set_target_properties(${cython_module} PROPERTIES INSTALL_RPATH "\$ORIGIN;\$ORIGIN/../library")
+endforeach()

python/cugraph/cugraph/components/CMakeLists.txt

@@ -0,0 +1,25 @@
+# =============================================================================
+# Copyright (c) 2022, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
+# in compliance with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+# or implied. See the License for the specific language governing permissions and limitations under
+# the License.
+# =============================================================================
+
+set(cython_sources connectivity_wrapper.pyx)
+set(linked_libraries cugraph::cugraph)
+rapids_cython_create_modules(
+  CXX
+  SOURCE_FILES "${cython_sources}"
+  LINKED_LIBRARIES "${linked_libraries}" MODULE_PREFIX components_
+)
+
+foreach(cython_module IN LISTS RAPIDS_CYTHON_CREATED_TARGETS)
+  set_target_properties(${cython_module} PROPERTIES INSTALL_RPATH "\$ORIGIN;\$ORIGIN/../library")
+endforeach()

python/cugraph/cugraph/cores/CMakeLists.txt

@@ -0,0 +1,25 @@
+# =============================================================================
+# Copyright (c) 2022, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
+# in compliance with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+# or implied. See the License for the specific language governing permissions and limitations under
+# the License.
+# =============================================================================
+
+set(cython_sources k_core_wrapper.pyx)
+set(linked_libraries cugraph::cugraph)
+rapids_cython_create_modules(
+  CXX
+  SOURCE_FILES "${cython_sources}"
+  LINKED_LIBRARIES "${linked_libraries}" MODULE_PREFIX cores_
+)
+
+foreach(cython_module IN LISTS RAPIDS_CYTHON_CREATED_TARGETS)
+  set_target_properties(${cython_module} PROPERTIES INSTALL_RPATH "\$ORIGIN;\$ORIGIN/../library")
+endforeach()

python/cugraph/cugraph/dask/centrality/CMakeLists.txt

@@ -0,0 +1,25 @@
+# =============================================================================
+# Copyright (c) 2022, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
+# in compliance with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+# or implied. See the License for the specific language governing permissions and limitations under
+# the License.
+# =============================================================================
+
+set(cython_sources mg_katz_centrality_wrapper.pyx)
+set(linked_libraries cugraph::cugraph)
+rapids_cython_create_modules(
+  CXX
+  SOURCE_FILES "${cython_sources}"
+  LINKED_LIBRARIES "${linked_libraries}" MODULE_PREFIX centrality_
+)
+
+foreach(cython_module IN LISTS RAPIDS_CYTHON_CREATED_TARGETS)
+  set_target_properties(${cython_module} PROPERTIES INSTALL_RPATH "\$ORIGIN;\$ORIGIN/../../library")
+endforeach()