Initial version of pylibcugraph source tree and build script updates (

rapidsai#1787)

Initial version of `pylibcugraph` source tree and build script updates plus connected_components placeholder.

Part of rapidsai#1782 

Tested by running the new tests part of this PR.  Also manually tested by building `pylibcugraph` importing it interactively, and confirming `pylibcugraph.weakly_connected_components` raise `NotImplementedError`

Note: there is likely still some cleanup to do in the various setup files, since those were copied as-is from cugraph and modified only to work with `pylibcugraph` dir names.

Authors:
  - Rick Ratzel (https://github.com/rlratzel)

Approvers:
  - Joseph Nke (https://github.com/jnke2016)
  - Brad Rees (https://github.com/BradReesWork)
  - https://github.com/Iroy30
  - Ray Douglass (https://github.com/raydouglass)

URL: rapidsai#1787

build.sh

@@ -19,13 +19,14 @@ ARGS=$*
 REPODIR=$(cd $(dirname $0); pwd)
 LIBCUGRAPH_BUILD_DIR=${LIBCUGRAPH_BUILD_DIR:=${REPODIR}/cpp/build}
 
-VALIDARGS="clean uninstall libcugraph cugraph cpp-mgtests docs -v -g -n --allgpuarch --buildfaiss --show_depr_warn --skip_cpp_tests -h --help"
+VALIDARGS="clean uninstall libcugraph cugraph pylibcugraph cpp-mgtests docs -v -g -n --allgpuarch --buildfaiss --show_depr_warn --skip_cpp_tests -h --help"
 HELP="$0 [<target> ...] [<flag> ...]
  where <target> is:
    clean            - remove all existing build artifacts and configuration (start over)
    uninstall        - uninstall libcugraph and cugraph from a prior build/install (see also -n)
    libcugraph       - build libcugraph.so and SG test binaries
    cugraph          - build the cugraph Python package
+   pylibcugraph     - build the pylibcugraph Python package
    cpp-mgtests      - build libcugraph MG tests. Builds MPI communicator, adding MPI as a dependency.
    docs             - build the docs
  and <flag> is:
@@ -125,11 +126,14 @@ if hasArg uninstall; then
     if [ -e ${LIBCUGRAPH_BUILD_DIR}/install_manifest.txt ]; then
         xargs rm -f < ${LIBCUGRAPH_BUILD_DIR}/install_manifest.txt > /dev/null 2>&1
     fi
-    # uninstall cugraph installed from a prior "setup.py install"
-    pip uninstall -y cugraph
+    # uninstall cugraph and pylibcugraph installed from a prior "setup.py
+    # install"
+    pip uninstall -y cugraph pylibcugraph
 fi
 
 if hasArg clean; then
+    # Ignore errors for clean since missing files, etc. are not failures
+    set +e
     # remove artifacts generated inplace
     # FIXME: ideally the "setup.py clean" command would be used for this, but
     # currently running any setup.py command has side effects (eg. cloning
@@ -154,6 +158,8 @@ if hasArg clean; then
 	    rmdir ${bd} || true
 	fi
     done
+    # Go back to failing on first error for all other operations
+    set -e
 fi
 
 ################################################################################
@@ -179,9 +185,22 @@ if buildAll || hasArg libcugraph; then
     cmake --build "${LIBCUGRAPH_BUILD_DIR}" -j${PARALLEL_LEVEL} --target ${INSTALL_TARGET} ${VERBOSE_FLAG}
 fi
 
+# Build, and install pylibcugraph
+if buildAll || hasArg pylibcugraph; then
+    cd ${REPODIR}/python/pylibcugraph
+    # setup.py references an env var CUGRAPH_BUILD_PATH to find the libcugraph
+    # build. If not set by the user, set it to LIBCUGRAPH_BUILD_DIR
+    CUGRAPH_BUILD_PATH=${CUGRAPH_BUILD_PATH:=${LIBCUGRAPH_BUILD_DIR}}
+    env CUGRAPH_BUILD_PATH=${CUGRAPH_BUILD_PATH} python setup.py build_ext --inplace --library-dir=${LIBCUGRAPH_BUILD_DIR}
+    if [[ ${INSTALL_TARGET} != "" ]]; then
+	env CUGRAPH_BUILD_PATH=${CUGRAPH_BUILD_PATH} python setup.py install
+    fi
+fi
+
 # Build and install the cugraph Python package
 if buildAll || hasArg cugraph; then
-    cd ${REPODIR}/python
+    cd ${REPODIR}/python/cugraph
+    # FIXME: this needs to eventually reference the pylibcugraph build
     # setup.py references an env var CUGRAPH_BUILD_PATH to find the libcugraph
     # build. If not set by the user, set it to LIBCUGRAPH_BUILD_DIR
     CUGRAPH_BUILD_PATH=${CUGRAPH_BUILD_PATH:=${LIBCUGRAPH_BUILD_DIR}}

ci/checks/copyright.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2019-2020, NVIDIA CORPORATION.
+# Copyright (c) 2019-2021, NVIDIA CORPORATION.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -41,6 +41,9 @@ def checkThisFile(f):
         return False
     if git_helpers and git_helpers.isFileEmpty(f):
         return False
+    # Special case for versioneer.py - it uses a separate copyright.
+    if os.path.basename(f) == "versioneer.py":
+        return False
     for checker in FilesToCheck:
         if checker.search(f):
             return True

ci/test.sh

@@ -79,7 +79,7 @@ else
 fi
 
 echo "Python pytest for cuGraph..."
-cd ${CUGRAPH_ROOT}/python
+cd ${CUGRAPH_ROOT}/python/cugraph
 pytest --cache-clear --junitxml=${CUGRAPH_ROOT}/junit-cugraph.xml -v --cov-config=.coveragerc --cov=cugraph --cov-report=xml:${WORKSPACE}/python/cugraph/cugraph-coverage.xml --cov-report term --ignore=cugraph/raft --benchmark-disable
 echo "Ran Python pytest for cugraph : return code was: $?, test script exit code is now: $EXITCODE"
 

python/.coveragerc

@@ -1,4 +1,7 @@
 # Configuration file for Python coverage tests
 [run]
-include = cugraph/*
-omit = cugraph/tests/*
+include = cugraph/cugraph/*
+          pylibcugraph/pylibcugraph/*
+omit = cugraph/cugraph/tests/*
+       pylibcugraph/pylibcugraph/tests/*
+

python/.gitattributes

@@ -1 +1,2 @@
-cugraph/_version.py export-subst
+cugraph/cugraph/_version.py export-subst
+pylibcugraph/pylibcugraph/_version.py export-subst

python/cugraph/_version.py → python/cugraph/cugraph/_version.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2018-2020, NVIDIA CORPORATION.
+# Copyright (c) 2018-2021, NVIDIA CORPORATION.
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at

python/cugraph/community/ecg.py → python/cugraph/cugraph/community/ecg.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2019-2020, NVIDIA CORPORATION.
+# Copyright (c) 2019-2021, NVIDIA CORPORATION.
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at

python/cugraph/community/ecg_wrapper.pyx → python/cugraph/cugraph/community/ecg_wrapper.pyx

@@ -1,4 +1,4 @@
-# Copyright (c) 2019-2020, NVIDIA CORPORATION.
+# Copyright (c) 2019-2021, NVIDIA CORPORATION.
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at

python/cugraph/components/__init__.py → python/cugraph/cugraph/components/__init__.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2019-2020, NVIDIA CORPORATION.
+# Copyright (c) 2019-2021, NVIDIA CORPORATION.
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at

python/cugraph/cores/__init__.py → python/cugraph/cugraph/cores/__init__.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2019-2020, NVIDIA CORPORATION.
+# Copyright (c) 2019-2021, NVIDIA CORPORATION.
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at

python/cugraph/cores/core_number.py → python/cugraph/cugraph/cores/core_number.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2019-2020, NVIDIA CORPORATION.
+# Copyright (c) 2019-2021, NVIDIA CORPORATION.
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at

python/cugraph/dask/common/mg_utils.py → python/cugraph/cugraph/dask/common/mg_utils.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2020, NVIDIA CORPORATION.
+# Copyright (c) 2020-2021, NVIDIA CORPORATION.
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at

python/cugraph/dask/common/read_utils.py → python/cugraph/cugraph/dask/common/read_utils.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2019-2020, NVIDIA CORPORATION.
+# Copyright (c) 2019-2021, NVIDIA CORPORATION.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

python/cugraph/dask/community/__init__.py → python/cugraph/cugraph/dask/community/__init__.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2020, NVIDIA CORPORATION.
+# Copyright (c) 2020-2021, NVIDIA CORPORATION.
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at

python/cugraph/internals/__init__.py → python/cugraph/cugraph/internals/__init__.py

@@ -1,5 +1,4 @@
-#
-# Copyright (c) 2020, NVIDIA CORPORATION.
+# Copyright (c) 2020-2021, NVIDIA CORPORATION.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -12,6 +11,5 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-#
 
 from cugraph.internals.internals import GraphBasedDimRedCallback

python/cugraph/internals/internals.pyx → python/cugraph/cugraph/internals/internals.pyx

@@ -1,5 +1,4 @@
-#
-# Copyright (c) 2020, NVIDIA CORPORATION.
+# Copyright (c) 2020-2021, NVIDIA CORPORATION.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -12,7 +11,6 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-#
 
 # cython: profile=False
 # distutils: language = c++

python/cugraph/layout/__init__.py → python/cugraph/cugraph/layout/__init__.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2019-2020, NVIDIA CORPORATION.
+# Copyright (c) 2019-2021, NVIDIA CORPORATION.
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at

python/cugraph/link_analysis/__init__.py → python/cugraph/cugraph/link_analysis/__init__.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2019-2020, NVIDIA CORPORATION.
+# Copyright (c) 2019-2021, NVIDIA CORPORATION.
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at

python/cugraph/link_analysis/hits.py → python/cugraph/cugraph/link_analysis/hits.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2019-2020, NVIDIA CORPORATION.
+# Copyright (c) 2019-2021, NVIDIA CORPORATION.
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at

python/cugraph/link_prediction/__init__.py → python/cugraph/cugraph/link_prediction/__init__.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2019-2020, NVIDIA CORPORATION.
+# Copyright (c) 2019-2021, NVIDIA CORPORATION.
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at

python/cugraph/proto/__init__.py → python/cugraph/cugraph/proto/__init__.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2019-2020, NVIDIA CORPORATION.
+# Copyright (c) 2019-2021, NVIDIA CORPORATION.
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
@@ -13,4 +13,3 @@
 
 from cugraph.proto.components import strong_connected_component
 from cugraph.proto.structure  import find_bicliques
-

python/cugraph/proto/components/__init__.py → python/cugraph/cugraph/proto/components/__init__.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2019-2020, NVIDIA CORPORATION.
+# Copyright (c) 2019-2021, NVIDIA CORPORATION.
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
@@ -12,4 +12,3 @@
 # limitations under the License.
 
 from cugraph.proto.components.scc import strong_connected_component
-

python/cugraph/proto/components/scc.py → python/cugraph/cugraph/proto/components/scc.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2019-2020, NVIDIA CORPORATION.
+# Copyright (c) 2019-2021, NVIDIA CORPORATION.
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at

python/cugraph/proto/structure/__init__.py → python/cugraph/cugraph/proto/structure/__init__.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2019-2020, NVIDIA CORPORATION.
+# Copyright (c) 2019-2021, NVIDIA CORPORATION.
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
@@ -12,4 +12,3 @@
 # limitations under the License.
 
 from cugraph.proto.structure.bicliques import find_bicliques
-

python/cugraph/proto/structure/bicliques.py → python/cugraph/cugraph/proto/structure/bicliques.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2019-2020, NVIDIA CORPORATION.
+# Copyright (c) 2019-2021, NVIDIA CORPORATION.
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
@@ -123,7 +123,7 @@ def find_bicliques(
             # summarize occurances
             ic = _count_features(feature_list, True)
 
-            goal = int(degree * support)  # NOQA 
+            goal = int(degree * support)  # NOQA
 
             # only get dst nodes with the same degree
             c = ic.query('count >= @goal')

python/cugraph/structure/graph_primtypes.pyx → python/cugraph/cugraph/structure/graph_primtypes.pyx

@@ -1,4 +1,4 @@
-# Copyright (c) 2020, NVIDIA CORPORATION.
+# Copyright (c) 2020-2021, NVIDIA CORPORATION.
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at

python/cugraph/structure/shuffle.py → python/cugraph/cugraph/structure/shuffle.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2020, NVIDIA CORPORATION.
+# Copyright (c) 2020-2021, NVIDIA CORPORATION.
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at

python/cugraph/tests/__init__.py → python/cugraph/cugraph/tests/__init__.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2019-2020, NVIDIA CORPORATION.
+# Copyright (c) 2019-2021, NVIDIA CORPORATION.
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at

python/cugraph/tests/dask/__init__.py → python/cugraph/cugraph/tests/dask/__init__.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2018-2020, NVIDIA CORPORATION.
+# Copyright (c) 2018-2021, NVIDIA CORPORATION.
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at

python/cugraph/tests/test_bfs_bsp.py → python/cugraph/cugraph/tests/test_bfs_bsp.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2019-2020, NVIDIA CORPORATION.
+# Copyright (c) 2019-2021, NVIDIA CORPORATION.
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at

python/cugraph/tests/test_raft.py → python/cugraph/cugraph/tests/test_raft.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2020, NVIDIA CORPORATION.
+# Copyright (c) 2020-2021, NVIDIA CORPORATION.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

python/cugraph/tests/test_symmetrize.py → python/cugraph/cugraph/tests/test_symmetrize.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2019-2020, NVIDIA CORPORATION.
+# Copyright (c) 2019-2021, NVIDIA CORPORATION.
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at

python/cugraph/tree/__init__.py → python/cugraph/cugraph/tree/__init__.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2020, NVIDIA CORPORATION.
+# Copyright (c) 2020-2021, NVIDIA CORPORATION.
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at

python/cugraph/tree/minimum_spanning_tree_wrapper.pyx → python/cugraph/cugraph/tree/minimum_spanning_tree_wrapper.pyx

@@ -1,4 +1,4 @@
-# Copyright (c) 2020, NVIDIA CORPORATION.
+# Copyright (c) 2020-2021, NVIDIA CORPORATION.
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at

python/cugraph/utilities/grmat.py → python/cugraph/cugraph/utilities/grmat.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2019-2020, NVIDIA CORPORATION.
+# Copyright (c) 2019-2021, NVIDIA CORPORATION.
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at

python/cugraph/utilities/pointer_utils.pyx → python/cugraph/cugraph/utilities/pointer_utils.pyx

@@ -1,4 +1,4 @@
-# Copyright (c) 2019-2020, NVIDIA CORPORATION.
+# Copyright (c) 2019-2021, NVIDIA CORPORATION.
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at

python/setup.cfg → python/cugraph/setup.cfg

@@ -1,4 +1,4 @@
-# Copyright (c) 2018-2020, NVIDIA CORPORATION.
+# Copyright (c) 2018-2021, NVIDIA CORPORATION.
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at

python/setup.py → python/cugraph/setup.py

@@ -116,8 +116,8 @@ def run(self):
               include_dirs=[
                   conda_include_dir,
                   ucx_include_dir,
-                  '../cpp/include',
-                  "../thirdparty/cub",
+                  "../../cpp/include",
+                  "../../thirdparty/cub",
                   raft_include_dir,
                   os.path.join(conda_include_dir, "libcudacxx"),
                   cuda_include_dir,
@@ -142,7 +142,7 @@ def run(self):
     )
 
 setup(name='cugraph',
-      description="cuGraph - GPU Graph Analytics",
+      description="cuGraph - RAPIDS GPU Graph Analytics",
       version=versioneer.get_version(),
       classifiers=[
           # "Development Status :: 4 - Beta",

python/setuputils.py → python/cugraph/setuputils.py

@@ -115,7 +115,7 @@ def use_raft_package(raft_path, cpp_build_path,
         raft_path, raft_cloned = \
             clone_repo_if_needed('raft', cpp_build_path,
                                  git_info_file=git_info_file)
-        raft_path = os.path.join('../', raft_path)
+        raft_path = os.path.join('../../', raft_path)
 
     raft_path = os.path.realpath(raft_path)
     print('-- RAFT found at: ' + str(raft_path))
@@ -151,8 +151,12 @@ def clone_repo_if_needed(name, cpp_build_path=None,
                                            git_info_file=git_info_file)
 
     if repo_cloned:
-        repo_path = (
-            _get_repo_path() + '/python/_external_repositories/' + name + '/')
+        # FIXME: should _external_repositories go in the "python" dir instead,
+        # to be shared by both packages?
+        repo_path = (_get_repo_path() +
+                     '/python/cugraph/_external_repositories/' +
+                     name +
+                     '/')
     else:
         repo_path = os.path.join(cpp_build_path, name + '-src/')
 
@@ -305,5 +309,5 @@ def get_repo_cmake_info(names, file_path):
 
 
 def _get_repo_path():
-    python_dir = Path(__file__).resolve()
+    python_dir = Path(__file__).resolve().parent
     return str(python_dir.parent.parent.absolute())