Experiment with CI

.flake8

@@ -13,3 +13,4 @@ per-file-ignores =
     nx_cugraph/tests/*.py:T201,
     __init__.py:F401,F403,
     _nx_cugraph/__init__.py:E501,
+    nx_cugraph/__init__.py:E402,F401,F403,

.github/workflows/add-to-project.yaml

@@ -17,4 +17,4 @@ jobs:
       - uses: actions/[email protected]
         with:
           project-url: https://github.com/orgs/rapidsai/projects/47
-          github-token: ${{ secrets.ADD_TO_PROJECT_GITHUB_TOKEN }}
+          github-token: ${{ secrets.ADD_TO_PROJECT_GITHUB_TOKEN }}

.github/workflows/labeler.yaml

@@ -8,4 +8,4 @@ jobs:
     steps:
     - uses: actions/labeler@v4
       with:
-        repo-token: "${{ secrets.GITHUB_TOKEN }}"
+        repo-token: "${{ secrets.GITHUB_TOKEN }}"

.github/workflows/pr.yaml

@@ -12,21 +12,63 @@ concurrency:
 jobs:
   pr-builder:
     needs:
+      - changed-files
       - checks
+      - conda-notebook-tests
+      - conda-python-build
+      - conda-python-tests
       - docs-build
-      - wheel-build-nx-cugraph
-      - wheel-tests-nx-cugraph
     secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
+    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
+    if: always()
+    with:
+      needs: ${{ toJSON(needs) }}
+  changed-files:
+    secrets: inherit
+    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
+    with:
+      files_yaml: |
+        test_notebooks:
+          - '**'
+          - '!.devcontainers/**'
+          - '!CONTRIBUTING.md'
+          - '!README.md'
+          - '!docs/**'
+          # TODO: Remove this before merging
+          - '!.github/**'
+        test_python:
+          - '**'
+          - '!.devcontainers/**'
+          - '!CONTRIBUTING.md'
+          - '!README.md'
+          - '!docs/**'
+          - '!img/**'
+          - '!notebooks/**'
+          # TODO: Remove this before merging
+          - '!.github/**'
   checks:
     secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@python-3.12
+    uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@branch-24.12
     with:
       enable_check_generated_files: false
+  conda-python-build:
+    needs: checks
+    secrets: inherit
+    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
+    with:
+      build_type: pull-request
+  conda-python-tests:
+    needs: [conda-python-build, changed-files]
+    secrets: inherit
+    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
+    if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python
+    with:
+      build_type: pull-request
   conda-notebook-tests:
-    needs: conda-python-build
+    needs: [conda-python-build, changed-files]
     secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
+    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
+    if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_notebooks
     with:
       build_type: pull-request
       node_type: "gpu-v100-latest-1"
@@ -36,24 +78,10 @@ jobs:
   docs-build:
     needs: conda-python-build
     secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@python-3.12
+    uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.12
     with:
       build_type: pull-request
       node_type: "gpu-v100-latest-1"
       arch: "amd64"
       container_image: "rapidsai/ci-conda:cuda11.8.0-ubuntu22.04-py3.10"
       run_script: "ci/build_docs.sh"
-  wheel-build-nx-cugraph:
-    needs: wheel-tests-pylibcugraph
-    secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
-    with:
-      build_type: pull-request
-      script: ci/build_wheel_nx-cugraph.sh
-  wheel-tests-nx-cugraph:
-    needs: wheel-build-nx-cugraph
-    secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
-    with:
-      build_type: pull-request
-      script: ci/test_wheel_nx-cugraph.sh

.gitignore

@@ -79,7 +79,7 @@ datasets/*
 !datasets/netscience.csv
 
 # nx-cugraph side effects
-python/nx-cugraph/objects.inv
+objects.inv
 
 .pydevproject
 

.pre-commit-config.yaml

@@ -63,8 +63,8 @@ repos:
     hooks:
       - id: nx-cugraph-meta-data-update
         name: nx-cugraph meta-data updater
-        entry: bash -c "PYTHONPATH=./python/nx-cugraph python ./python/nx-cugraph/_nx_cugraph/__init__.py"
-        files: ^python/nx-cugraph/
+        entry: bash -c "PYTHONPATH=. python _nx_cugraph/__init__.py"
+        files: ^(nx_cugraph|_nx_cugraph)/
         types: [python]
         language: python
         pass_filenames: false
@@ -73,8 +73,8 @@ repos:
     hooks:
       - id: nx-cugraph-readme-update
         name: nx-cugraph README updater
-        entry: bash -c "PYTHONPATH=./python/nx-cugraph python ./python/nx-cugraph/scripts/update_readme.py ./python/nx-cugraph/README.md"
-        files: ^python/nx-cugraph/
+        entry: bash -c "PYTHONPATH=. python scripts/update_readme.py README.md"
+        files: ^(nx_cugraph|_nx_cugraph)/
         types_or: [python, markdown]
         language: python
         pass_filenames: false

_nx_cugraph/VERSION

@@ -1 +1 @@
-../../../VERSION
+../VERSION

build.sh

@@ -159,8 +159,8 @@ fi
 # Build and install the nx-cugraph Python package
 if buildDefault || hasArg nx-cugraph || hasArg all; then
     if hasArg --clean; then
-        cleanPythonDir ${REPODIR}/python/nx-cugraph
+        cleanPythonDir ${REPODIR}
     else
-        python ${PYTHON_ARGS_FOR_INSTALL} ${REPODIR}/python/nx-cugraph
+        python ${PYTHON_ARGS_FOR_INSTALL} ${REPODIR}
     fi
 fi

ci/notebook_list.py

@@ -0,0 +1,105 @@
+# Copyright (c) 2021-2024, NVIDIA CORPORATION.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import re
+import argparse
+import sys
+import glob
+from pathlib import Path
+
+from numba import cuda
+
+# for adding another run type and skip file name add to this dictionary
+runtype_dict = {
+    "all": "",
+    "ci": "SKIP_CI_TESTING",
+    "nightly": "SKIP_NIGHTLY",
+    "weekly": "SKIP_WEEKLY",
+}
+
+
+def skip_book_dir(runtype):
+    # Add all run types here, currently only CI supported
+
+    if runtype in runtype_dict.keys():
+        if Path(runtype_dict.get(runtype)).is_file():
+            return True
+    return False
+
+
+cuda_version_string = ".".join([str(n) for n in cuda.runtime.get_version()])
+#
+# Not strictly true... however what we mean is
+# Pascal or earlier
+#
+ampere = False
+device = cuda.get_current_device()
+
+parser = argparse.ArgumentParser(description="Condition for running the notebook tests")
+parser.add_argument("runtype", type=str)
+
+args = parser.parse_args()
+
+runtype = args.runtype
+
+if runtype not in runtype_dict.keys():
+    print(f"Unknown Run Type  = {runtype}", file=sys.stderr)
+    exit()
+
+
+# check for the attribute using both pre and post numba 0.53 names
+cc = getattr(device, "COMPUTE_CAPABILITY", None) or getattr(
+    device, "compute_capability"
+)
+if cc[0] >= 8:
+    ampere = True
+
+skip = False
+for filename in glob.iglob("**/*.ipynb", recursive=True):
+    skip = False
+    if skip_book_dir(runtype):
+        print(
+            f"SKIPPING {filename} (Notebook skipped for run type {runtype}) "
+            "due to skip file in folder.",
+            file=sys.stderr,
+        )
+        skip = True
+    else:
+        for line in open(filename, "r"):
+            if re.search("# Skip notebook test", line):
+                skip = True
+                print(f"SKIPPING {filename} (marked as skip)", file=sys.stderr)
+                break
+            elif re.search("dask", line):
+                print(
+                    f"SKIPPING {filename} (suspected Dask usage, not "
+                    "currently automatable)",
+                    file=sys.stderr,
+                )
+                skip = True
+                break
+            elif ampere and re.search("# Does not run on Ampere", line):
+                print(f"SKIPPING {filename} (does not run on Ampere)", file=sys.stderr)
+                skip = True
+                break
+            elif re.search("# Does not run on CUDA ", line) and (
+                cuda_version_string in line
+            ):
+                print(
+                    f"SKIPPING {filename} (does not run on CUDA {cuda_version_string})",
+                    file=sys.stderr,
+                )
+                skip = True
+                break
+    if not skip:
+        print(filename)

ci/run_nx_cugraph_pytests.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+# Copyright (c) 2024, NVIDIA CORPORATION.
+
+set -euo pipefail
+
+# Support invoking run_nx_cugraph_pytests.sh outside the script directory
+cd "$(dirname "$(realpath "${BASH_SOURCE[0]}")")"/../nx_cugraph
+
+NX_CUGRAPH_USE_COMPAT_GRAPHS=False pytest --capture=no --cache-clear --benchmark-disable "$@" tests
+NX_CUGRAPH_USE_COMPAT_GRAPHS=True pytest --capture=no --cache-clear --benchmark-disable "$@" tests

ci/test_notebooks.sh

@@ -0,0 +1,66 @@
+#!/bin/bash
+# Copyright (c) 2020-2024, NVIDIA CORPORATION.
+
+set -Eeuo pipefail
+
+. /opt/conda/etc/profile.d/conda.sh
+
+RAPIDS_VERSION_MAJOR_MINOR="$(rapids-version-major-minor)"
+
+rapids-logger "Generate notebook testing dependencies"
+rapids-dependency-file-generator \
+  --output conda \
+  --file-key test_notebooks \
+  --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee env.yaml
+
+rapids-mamba-retry env create --yes -f env.yaml -n test
+
+# Temporarily allow unbound variables for conda activation.
+set +u
+conda activate test
+set -u
+
+rapids-print-env
+
+rapids-logger "Downloading artifacts from previous jobs"
+CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp)
+PYTHON_CHANNEL=$(rapids-download-conda-from-s3 python)
+
+rapids-mamba-retry install \
+  --channel "${CPP_CHANNEL}" \
+  --channel "${PYTHON_CHANNEL}" \
+  "libcugraph=${RAPIDS_VERSION_MAJOR_MINOR}.*" \
+  "pylibcugraph=${RAPIDS_VERSION_MAJOR_MINOR}.*" \
+  "nx-cugraph=${RAPIDS_VERSION_MAJOR_MINOR}.*"
+
+NBTEST="$(realpath "$(dirname "$0")/utils/nbtest.sh")"
+NOTEBOOK_LIST="$(realpath "$(dirname "$0")/notebook_list.py")"
+EXITCODE=0
+trap "EXITCODE=1" ERR
+
+
+pushd notebooks
+TOPLEVEL_NB_FOLDERS="$(find . -name "*.ipynb" | cut -d'/' -f2 | sort -u)"
+set +e
+# Always run nbtest in all TOPLEVEL_NB_FOLDERS, set EXITCODE to failure
+# if any run fails
+for folder in ${TOPLEVEL_NB_FOLDERS}; do
+    rapids-logger "Folder: ${folder}"
+    pushd "${folder}"
+    NBLIST=$(python "${NOTEBOOK_LIST}" ci)
+    for nb in ${NBLIST}; do
+        nbBasename=$(basename "${nb}")
+        pushd "$(dirname "${nb}")"
+        nvidia-smi
+        ${NBTEST} "${nbBasename}"
+        echo "Ran nbtest for $nb : return code was: $?, test script exit code is now: $EXITCODE"
+        echo
+        popd
+    done
+    popd
+done
+
+nvidia-smi
+
+echo "Notebook test script exiting with value: ${EXITCODE}"
+exit ${EXITCODE}