add CI workflows running tests

.github/workflows/build.yaml

@@ -38,7 +38,7 @@ jobs:
       sha: ${{ inputs.sha }}
       date: ${{ inputs.date }}
       script: ci/build_wheel_nx-cugraph.sh
-      # This selects "ARCH=amd64 + the latest supported Python + CUDA".
+      # This selects "ARCH=amd64 + the latest supported Python, 1 job per major CUDA version".
       matrix_filter: map(select(.ARCH == "amd64")) | group_by(.CUDA_VER|split(".")|map(tonumber)|.[0]) | map(max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))]))
   wheel-publish-nx-cugraph:
     needs: wheel-build-nx-cugraph

.github/workflows/pr.yaml

@@ -15,7 +15,9 @@ jobs:
       - changed-files
       - checks
       - conda-python-build
+      - conda-python-tests
       - wheel-build-nx-cugraph
+      - wheel-tests-nx-cugraph
     secrets: inherit
     uses: rapidsai/shared-workflows/.github/workflows/[email protected]
     if: always()
@@ -46,14 +48,30 @@ jobs:
     uses: rapidsai/shared-workflows/.github/workflows/[email protected]
     with:
       build_type: pull-request
-      # This selects "ARCH=amd64 + the latest supported Python + CUDA".
-      matrix_filter: map(select(.ARCH == "amd64")) | group_by(.CUDA_VER|split(".")|map(tonumber)|.[0]) | map(max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))]))
+  conda-python-tests:
+    needs: [conda-python-build, changed-files]
+    secrets: inherit
+    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
+    if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python
+    with:
+      build_type: pull-request
+      run_codecov: false
   wheel-build-nx-cugraph:
     needs: [checks]
     secrets: inherit
     uses: rapidsai/shared-workflows/.github/workflows/[email protected]
     with:
       build_type: pull-request
       script: ci/build_wheel_nx-cugraph.sh
-      # This selects "ARCH=amd64 + the latest supported Python + CUDA".
+      # This selects "ARCH=amd64 + the latest supported Python, 1 job per major CUDA version".
+      matrix_filter: map(select(.ARCH == "amd64")) | group_by(.CUDA_VER|split(".")|map(tonumber)|.[0]) | map(max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))]))
+  wheel-tests-nx-cugraph:
+    needs: [wheel-build-nx-cugraph, changed-files]
+    secrets: inherit
+    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
+    if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python
+    with:
+      build_type: pull-request
+      script: ci/test_wheel_nx-cugraph.sh
+      # This selects "ARCH=amd64 + the latest supported Python, 1 job per major CUDA version".
       matrix_filter: map(select(.ARCH == "amd64")) | group_by(.CUDA_VER|split(".")|map(tonumber)|.[0]) | map(max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))]))

.github/workflows/test.yaml

@@ -22,6 +22,9 @@ jobs:
       branch: ${{ inputs.branch }}
       date: ${{ inputs.date }}
       sha: ${{ inputs.sha }}
+      # This selects "ARCH=amd64 + the latest supported Python, 1 job per major CUDA version".
+      matrix_filter: map(select(.ARCH == "amd64")) | group_by(.CUDA_VER|split(".")|map(tonumber)|.[0]) | map(max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))]))
+      run_codecov: false
   wheel-tests-nx-cugraph:
     secrets: inherit
     uses: rapidsai/shared-workflows/.github/workflows/[email protected]
@@ -31,3 +34,5 @@ jobs:
       date: ${{ inputs.date }}
       sha: ${{ inputs.sha }}
       script: ci/test_wheel_nx-cugraph.sh
+      # This selects "ARCH=amd64 + the latest supported Python, 1 job per major CUDA version".
+      matrix_filter: map(select(.ARCH == "amd64")) | group_by(.CUDA_VER|split(".")|map(tonumber)|.[0]) | map(max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))]))

.pre-commit-config.yaml

@@ -1,8 +1,8 @@
 ## https://pre-commit.com/
 #
 # Before first use: `pre-commit install`
-# To run: `make lint`
-# To update: `make lint-update`
+# To run: `pre-commit run --all-files`
+# To update: `pre-commit autoupdate`
 #  - &flake8_dependencies below needs updated manually
 exclude: '^thirdparty'
 fail_fast: false
@@ -24,8 +24,6 @@ repos:
       - id: end-of-file-fixer
         exclude_types: [svg]
       - id: mixed-line-ending
-      - id: no-commit-to-branch
-        args: [-p, "^branch-2....$"]
       - id: trailing-whitespace
   - repo: https://github.com/PyCQA/flake8
     rev: 7.1.1
@@ -35,7 +33,7 @@ repos:
         additional_dependencies: &flake8_dependencies
           # These versions need updated manually
           - flake8==7.1.1
-          - flake8-bugbear==24.8.19
+          - flake8-bugbear==24.10.31
           - flake8-simplify==0.21.0
   - repo: https://github.com/asottile/yesqa
     rev: v1.5.0
@@ -95,7 +93,7 @@ repos:
       - id: nx-cugraph-meta-data-update
         name: nx-cugraph meta-data updater
         entry: bash -c "PYTHONPATH=. python _nx_cugraph/__init__.py"
-        files: ^nx_cugraph
+        files: ^(nx_cugraph|_nx_cugraph)/
         types: [python]
         language: python
         pass_filenames: false
@@ -105,8 +103,13 @@ repos:
       - id: nx-cugraph-readme-update
         name: nx-cugraph README updater
         entry: bash -c "PYTHONPATH=. python ./scripts/update_readme.py ./README.md"
-        files: ^nx_cugraph/
+        files: ^(nx_cugraph|_nx_cugraph)/
         types_or: [python, markdown]
         language: python
         pass_filenames: false
-        additional_dependencies: ["networkx>=3.3"]
+        additional_dependencies: ["networkx>=3.4"]
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v5.0.0
+    hooks:
+      - id: no-commit-to-branch
+        args: [-p, "^branch-2....$"]

_nx_cugraph/__init__.py

@@ -343,14 +343,16 @@ def update_env_var(varname):
     return d
 
 
-def _check_networkx_version() -> tuple[int, int]:
+def _check_networkx_version() -> tuple[int, int] | tuple[int, int, int]:
     """Check the version of networkx and return ``(major, minor)`` version tuple."""
     import re
     import warnings
 
     import networkx as nx
 
-    version_major, version_minor = nx.__version__.split(".")[:2]
+    version_major, version_minor, *version_bug = nx.__version__.split(".")[:3]
+    if has_bug := bool(version_bug):
+        version_bug = version_bug[0]
     if version_major != "3":
         warnings.warn(
             f"nx-cugraph version {__version__} is only known to work with networkx "
@@ -363,15 +365,19 @@ def _check_networkx_version() -> tuple[int, int]:
     # Allow single-digit minor versions, e.g. 3.4 and release candidates, e.g. 3.4rc0
     pattern = r"^\d(rc\d+)?$"
 
-    if not re.match(pattern, version_minor):
+    if not re.match(pattern, version_bug if has_bug else version_minor):
         raise RuntimeWarning(
             f"nx-cugraph version {__version__} does not work with networkx version "
             f"{nx.__version__}. Please upgrade (or fix) your Python environment."
         )
 
     nxver_major = int(version_major)
-    nxver_minor = int(re.match(r"^\d+", version_minor).group())
-    return (nxver_major, nxver_minor)
+    if not has_bug:
+        nxver_minor = int(re.match(r"^\d+", version_minor).group())
+        return (nxver_major, nxver_minor)
+    nxver_minor = int(version_minor)
+    nxver_bug = int(re.match(r"^\d+", version_bug).group())
+    return (nxver_major, nxver_minor, nxver_bug)
 
 
 if __name__ == "__main__":

ci/build_python.sh

@@ -9,7 +9,11 @@ source rapids-date-string
 
 rapids-print-env
 
-rapids-generate-version > ./VERSION
+# TODO: revert this once we start publishing nightly packages from the
+#       'nx-cugraph' repo and stop publishing them from the 'cugraph' repo
+# rapids-generate-version > ./VERSION
+echo "24.12.00a1000" > ./VERSION
+
 rapids-logger "Begin py build"
 
 # TODO: Remove `--no-test` flags once importing on a CPU

ci/run_nx_cugraph_pytests.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+# Copyright (c) 2024, NVIDIA CORPORATION.
+
+set -euo pipefail
+
+# Support invoking run_nx_cugraph_pytests.sh outside the script directory
+cd "$(dirname "$(realpath "${BASH_SOURCE[0]}")")"/..
+
+# Only be verbose and display print information for the first pytest command here
+NX_CUGRAPH_USE_COMPAT_GRAPHS=False pytest --capture=no --verbose --cache-clear --benchmark-disable "$@" ./nx_cugraph/tests
+NX_CUGRAPH_USE_COMPAT_GRAPHS=True pytest --cache-clear --benchmark-disable "$@" ./nx_cugraph/tests

ci/test_python.sh

@@ -0,0 +1,101 @@
+#!/bin/bash
+# Copyright (c) 2022-2024, NVIDIA CORPORATION.
+
+set -euo pipefail
+
+# Support invoking test_python.sh outside the script directory
+cd "$(dirname "$(realpath "${BASH_SOURCE[0]}")")"/../
+
+. /opt/conda/etc/profile.d/conda.sh
+
+RAPIDS_VERSION="$(rapids-version)"
+
+rapids-logger "Generate Python testing dependencies"
+rapids-dependency-file-generator \
+  --output conda \
+  --file-key test_python \
+  --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee env.yaml
+
+rapids-mamba-retry env create --yes -f env.yaml -n test
+
+# Temporarily allow unbound variables for conda activation.
+set +u
+conda activate test
+set -u
+
+rapids-logger "Downloading artifacts from previous jobs"
+PYTHON_CHANNEL=$(rapids-download-conda-from-s3 python)
+
+RAPIDS_TESTS_DIR=${RAPIDS_TESTS_DIR:-"${PWD}/test-results"}
+RAPIDS_COVERAGE_DIR=${RAPIDS_COVERAGE_DIR:-"${PWD}/coverage-results"}
+mkdir -p "${RAPIDS_TESTS_DIR}" "${RAPIDS_COVERAGE_DIR}"
+
+rapids-print-env
+
+  # TODO: remove the '>=24.12.00a1000' once we start publishing nightly packages
+  #       from the 'nx-cugraph' repo and stop publishing them from the 'cugraph' repo
+rapids-mamba-retry install \
+  --channel "${PYTHON_CHANNEL}" \
+  "nx-cugraph=${RAPIDS_VERSION}.*,>=24.12.00a1000"
+
+rapids-logger "Check GPU usage"
+nvidia-smi
+
+# export LD_PRELOAD="${CONDA_PREFIX}/lib/libgomp.so.1"
+
+# RAPIDS_DATASET_ROOT_DIR is used by test scripts
+# export RAPIDS_DATASET_ROOT_DIR="$(realpath datasets)"
+# pushd "${RAPIDS_DATASET_ROOT_DIR}"
+# ./get_test_data.sh --benchmark
+# popd
+
+EXITCODE=0
+trap "EXITCODE=1" ERR
+set +e
+
+rapids-logger "pytest nx-cugraph"
+./ci/run_nx_cugraph_pytests.sh \
+  --junitxml="${RAPIDS_TESTS_DIR}/junit-nx-cugraph.xml" \
+  --cov=nx_cugraph \
+  --cov-report=xml:"${RAPIDS_COVERAGE_DIR}/nx-cugraph-coverage.xml" \
+  --cov-report=term
+
+rapids-logger "pytest networkx using nx-cugraph backend"
+
+pushd nx_cugraph
+../run_nx_tests.sh
+
+# run_nx_tests.sh outputs coverage data, so check that total coverage is >0.0%
+# in case nx-cugraph failed to load but fallback mode allowed the run to pass.
+_coverage=$(coverage report | grep "^TOTAL")
+
+echo "nx-cugraph coverage from networkx tests: $_coverage"
+echo $_coverage | awk '{ if ($NF == "0.0%") exit 1 }'
+
+# Ensure all algorithms were called by comparing covered lines to function lines.
+# Run our tests again (they're fast enough) to add their coverage, then create coverage.json
+NX_CUGRAPH_USE_COMPAT_GRAPHS=False pytest \
+  --pyargs nx_cugraph \
+  --config-file=../pyproject.toml \
+  --cov-config=../pyproject.toml \
+  --cov=nx_cugraph \
+  --cov-append \
+  --cov-report=
+
+coverage report \
+  --include="*/nx_cugraph/algorithms/*" \
+  --omit=__init__.py \
+  --show-missing \
+  --rcfile=../pyproject.toml
+
+coverage json --rcfile=../pyproject.toml
+
+python -m nx_cugraph.tests.ensure_algos_covered
+
+# Exercise (and show results of) scripts that show implemented networkx algorithms
+python -m nx_cugraph.scripts.print_tree --dispatch-name --plc --incomplete --different
+python -m nx_cugraph.scripts.print_table
+popd
+
+rapids-logger "Test script exiting with value: $EXITCODE"
+exit ${EXITCODE}

ci/test_wheel_nx-cugraph.sh

@@ -0,0 +1,36 @@
+#!/bin/bash
+# Copyright (c) 2023-2024, NVIDIA CORPORATION.
+
+set -eoxu pipefail
+
+package_name="nx-cugraph"
+python_package_name=$(echo ${package_name}|sed 's/-/_/g')
+
+mkdir -p ./dist
+RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})"
+
+# nx-cugraph is a pure wheel, which is part of generating the download path
+RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" RAPIDS_PY_WHEEL_PURE="1" rapids-download-wheels-from-s3 ./dist
+
+# echo to expand wildcard before adding `[extra]` requires for pip
+python -m pip install \
+    "$(echo ./dist/${python_package_name}*.whl)[test]"
+
+# Run smoke tests for aarch64 pull requests
+arch=$(uname -m)
+if [[ "${arch}" == "aarch64" && ${RAPIDS_BUILD_TYPE} == "pull-request" ]]; then
+    python ./ci/wheel_smoke_test_${package_name}.py
+else
+    # --import-mode=append. See test_python.sh for details.
+    # FIXME: Adding PY_IGNORE_IMPORTMISMATCH=1 to workaround conftest.py import
+    # mismatch error seen by nx-cugraph after using pytest 8 and
+    # --import-mode=append.
+    RAPIDS_DATASET_ROOT_DIR=`pwd`/datasets \
+    PY_IGNORE_IMPORTMISMATCH=1 \
+    NX_CUGRAPH_USE_COMPAT_GRAPHS=False \
+    python -m pytest \
+       -v \
+       --import-mode=append \
+       --benchmark-disable \
+       ./${python_package_name}/tests
+fi

ci/wheel_smoke_test_nx-cugraph.py

@@ -0,0 +1,39 @@
+# Copyright (c) 2023-2024, NVIDIA CORPORATION.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import math
+
+import networkx as nx
+
+import nx_cugraph as nxcg
+
+if __name__ == "__main__":
+    G = nx.Graph()
+    G.add_edges_from([(0, 1), (1, 2), (2, 3)])
+
+    nx_result = nx.betweenness_centrality(G)
+    # nx_cugraph is intended to be called via the NetworkX dispatcher, like
+    # this:
+    #    nxcu_result = nx.betweenness_centrality(G, backend="cugraph")
+    #
+    # but here it is being called directly since the NetworkX version that
+    # supports the "backend" kwarg may not be available in the testing env.
+    nxcu_result = nxcg.betweenness_centrality(G)
+
+    nx_nodes, nxcu_nodes = nx_result.keys(), nxcu_result.keys()
+    assert nxcu_nodes == nx_nodes
+    for node_id in nx_nodes:
+        nx_bc, nxcu_bc = nx_result[node_id], nxcu_result[node_id]
+        assert math.isclose(
+            nx_bc, nxcu_bc, rel_tol=1e-6
+        ), f"bc for {node_id=} exceeds tolerance: {nx_bc=}, {nxcu_bc=}"

dependencies.yaml

@@ -34,6 +34,14 @@ files:
       - cuda_version
       - py_version
       - test_notebook
+  test_python:
+    output: none
+    includes:
+      - cuda_version
+      - depends_on_cudf
+      - py_version
+      - test_python_common
+      - test_python_nx_cugraph
   py_build_nx_cugraph:
     output: pyproject
     pyproject_dir: .

nx_cugraph/__init__.py

@@ -15,7 +15,7 @@
 from _nx_cugraph._version import __git_commit__, __version__
 from _nx_cugraph import _check_networkx_version
 
-_nxver: tuple[int, int] = _check_networkx_version()
+_nxver: tuple[int, int] | tuple[int, int, int] = _check_networkx_version()
 
 from . import utils