manage more dependencies in dependencies.yaml, declare 'numpy' runtim…

…e dependency (#230)

While working on adding CI for `wholegraph` in the new `cugraph-gnn` repo (rapidsai/cugraph-gnn#58), I noticed a few issues here.

* `pylibwholegraph` imports `numpy` at runtime, but its wheels and conda package don't declare a runtime dependency on `numpy`
* wheel runtime and testing dependencies are not being managed by `rapids-dependency-file-generator`
* `wholegraph_binding` Cython code imports NumPy but doesn't use it
* the wheel-testing CI environment is built up with a sequence of `pip install` calls instead of a single one to get all dependencies (see rapidsai/cugraph#4701)

This proposes fixes for those things.

## Notes for Reviewers

### Where is the NumPy runtime dependency coming from?

https://github.com/rapidsai/wholegraph/blob/0efba33835d6e4e104b5d7101a91e0ea55a6ca53/python/pylibwholegraph/pylibwholegraph/torch/data_loader.py#L14

Authors:
  - James Lamb (https://github.com/jameslamb)

Approvers:
  - Kyle Edwards (https://github.com/KyleFromNVIDIA)

URL: #230

ci/test_wheel.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
-# Copyright (c) 2023, NVIDIA CORPORATION.
+# Copyright (c) 2023-2024, NVIDIA CORPORATION.
 
 set -e          # abort the script on error
 set -o pipefail # piped commands propagate their error
@@ -9,23 +9,26 @@ mkdir -p ./dist
 RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})"
 RAPIDS_PY_WHEEL_NAME="pylibwholegraph_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./dist
 
-# echo to expand wildcard before adding `[extra]` requires for pip
-python -m pip install $(echo ./dist/pylibwholegraph*.whl)
-
+# determine PyTorch source
 PKG_CUDA_VER="$(echo ${CUDA_VERSION} | cut -d '.' -f1,2 | tr -d '.')"
 PKG_CUDA_VER_MAJOR=${PKG_CUDA_VER:0:2}
 if [[ "${PKG_CUDA_VER_MAJOR}" == "12" ]]; then
   INDEX_URL="https://download.pytorch.org/whl/cu121"
 else
   INDEX_URL="https://download.pytorch.org/whl/cu${PKG_CUDA_VER}"
 fi
+
+# echo to expand wildcard before adding `[extra]` requires for pip
+python -m pip install \
+  -v \
+  --extra-index-url "${INDEX_URL}" \
+  "$(echo ./dist/pylibwholegraph_${RAPIDS_PY_CUDA_SUFFIX}*.whl)[test]" \
+  'torch>=2.0,<2.4.0a0'
+
 RAPIDS_TESTS_DIR=${RAPIDS_TESTS_DIR:-"${PWD}/test-results"}
 RAPIDS_COVERAGE_DIR=${RAPIDS_COVERAGE_DIR:-"${PWD}/coverage-results"}
 mkdir -p "${RAPIDS_TESTS_DIR}" "${RAPIDS_COVERAGE_DIR}"
 
-rapids-logger "Installing PyTorch"
-rapids-retry python -m pip install --pre torch --index-url ${INDEX_URL}
-rapids-retry python -m pip install pytest pytest-forked numpy
 rapids-logger "pytest pylibwholegraph"
 cd python/pylibwholegraph/pylibwholegraph/tests
 python -m pytest \

conda/recipes/pylibwholegraph/meta.yaml

@@ -71,6 +71,7 @@ requirements:
     - cudatoolkit
     {% endif %}
     - libwholegraph ={{ version }}
+    - numpy>=1.23,<3.0a0
     - python
 
 about:

dependencies.yaml

@@ -64,6 +64,21 @@ files:
       key: requires
     includes:
       - python_build_wheel
+  py_run_pylibwholegraph:
+    output: pyproject
+    pyproject_dir: python/pylibwholegraph
+    extras:
+      table: project
+    includes:
+      - run
+  py_test_pylibwholegraph:
+    output: pyproject
+    pyproject_dir: python/pylibwholegraph
+    extras:
+      table: project.optional-dependencies
+      key: test
+    includes:
+      - test_python
 channels:
   - rapidsai
   - rapidsai-nightly
@@ -210,8 +225,9 @@ dependencies:
               - python>=3.10,<3.13
   run:
     common:
-      - output_types: [conda, requirements]
-        packages: []
+      - output_types: [conda, pyproject, requirements]
+        packages:
+          - numpy>=1.23,<3.0a0
   test_cpp:
     common:
       - output_types: [conda]
@@ -224,10 +240,9 @@ dependencies:
           - c-compiler
           - cxx-compiler
           - *nccl
-      - output_types: [conda, requirements]
-        packages:
           - ninja
-          - numpy>=1.23,<3.0a0
+      - output_types: [conda, pyproject, requirements]
+        packages:
           - pytest
           - pytest-forked
           - pytest-xdist

python/pylibwholegraph/pylibwholegraph/binding/wholememory_binding.pyx

@@ -27,7 +27,6 @@ from libcpp cimport bool
 from cpython cimport Py_buffer
 from cpython cimport array
 import array
-import numpy as np
 from cpython.ref cimport PyObject, Py_INCREF, Py_DECREF
 from cpython.object cimport Py_TYPE, PyObject_CallObject
 from cpython.tuple cimport *

python/pylibwholegraph/pyproject.toml

@@ -36,6 +36,17 @@ classifiers = [
     "Programming Language :: Python :: 3.11",
     "Programming Language :: Python :: 3.12",
 ]
+dependencies = [
+    "numpy>=1.23,<3.0a0",
+] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`.
+
+
+[project.optional-dependencies]
+test = [
+    "pytest",
+    "pytest-forked",
+    "pytest-xdist",
+] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`.
 
 [tool.rapids-build-backend]
 build-backend = "scikit_build_core.build"