Generate pyproject dependencies with dfg (#1364)

This PR uses dependencies.yaml to generate the dependency lists in pyproject.toml Authors: - Vyas Ramasubramani (https://github.com/vyasr) - Corey J. Nolet (https://github.com/cjnolet) Approvers: - AJ Schmidt (https://github.com/ajschmidt8) - Corey J. Nolet (https://github.com/cjnolet) URL: #1364
rapidsai · Mar 23, 2023 · 419f0c2 · 419f0c2
1 parent 7c73f23
commit 419f0c2
Show file tree

Hide file tree

Showing 6 changed files with 127 additions and 40 deletions.
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -101,7 +101,7 @@ repos:
                 args: ["--toml", "pyproject.toml"]
                 exclude: (?x)^(^CHANGELOG.md$)
       - repo: https://github.com/rapidsai/dependency-file-generator
-        rev: v1.4.0
+        rev: v1.5.1
         hooks:
             - id: rapids-dependency-file-generator
               args: ["--clean"]

diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml
@@ -18,13 +18,14 @@ dependencies:
 - cupy
 - cxx-compiler
 - cython>=0.29,<0.30
-- dask-cuda=23.04
+- dask-cuda==23.4.*
 - dask>=2023.1.1
 - distributed>=2023.1.1
 - doxygen>=1.8.20
 - gcc_linux-64=11.*
 - graphviz
 - ipython
+- joblib>=0.11
 - libcublas-dev=11.11.3.6
 - libcublas=11.11.3.6
 - libcurand-dev=10.3.0.86
@@ -35,19 +36,21 @@ dependencies:
 - libcusparse=11.7.5.86
 - nccl>=2.9.9
 - ninja
+- numba>=0.49
+- numpy>=1.21
 - numpydoc
 - pydata-sphinx-theme
 - pytest
 - pytest-cov
 - recommonmark
-- rmm=23.04
+- rmm==23.4.*
 - scikit-build>=0.13.1
 - scikit-learn
 - scipy
 - sphinx-copybutton
 - sphinx-markdown-tables
 - sysroot_linux-64==2.17
 - ucx-proc=*=gpu
-- ucx-py=0.31.*
+- ucx-py==0.31.*
 - ucx>=1.13.0
 name: all_cuda-118_arch-x86_64
diff --git a/conda/recipes/pylibraft/meta.yaml b/conda/recipes/pylibraft/meta.yaml
@@ -36,6 +36,7 @@ requirements:
     - cython >=0.29,<0.30
     - libraft {{ version }}
     - libraft-headers {{ version }}
+    - numpy >=1.21
     - python x.x
     - rmm ={{ minor_version }}
     - scikit-build >=0.13.1

diff --git a/dependencies.yaml b/dependencies.yaml
@@ -7,11 +7,14 @@ files:
       arch: [x86_64]
     includes:
       - build
+      - build_pylibraft
       - cudatoolkit
       - develop
       - docs
-      - run
-      - test_python
+      - run_raft_dask
+      - run_pylibraft
+      - test_python_common
+      - test_pylibraft
   test_cpp:
     output: none
     includes:
@@ -21,7 +24,8 @@ files:
     includes:
       - cudatoolkit
       - py_version
-      - test_python
+      - test_python_common
+      - test_pylibraft
   checks:
     output: none
     includes:
@@ -33,6 +37,54 @@ files:
       - cudatoolkit
       - docs
       - py_version
+  py_build_pylibraft:
+    output: pyproject
+    pyproject_dir: python/pylibraft
+    extras:
+      table: build-system
+    includes:
+      - build
+      - build_pylibraft
+      - build_wheels
+  py_run_pylibraft:
+    output: pyproject
+    pyproject_dir: python/pylibraft
+    extras:
+      table: project
+    includes:
+      - run_pylibraft
+  py_test_pylibraft:
+    output: pyproject
+    pyproject_dir: python/pylibraft
+    extras:
+      table: project.optional-dependencies
+      key: test
+    includes:
+      - test_python_common
+      - test_pylibraft
+  py_build_raft_dask:
+    output: pyproject
+    pyproject_dir: python/raft-dask
+    extras:
+      table: build-system
+    includes:
+      - build
+      - build_wheels
+  py_run_raft_dask:
+    output: pyproject
+    pyproject_dir: python/raft-dask
+    extras:
+      table: project
+    includes:
+      - run_raft_dask
+  py_test_raft_dask:
+    output: pyproject
+    pyproject_dir: python/raft-dask
+    extras:
+      table: project.optional-dependencies
+      key: test
+    includes:
+      - test_python_common
 channels:
   - rapidsai
   - rapidsai-nightly
@@ -42,10 +94,9 @@ channels:
 dependencies:
   build:
     common:
-      - output_types: [conda, requirements]
+      - output_types: [conda, requirements, pyproject]
         packages:
           - cmake>=3.23.1,!=3.25.0
-          - cuda-python >=11.7.1,<12.0
           - cython>=0.29,<0.30
           - ninja
           - scikit-build>=0.13.1
@@ -67,6 +118,12 @@ dependencies:
             packages:
               - gcc_linux-aarch64=11.*
               - sysroot_linux-aarch64==2.17
+  build_pylibraft:
+    common:
+      - output_types: [conda, requirements, pyproject]
+        packages:
+          - &cuda_python cuda-python >=11.7.1,<12.0
+          - &rmm rmm==23.4.*
   checks:
     common:
       - output_types: [conda, requirements]
@@ -151,6 +208,12 @@ dependencies:
           - recommonmark
           - sphinx-copybutton
           - sphinx-markdown-tables
+  build_wheels:
+    common:
+      - output_types: pyproject
+        packages:
+          - wheel
+          - setuptools
   py_version:
     specific:
       - output_types: conda
@@ -170,23 +233,41 @@ dependencies:
           - matrix:
             packages:
               - python>=3.8,<3.11
-  run:
+  run_pylibraft:
     common:
-      - output_types: [conda]
+      - output_types: [conda, pyproject]
+        packages:
+          - &numpy numpy>=1.21
+          - *cuda_python
+          - *rmm
+  run_raft_dask:
+    common:
+      - output_types: [conda, pyproject]
         packages:
           - dask>=2023.1.1
+          - dask-cuda==23.4.*
           - distributed>=2023.1.1
+          - joblib>=0.11
+          - numba>=0.49
+          - *numpy
+          - ucx-py==0.31.*
+      - output_types: conda
+        packages:
           - ucx>=1.13.0
-          - ucx-py=0.31.*
           - ucx-proc=*=gpu
-          - rmm=23.04
-          - dask-cuda=23.04
-  test_python:
+      - output_types: pyproject
+        packages:
+          - pylibraft==23.4.*
+  test_python_common:
     common:
-      - output_types: [conda, requirements]
+      - output_types: [conda, requirements, pyproject]
         packages:
-          - cupy
           - pytest
           - pytest-cov
+  test_pylibraft:
+    common:
+      - output_types: [conda, requirements, pyproject]
+        packages:
+          - cupy
           - scikit-learn
           - scipy
diff --git a/python/pylibraft/pyproject.toml b/python/pylibraft/pyproject.toml
@@ -15,15 +15,15 @@
 [build-system]
 
 requires = [
-    "wheel",
-    "setuptools",
-    "cython>=0.29,<0.30",
-    "cuda-python>=11.7.1,<12.0",
-    "scikit-build>=0.13.1",
     "cmake>=3.23.1,!=3.25.0",
+    "cuda-python >=11.7.1,<12.0",
+    "cython>=0.29,<0.30",
     "ninja",
     "rmm==23.4.*",
-]
+    "scikit-build>=0.13.1",
+    "setuptools",
+    "wheel",
+] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`.
 build-backend = "setuptools.build_meta"
 
 [project]
@@ -37,10 +37,10 @@ authors = [
 license = { text = "Apache 2.0" }
 requires-python = ">=3.8"
 dependencies = [
-    "numpy",
-    "cuda-python>=11.7.1,<12.0",
+    "cuda-python >=11.7.1,<12.0",
+    "numpy>=1.21",
     "rmm==23.4.*",
-]
+] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`.
 classifiers = [
     "Intended Audience :: Developers",
     "Programming Language :: Python",
@@ -50,10 +50,12 @@ classifiers = [
 
 [project.optional-dependencies]
 test = [
+    "cupy",
     "pytest",
-    "scipy",
+    "pytest-cov",
     "scikit-learn",
-]
+    "scipy",
+] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`.
 
 [project.urls]
 Homepage = "https://github.com/rapidsai/raft"

diff --git a/python/raft-dask/pyproject.toml b/python/raft-dask/pyproject.toml
@@ -15,13 +15,13 @@
 [build-system]
 
 requires = [
-    "wheel",
-    "setuptools",
-    "cython>=0.29,<0.30",
-    "scikit-build>=0.13.1",
     "cmake>=3.23.1,!=3.25.0",
+    "cython>=0.29,<0.30",
     "ninja",
-]
+    "scikit-build>=0.13.1",
+    "setuptools",
+    "wheel",
+] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`.
 
 [project]
 name = "raft-dask"
@@ -34,15 +34,15 @@ authors = [
 license = { text = "Apache 2.0" }
 requires-python = ">=3.8"
 dependencies = [
-    "numpy",
-    "numba>=0.49",
-    "joblib>=0.11",
     "dask-cuda==23.4.*",
     "dask>=2023.1.1",
-    "ucx-py==0.31.*",
     "distributed>=2023.1.1",
+    "joblib>=0.11",
+    "numba>=0.49",
+    "numpy>=1.21",
     "pylibraft==23.4.*",
-]
+    "ucx-py==0.31.*",
+] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`.
 classifiers = [
     "Intended Audience :: Developers",
     "Programming Language :: Python",
@@ -53,8 +53,8 @@ classifiers = [
 [project.optional-dependencies]
 test = [
     "pytest",
-    "dask[distributed,dataframe]",
-]
+    "pytest-cov",
+] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`.
 
 [project.urls]
 Homepage = "https://github.com/rapidsai/raft"