From 419f0c28cd18654064bfd3a6ed2f638e239f46b3 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Thu, 23 Mar 2023 09:28:16 -0400 Subject: [PATCH] Generate pyproject dependencies with dfg (#1364) This PR uses dependencies.yaml to generate the dependency lists in pyproject.toml Authors: - Vyas Ramasubramani (https://github.com/vyasr) - Corey J. Nolet (https://github.com/cjnolet) Approvers: - AJ Schmidt (https://github.com/ajschmidt8) - Corey J. Nolet (https://github.com/cjnolet) URL: https://github.com/rapidsai/raft/pull/1364 --- .pre-commit-config.yaml | 2 +- .../all_cuda-118_arch-x86_64.yaml | 9 +- conda/recipes/pylibraft/meta.yaml | 1 + dependencies.yaml | 107 +++++++++++++++--- python/pylibraft/pyproject.toml | 24 ++-- python/raft-dask/pyproject.toml | 24 ++-- 6 files changed, 127 insertions(+), 40 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 7606914589..630b8788f8 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -101,7 +101,7 @@ repos: args: ["--toml", "pyproject.toml"] exclude: (?x)^(^CHANGELOG.md$) - repo: https://github.com/rapidsai/dependency-file-generator - rev: v1.4.0 + rev: v1.5.1 hooks: - id: rapids-dependency-file-generator args: ["--clean"] diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml index 7972a8824d..1afebc98e6 100644 --- a/conda/environments/all_cuda-118_arch-x86_64.yaml +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -18,13 +18,14 @@ dependencies: - cupy - cxx-compiler - cython>=0.29,<0.30 -- dask-cuda=23.04 +- dask-cuda==23.4.* - dask>=2023.1.1 - distributed>=2023.1.1 - doxygen>=1.8.20 - gcc_linux-64=11.* - graphviz - ipython +- joblib>=0.11 - libcublas-dev=11.11.3.6 - libcublas=11.11.3.6 - libcurand-dev=10.3.0.86 @@ -35,12 +36,14 @@ dependencies: - libcusparse=11.7.5.86 - nccl>=2.9.9 - ninja +- numba>=0.49 +- numpy>=1.21 - numpydoc - pydata-sphinx-theme - pytest - pytest-cov - recommonmark -- rmm=23.04 +- rmm==23.4.* - scikit-build>=0.13.1 - scikit-learn - scipy @@ -48,6 +51,6 @@ dependencies: - sphinx-markdown-tables - sysroot_linux-64==2.17 - ucx-proc=*=gpu -- ucx-py=0.31.* +- ucx-py==0.31.* - ucx>=1.13.0 name: all_cuda-118_arch-x86_64 diff --git a/conda/recipes/pylibraft/meta.yaml b/conda/recipes/pylibraft/meta.yaml index a528064348..7730801801 100644 --- a/conda/recipes/pylibraft/meta.yaml +++ b/conda/recipes/pylibraft/meta.yaml @@ -36,6 +36,7 @@ requirements: - cython >=0.29,<0.30 - libraft {{ version }} - libraft-headers {{ version }} + - numpy >=1.21 - python x.x - rmm ={{ minor_version }} - scikit-build >=0.13.1 diff --git a/dependencies.yaml b/dependencies.yaml index c06ce4a20f..dd361a0cdf 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -7,11 +7,14 @@ files: arch: [x86_64] includes: - build + - build_pylibraft - cudatoolkit - develop - docs - - run - - test_python + - run_raft_dask + - run_pylibraft + - test_python_common + - test_pylibraft test_cpp: output: none includes: @@ -21,7 +24,8 @@ files: includes: - cudatoolkit - py_version - - test_python + - test_python_common + - test_pylibraft checks: output: none includes: @@ -33,6 +37,54 @@ files: - cudatoolkit - docs - py_version + py_build_pylibraft: + output: pyproject + pyproject_dir: python/pylibraft + extras: + table: build-system + includes: + - build + - build_pylibraft + - build_wheels + py_run_pylibraft: + output: pyproject + pyproject_dir: python/pylibraft + extras: + table: project + includes: + - run_pylibraft + py_test_pylibraft: + output: pyproject + pyproject_dir: python/pylibraft + extras: + table: project.optional-dependencies + key: test + includes: + - test_python_common + - test_pylibraft + py_build_raft_dask: + output: pyproject + pyproject_dir: python/raft-dask + extras: + table: build-system + includes: + - build + - build_wheels + py_run_raft_dask: + output: pyproject + pyproject_dir: python/raft-dask + extras: + table: project + includes: + - run_raft_dask + py_test_raft_dask: + output: pyproject + pyproject_dir: python/raft-dask + extras: + table: project.optional-dependencies + key: test + includes: + - test_python_common channels: - rapidsai - rapidsai-nightly @@ -42,10 +94,9 @@ channels: dependencies: build: common: - - output_types: [conda, requirements] + - output_types: [conda, requirements, pyproject] packages: - cmake>=3.23.1,!=3.25.0 - - cuda-python >=11.7.1,<12.0 - cython>=0.29,<0.30 - ninja - scikit-build>=0.13.1 @@ -67,6 +118,12 @@ dependencies: packages: - gcc_linux-aarch64=11.* - sysroot_linux-aarch64==2.17 + build_pylibraft: + common: + - output_types: [conda, requirements, pyproject] + packages: + - &cuda_python cuda-python >=11.7.1,<12.0 + - &rmm rmm==23.4.* checks: common: - output_types: [conda, requirements] @@ -151,6 +208,12 @@ dependencies: - recommonmark - sphinx-copybutton - sphinx-markdown-tables + build_wheels: + common: + - output_types: pyproject + packages: + - wheel + - setuptools py_version: specific: - output_types: conda @@ -170,23 +233,41 @@ dependencies: - matrix: packages: - python>=3.8,<3.11 - run: + run_pylibraft: common: - - output_types: [conda] + - output_types: [conda, pyproject] + packages: + - &numpy numpy>=1.21 + - *cuda_python + - *rmm + run_raft_dask: + common: + - output_types: [conda, pyproject] packages: - dask>=2023.1.1 + - dask-cuda==23.4.* - distributed>=2023.1.1 + - joblib>=0.11 + - numba>=0.49 + - *numpy + - ucx-py==0.31.* + - output_types: conda + packages: - ucx>=1.13.0 - - ucx-py=0.31.* - ucx-proc=*=gpu - - rmm=23.04 - - dask-cuda=23.04 - test_python: + - output_types: pyproject + packages: + - pylibraft==23.4.* + test_python_common: common: - - output_types: [conda, requirements] + - output_types: [conda, requirements, pyproject] packages: - - cupy - pytest - pytest-cov + test_pylibraft: + common: + - output_types: [conda, requirements, pyproject] + packages: + - cupy - scikit-learn - scipy diff --git a/python/pylibraft/pyproject.toml b/python/pylibraft/pyproject.toml index 7d92fd0763..fed15bbab0 100644 --- a/python/pylibraft/pyproject.toml +++ b/python/pylibraft/pyproject.toml @@ -15,15 +15,15 @@ [build-system] requires = [ - "wheel", - "setuptools", - "cython>=0.29,<0.30", - "cuda-python>=11.7.1,<12.0", - "scikit-build>=0.13.1", "cmake>=3.23.1,!=3.25.0", + "cuda-python >=11.7.1,<12.0", + "cython>=0.29,<0.30", "ninja", "rmm==23.4.*", -] + "scikit-build>=0.13.1", + "setuptools", + "wheel", +] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. build-backend = "setuptools.build_meta" [project] @@ -37,10 +37,10 @@ authors = [ license = { text = "Apache 2.0" } requires-python = ">=3.8" dependencies = [ - "numpy", - "cuda-python>=11.7.1,<12.0", + "cuda-python >=11.7.1,<12.0", + "numpy>=1.21", "rmm==23.4.*", -] +] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. classifiers = [ "Intended Audience :: Developers", "Programming Language :: Python", @@ -50,10 +50,12 @@ classifiers = [ [project.optional-dependencies] test = [ + "cupy", "pytest", - "scipy", + "pytest-cov", "scikit-learn", -] + "scipy", +] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. [project.urls] Homepage = "https://github.com/rapidsai/raft" diff --git a/python/raft-dask/pyproject.toml b/python/raft-dask/pyproject.toml index 2fe6522f57..fe490ea117 100644 --- a/python/raft-dask/pyproject.toml +++ b/python/raft-dask/pyproject.toml @@ -15,13 +15,13 @@ [build-system] requires = [ - "wheel", - "setuptools", - "cython>=0.29,<0.30", - "scikit-build>=0.13.1", "cmake>=3.23.1,!=3.25.0", + "cython>=0.29,<0.30", "ninja", -] + "scikit-build>=0.13.1", + "setuptools", + "wheel", +] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. [project] name = "raft-dask" @@ -34,15 +34,15 @@ authors = [ license = { text = "Apache 2.0" } requires-python = ">=3.8" dependencies = [ - "numpy", - "numba>=0.49", - "joblib>=0.11", "dask-cuda==23.4.*", "dask>=2023.1.1", - "ucx-py==0.31.*", "distributed>=2023.1.1", + "joblib>=0.11", + "numba>=0.49", + "numpy>=1.21", "pylibraft==23.4.*", -] + "ucx-py==0.31.*", +] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. classifiers = [ "Intended Audience :: Developers", "Programming Language :: Python", @@ -53,8 +53,8 @@ classifiers = [ [project.optional-dependencies] test = [ "pytest", - "dask[distributed,dataframe]", -] + "pytest-cov", +] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. [project.urls] Homepage = "https://github.com/rapidsai/raft"