Migrate as much as possible to pyproject.toml, stop using versioneer …

…to manage versions, update dependencies.yaml. (#232) Modernizes kvikio's Python build system to help with CUDA 12 migration. Some of these changes were made for other RAPIDS repos in separate PRs over the course of time but I am doing them all together in this PR to help kvikio catch up to the state of other repos. Major changes: - Removed versioneer. - Uses `pyproject.toml` for as much metadata as possible. - Updates/copies/fixes/centralizes linter configurations for isort, mypy, etc. Closes #194. Addresses some things mentioned in #186. Authors: - Bradley Dice (https://github.com/bdice) Approvers: - Vyas Ramasubramani (https://github.com/vyasr) - Mads R. B. Kristensen (https://github.com/madsbk) - Lawrence Mitchell (https://github.com/wence-) URL: #232
rapidsai · Jun 7, 2023 · 4efb31c · 4efb31c
1 parent d5653a3
commit 4efb31c
Show file tree

Hide file tree

Showing 21 changed files with 307 additions and 6,080 deletions.
diff --git a/.codespellrc b/.codespellrc
@@ -3,7 +3,7 @@
 [codespell]
 # note: pre-commit passes explicit lists of files here, which this skip file list doesn't override -
 # this is only to allow you to run codespell interactively
-skip = ./.git,./.github,./cpp/build,.*egg-info.*,versioneer.py,./python/tests
+skip = ./.git,./.github,./cpp/build,.*egg-info.*,./python/tests
 # ignore short words, and typename parameters like OffsetT
 ignore-regex = \b(.{1,4}|[A-Z]\w*T)\b
 ignore-words-list = inout,unparseable

diff --git a/legate/setup.cfg → .flake8 b/legate/setup.cfg → .flake8
@@ -1,18 +1,5 @@
 # Copyright (c) 2023, NVIDIA CORPORATION.
 
-# See the docstring in versioneer.py for instructions. Note that you must
-# re-run 'versioneer.py setup' after changing this section, and commit the
-# resulting files.
-
-
-[versioneer]
-VCS = git
-style = pep440
-versionfile_source = legate_kvikio/_version.py
-versionfile_build = legate_kvikio/_version.py
-tag_prefix = v
-parentdir_prefix = legate_kvikio-
-
 [flake8]
 filename = *.py, *.pyx, *.pxd, *.pxi
 force-check = True
@@ -23,8 +10,6 @@ exclude =
     build,
     docs,
     .git,
-    versioneer.py,
-    _version.py,
     _skbuild,
 ignore =
     # line break before binary operator
@@ -47,48 +32,3 @@ per-file-ignores =
     *.pyx: E211, E225, E226, E227, E275, E402, E999, W503, W504
     *.pxd: E211, E225, E226, E227, E275, E402, E999, W503, W504
     *.pxi: E211, E225, E226, E227, E275, E402, E999, W503, W504
-
-[isort]
-line_length=88
-multi_line_output=3
-include_trailing_comma=True
-force_grid_wrap=0
-combine_as_imports=True
-order_by_type=True
-known_rapids=
-    nvtext
-    cudf
-    cuml
-    cugraph
-    dask_cudf
-known_first_party=
-    cufile
-    kvikio
-    legate_kvikio
-default_section=THIRDPARTY
-sections=FUTURE,STDLIB,THIRDPARTY,RAPIDS,FIRSTPARTY,LOCALFOLDER
-skip=
-    thirdparty
-    .eggs
-    .git
-    .hg
-    .mypy_cache
-    .tox
-    .venv
-    _build
-    buck-out
-    build
-    dist
-    __init__.py
-
-[options]
-packages = find:
-python_requires = >=3.9
-
-[options.extras_require]
-nvcomp =
-    numpy >=1.20
-    cupy >=9.5.0
-
-[mypy]
-ignore_missing_imports = True
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -18,7 +18,7 @@ repos:
         files: legate/.*
         types_or: [python, cython, pyi]
   - repo: https://github.com/psf/black
-    rev: 22.12.0
+    rev: 23.3.0
     hooks:
       - id: black
         files: (python|legate)/.*
@@ -27,28 +27,28 @@ repos:
     rev: 6.0.0
     hooks:
       - id: flake8
-        args: ["--config=python/setup.cfg"]
+        args: ["--config=.flake8"]
         files: (python|legate)/.*$
         types: [file]
         types_or: [python, cython]
         additional_dependencies: ["flake8-force"]
   - repo: https://github.com/MarcoGorelli/cython-lint
-    rev: v0.10.1
+    rev: v0.15.0
     hooks:
       - id: cython-lint
   - repo: https://github.com/pre-commit/mirrors-mypy
     rev: 'v1.3.0'
     hooks:
       - id: mypy
         additional_dependencies: [types-cachetools]
-        args: ["--config-file=python/setup.cfg",
+        args: ["--config-file=python/pyproject.toml",
                 "python/kvikio",
                 "python/tests",
                 "python/examples",
                 "python/benchmarks"]
         pass_filenames: false
   - repo: https://github.com/pre-commit/mirrors-clang-format
-    rev: v16.0.1
+    rev: v16.0.4
     hooks:
       - id: clang-format
         types_or: [c, c++, cuda]
@@ -84,14 +84,18 @@ repos:
         verbose: true
         require_serial: true
   - repo: https://github.com/codespell-project/codespell
-    rev: v2.2.2
+    rev: v2.2.4
     hooks:
       - id: codespell
         exclude: |
           (?x)^(
-            ^CHANGELOG.md$|
-            ^.*versioneer.py$
+            ^CHANGELOG.md$
           )
+  - repo: https://github.com/rapidsai/dependency-file-generator
+    rev: v1.5.2
+    hooks:
+      - id: rapids-dependency-file-generator
+        args: ["--clean"]
 
 default_language_version:
       python: python3
diff --git a/ci/checks/copyright.py b/ci/checks/copyright.py
@@ -28,7 +28,6 @@
     re.compile(r"meta[.]yaml$"),
 ]
 ExemptFiles = [
-    re.compile(r"versioneer[.]py"),
     re.compile(r"get_cpm[.]cmake"),
 ]
 

diff --git a/ci/release/update-version.sh b/ci/release/update-version.sh
@@ -33,9 +33,17 @@ function sed_runner() {
 # cpp update
 sed_runner "/project(/,/)/s/VERSION.*/VERSION ${NEXT_FULL_TAG}/" cpp/CMakeLists.txt
 
-# python update
+# Python CMakeLists updates
 sed_runner 's/set(kvikio_version.*)/set(kvikio_version '${NEXT_FULL_TAG}')/g' python/CMakeLists.txt
 
+# Python __init__.py updates
+sed_runner "s/__version__ = .*/__version__ = \"${NEXT_FULL_TAG}\"/g" python/kvikio/__init__.py
+sed_runner "s/__version__ = .*/__version__ = \"${NEXT_FULL_TAG}\"/g" legate/legate_kvikio/__init__.py
+
+# Python pyproject.toml updates
+sed_runner "s/^version = .*/version = \"${NEXT_FULL_TAG}\"/g" python/pyproject.toml
+sed_runner "s/^version = .*/version = \"${NEXT_FULL_TAG}\"/g" legate/pyproject.toml
+
 # rapids-cmake version
 sed_runner 's/'"branch-.*\/RAPIDS.cmake"'/'"branch-${NEXT_SHORT_TAG}\/RAPIDS.cmake"'/g' cpp/cmake/fetch_rapids.cmake
 
@@ -49,9 +57,19 @@ sed_runner 's/PROJECT_NUMBER         = .*/PROJECT_NUMBER         = '${NEXT_FULL_
 sed_runner 's/version = .*/version = '"'${NEXT_SHORT_TAG}'"'/g' docs/source/conf.py
 sed_runner 's/release = .*/release = '"'${NEXT_FULL_TAG}'"'/g' docs/source/conf.py
 
-# bump cudf
-for FILE in conda/environments/*.yaml dependencies.yaml; do
-  sed_runner "s/cudf=.*/cudf=${NEXT_SHORT_TAG}/g" ${FILE};
+# Need to distutils-normalize the original version
+NEXT_SHORT_TAG_PEP440=$(python -c "from setuptools.extern import packaging; print(packaging.version.Version('${NEXT_SHORT_TAG}'))")
+
+DEPENDENCIES=(
+  cudf
+)
+for DEP in "${DEPENDENCIES[@]}"; do
+  for FILE in dependencies.yaml conda/environments/*.yaml; do
+    sed_runner "/-.* ${DEP}==/ s/==.*/==${NEXT_SHORT_TAG_PEP440}.*/g" ${FILE}
+  done
+  for FILE in python/pyproject.toml legate/pyproject.toml; do
+    sed_runner "/\"${DEP}==/ s/==.*\"/==${NEXT_SHORT_TAG_PEP440}.*\"/g" ${FILE}
+  done
 done
 
 # CI files

diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml
@@ -8,9 +8,10 @@ channels:
 dependencies:
 - c-compiler
 - cmake>=3.23.1,!=3.25.0
+- cuda-python>=11.7.1,<12.0
 - cudatoolkit=11.8
-- cudf=23.08
-- cupy >=12.0.0
+- cudf==23.8.*
+- cupy>=12.0.0
 - cxx-compiler
 - cython>=0.29,<0.30
 - dask>=2022.05.2
@@ -20,7 +21,7 @@ dependencies:
 - libcufile-dev=1.4.0.31
 - libcufile=1.4.0.31
 - ninja
-- numpy >=1.20
+- numpy>=1.21
 - pre-commit
 - pydata-sphinx-theme
 - pytest

diff --git a/cpp/include/kvikio/batch.hpp b/cpp/include/kvikio/batch.hpp
@@ -27,7 +27,7 @@
 namespace kvikio {
 
 /**
- * @brief IO operation used when submiting batches
+ * @brief IO operation used when submitting batches
  */
 struct BatchOp {
   // The file handle of the file to read or write

diff --git a/dependencies.yaml b/dependencies.yaml
@@ -34,6 +34,50 @@ files:
       - cudatoolkit
       - docs
       - py_version
+  py_build:
+    output: pyproject
+    pyproject_dir: python
+    extras:
+      table: build-system
+    includes:
+      - build
+  py_run:
+    output: pyproject
+    pyproject_dir: python
+    extras:
+      table: project
+    includes:
+      - run
+  py_optional_test:
+    output: pyproject
+    pyproject_dir: python
+    extras:
+      table: project.optional-dependencies
+      key: test
+    includes:
+      - test_python
+  legate_py_build:
+    output: pyproject
+    pyproject_dir: legate
+    extras:
+      table: build-system
+    includes:
+      - build
+  legate_py_run:
+    output: pyproject
+    pyproject_dir: legate
+    extras:
+      table: project
+    includes:
+      - run
+  legate_py_optional_test:
+    output: pyproject
+    pyproject_dir: legate
+    extras:
+      table: project.optional-dependencies
+      key: test
+    includes:
+      - test_python
 channels:
   - rapidsai
   - rapidsai-nightly
@@ -42,7 +86,7 @@ channels:
 dependencies:
   build:
     common:
-      - output_types: [conda, requirements]
+      - output_types: [conda, requirements, pyproject]
         packages:
           - cmake>=3.23.1,!=3.25.0
           - cython>=0.29,<0.30
@@ -52,6 +96,10 @@ dependencies:
         packages:
           - c-compiler
           - cxx-compiler
+      - output_types: [requirements, pyproject]
+        packages:
+          - setuptools
+          - wheel
     specific:
       - output_types: conda
         matrices:
@@ -132,13 +180,16 @@ dependencies:
               - python>=3.9,<3.11
   run:
     common:
-      - output_types: [conda, requirements]
+      - output_types: [conda, requirements, pyproject]
         packages:
-          - numpy >=1.20
+          - numpy>=1.21
           - zarr
       - output_types: conda
         packages:
-          - cupy >=12.0.0
+          - cupy>=12.0.0
+      - output_types: [requirements, pyproject]
+        packages:
+          - cupy-cuda11x>=12.0.0
     specific:
       - output_types: requirements
         matrices:
@@ -149,9 +200,10 @@ dependencies:
               - libcufile-dev
   test_python:
     common:
-      - output_types: [conda, requirements]
+      - output_types: [conda, requirements, pyproject]
         packages:
-          - cudf=23.08
+          - cuda-python>=11.7.1,<12.0
+          - cudf==23.8.*
           - dask>=2022.05.2
           - distributed>=2022.05.2
           - pytest

diff --git a/legate/benchmarks/single-node-io.py b/legate/benchmarks/single-node-io.py
@@ -56,7 +56,6 @@ def run_cufile(args):
 
 
 def main(args):
-
     props = kvikio.DriverProperties()
     try:
         import pynvml.smi

diff --git a/legate/legate_kvikio/__init__.py b/legate/legate_kvikio/__init__.py
@@ -1,7 +1,6 @@
 # Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved.
 # See file LICENSE for terms.
 
-from . import _version
 from .cufile import CuFile  # noqa: F401
 
-__version__ = _version.get_versions()["version"]
+__version__ = "23.08.00"
-Original file line number
+Diff line change
@@ Expand Up / @@ -56,7 +56,6 @@ def run_cufile(args): @@
     def main(args):
         props = kvikio.DriverProperties()
         try:
             import pynvml.smi
@@ Expand Down @@