Consolidate linter configuration into pyproject.toml (#1317)

raft currently stores config information for linters across many different pyproject.toml and setup.cfg files. This PR moves all linter configuration out of setup.cfg files into pyproject.toml where possible. flake8 does not support pyproject.toml, so that data is placed in the flake8-specific .flake8 file. Old and unused .flake8 files have been removed. These changes are a precursor to build system migrations that should allow further consolidation of the data in these files.

Authors:
  - Vyas Ramasubramani (https://github.com/vyasr)

Approvers:
  - Ben Frederickson (https://github.com/benfred)

URL: #1317

.flake8

@@ -0,0 +1,24 @@
+# Copyright (c) 2022-2023, NVIDIA CORPORATION.
+
+[flake8]
+filename = *.py, *.pyx, *.pxd, *.pxi
+exclude = __init__.py, *.egg, build, docs, .git
+force-check = True
+ignore =
+    # line break before binary operator
+    W503,
+    # whitespace before :
+    E203
+per-file-ignores =
+    # Rules ignored only in Cython:
+    # E211: whitespace before '(' (used in multi-line imports)
+    # E225: Missing whitespace around operators (breaks cython casting syntax like <int>)
+    # E226: Missing whitespace around arithmetic operators (breaks cython pointer syntax like int*)
+    # E227: Missing whitespace around bitwise or shift operator (Can also break casting syntax)
+    # E275: Missing whitespace after keyword (Doesn't work with Cython except?)
+    # E402: invalid syntax (works for Python, not Cython)
+    # E999: invalid syntax (works for Python, not Cython)
+    # W504: line break after binary operator (breaks lines that end with a pointer)
+    *.pyx: E211, E225, E226, E227, E275, E402, E999, W504
+    *.pxd: E211, E225, E226, E227, E275, E402, E999, W504
+    *.pxi: E211, E225, E226, E227, E275, E402, E999, W504

.pre-commit-config.yaml

@@ -21,7 +21,7 @@ repos:
         rev: 5.0.4
         hooks:
               - id: flake8
-                args: ["--config=setup.cfg"]
+                args: ["--config=.flake8"]
                 files: python/.*$
                 types: [file]
                 types_or: [python, cython]
@@ -31,15 +31,17 @@ repos:
         hooks:
               - id: mypy
                 additional_dependencies: [types-cachetools]
-                args: ["--config-file=setup.cfg",
+                args: ["--config-file=pyproject.toml",
                        "python/pylibraft/pylibraft",
                        "python/raft-dask/raft_dask"]
                 pass_filenames: false
       - repo: https://github.com/PyCQA/pydocstyle
         rev: 6.1.1
         hooks:
               - id: pydocstyle
-                args: ["--config=setup.cfg"]
+                # https://github.com/PyCQA/pydocstyle/issues/603
+                additional_dependencies: [toml]
+                args: ["--config=pyproject.toml"]
       - repo: https://github.com/pre-commit/mirrors-clang-format
         rev: v11.1.0
         hooks:
@@ -92,9 +94,11 @@ repos:
                 language: python
                 additional_dependencies: [gitpython]
       - repo: https://github.com/codespell-project/codespell
-        rev: v2.1.0
+        rev: v2.2.2
         hooks:
               - id: codespell
+                additional_dependencies: [tomli]
+                args: ["--toml", "pyproject.toml"]
                 exclude: (?x)^(^CHANGELOG.md$)
       - repo: https://github.com/rapidsai/dependency-file-generator
         rev: v1.4.0

cpp/include/raft/matrix/detail/linewise_op.cuh

@@ -201,7 +201,7 @@ struct Linewise {
   }
 
   /**
-   * @brief Same as loadVec, but padds data with Ones
+   * @brief Same as loadVec, but pads data with Ones
    *
    * @tparam VecT Type of the vector to load
    * @param shm

cpp/include/raft/neighbors/ivf_pq_serialize.cuh

@@ -38,7 +38,7 @@ namespace raft::neighbors::ivf_pq {
  * // create an output stream
  * std::ostream os(std::cout.rdbuf());
  * // create an index with `auto index = ivf_pq::build(...);`
- * raft::serailize(handle, os, index);
+ * raft::serialize(handle, os, index);
  * @endcode
  *
  * @tparam IdxT type of the index
@@ -68,7 +68,7 @@ void serialize(raft::device_resources const& handle, std::ostream& os, const ind
  * // create a string with a filepath
  * std::string filename("/path/to/index");
  * // create an index with `auto index = ivf_pq::build(...);`
- * raft::serailize(handle, filename, index);
+ * raft::serialize(handle, filename, index);
  * @endcode
  *
  * @tparam IdxT type of the index

cpp/include/raft/random/detail/multi_variable_gaussian.cuh

@@ -32,7 +32,7 @@
 #include <stdio.h>
 #include <type_traits>
 
-// mvg.cuh takes in matrices that are colomn major (as in fortran)
+// mvg.cuh takes in matrices that are column major (as in fortran)
 #define IDX2C(i, j, ld) (j * ld + i)
 
 namespace raft::random {

cpp/scripts/gitutils.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2020-2022, NVIDIA CORPORATION.
+# Copyright (c) 2020-2023, NVIDIA CORPORATION.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -85,7 +85,7 @@ def repo_version_major_minor():
 def determine_merge_commit(current_branch="HEAD"):
     """
     When running outside of CI, this will estimate the target merge commit hash
-    of `current_branch` by finding a common ancester with the remote branch
+    of `current_branch` by finding a common ancestor with the remote branch
     'branch-{major}.{minor}' where {major} and {minor} are determined from the
     repo version.
 

cpp/test/util/bitonic_sort.cu

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2022, NVIDIA CORPORATION.
+ * Copyright (c) 2022-2023, NVIDIA CORPORATION.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -54,7 +54,7 @@ __global__ void bitonic_kernel(T* arr, bool ascending, int warp_width, int n_inp
   const int subwarp_id   = tid / warp_width;
   const int subwarp_lane = tid % warp_width;
   T local_arr[Capacity];  // NOLINT
-  // Split the data into chunks of size `warp_width * Capacity`, each thread poiting
+  // Split the data into chunks of size `warp_width * Capacity`, each thread pointing
   // to the beginning of its stride within the chunk.
   T* per_thread_arr = arr + subwarp_id * warp_width * Capacity + subwarp_lane;
 

pyproject.toml

@@ -17,3 +17,31 @@ force-exclude = '''
     dist
 )/
 '''
+
+[tool.pydocstyle]
+# Due to https://github.com/PyCQA/pydocstyle/issues/363, we must exclude rather
+# than include using match-dir. Note that as discussed in
+# https://stackoverflow.com/questions/65478393/how-to-filter-directories-using-the-match-dir-flag-for-pydocstyle,
+# unlike the match option above this match-dir will have no effect when
+# pydocstyle is invoked from pre-commit. Therefore this exclusion list must
+# also be maintained in the pre-commit config file.
+match-dir = "^(?!(ci|cpp|conda|docs)).*$"
+select = "D201, D204, D206, D207, D208, D209, D210, D211, D214, D215, D300, D301, D302, D403, D405, D406, D407, D408, D409, D410, D411, D412, D414, D418"
+    # Would like to enable the following rules in the future:
+    # D200, D202, D205, D400
+
+[tool.mypy]
+ignore_missing_imports = true
+# If we don't specify this, then mypy will check excluded files if
+# they are imported by a checked file.
+follow_imports = "skip"
+
+[tool.codespell]
+# note: pre-commit passes explicit lists of files here, which this skip file list doesn't override -
+# this is only to allow you to run codespell interactively
+skip = "./.git,./.github,./cpp/build,.*egg-info.*,./.mypy_cache,.*_skbuild"
+# ignore short words, and typename parameters like OffsetT
+ignore-regex = "\\b(.{1,4}|[A-Z]\\w*T)\\b"
+ignore-words-list = "inout,numer"
+builtin = "clear"
+quiet-level = 3

python/pylibraft/pyproject.toml

@@ -25,3 +25,46 @@ requires = [
     "rmm==23.4.*",
 ]
 build-backend = "setuptools.build_meta"
+
+[tool.isort]
+line_length = 79
+multi_line_output = 3
+include_trailing_comma = true
+force_grid_wrap = 0
+combine_as_imports = true
+order_by_type = true
+known_dask = [
+    "dask",
+    "distributed",
+    "dask_cuda",
+]
+known_rapids = [
+    "rmm",
+]
+known_first_party = [
+    "pylibraft",
+]
+default_section = "THIRDPARTY"
+sections = [
+    "FUTURE",
+    "STDLIB",
+    "THIRDPARTY",
+    "DASK",
+    "RAPIDS",
+    "FIRSTPARTY",
+    "LOCALFOLDER",
+]
+skip = [
+    "thirdparty",
+    ".eggs",
+    ".git",
+    ".hg",
+    ".mypy_cache",
+    ".tox",
+    ".venv",
+    "_build",
+    "buck-out",
+    "build",
+    "dist",
+    "__init__.py",
+]

python/raft-dask/pyproject.toml

@@ -22,3 +22,47 @@ requires = [
     "cmake>=3.23.1,!=3.25.0",
     "ninja",
 ]
+
+[tool.isort]
+line_length = 79
+multi_line_output = 3
+include_trailing_comma = true
+force_grid_wrap = 0
+combine_as_imports = true
+order_by_type = true
+known_dask = [
+    "dask",
+    "distributed",
+    "dask_cuda",
+]
+known_rapids = [
+    "pylibraft",
+    "rmm",
+]
+known_first_party = [
+    "raft_dask",
+]
+default_section = "THIRDPARTY"
+sections = [
+    "FUTURE",
+    "STDLIB",
+    "THIRDPARTY",
+    "DASK",
+    "RAPIDS",
+    "FIRSTPARTY",
+    "LOCALFOLDER",
+]
+skip = [
+    "thirdparty",
+    ".eggs",
+    ".git",
+    ".hg",
+    ".mypy_cache",
+    ".tox",
+    ".venv",
+    "_build",
+    "buck-out",
+    "build",
+    "dist",
+    "__init__.py",
+]

python/raft-dask/raft_dask/common/comms.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2020-2022, NVIDIA CORPORATION.
+# Copyright (c) 2020-2023, NVIDIA CORPORATION.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -19,10 +19,10 @@
 import warnings
 from collections import OrderedDict
 
-from pylibraft.common.handle import Handle
-
 from dask.distributed import default_client, get_worker
 
+from pylibraft.common.handle import Handle
+
 from .comms_utils import (
     inject_comms_on_handle,
     inject_comms_on_handle_coll_only,