Use pre-commit for CI

.pre-commit-config.yaml

@@ -48,6 +48,11 @@ repos:
       - repo: local
         hooks:
               - id: clang-format
+                # Using the pre-commit stage to simplify invocation of all
+                # other hooks simultaneously (via any other hook stage).  This
+                # can be removed if we also move to running clang-format
+                # entirely through pre-commit.
+                stages: [commit]
                 name: clang-format
                 description: Format files with ClangFormat.
                 entry: clang-format -i

ci/checks/style.sh

@@ -14,119 +14,13 @@ LANG=C.UTF-8
 . /opt/conda/etc/profile.d/conda.sh
 conda activate rapids
 
-# Run isort-cudf and get results/return code
-ISORT_CUDF=`isort python/cudf --check-only --settings-path=python/cudf/setup.cfg 2>&1`
-ISORT_CUDF_RETVAL=$?
-
-# Run isort-cudf-kafka and get results/return code
-ISORT_CUDF_KAFKA=`isort python/cudf_kafka --check-only --settings-path=python/cudf_kafka/setup.cfg 2>&1`
-ISORT_CUDF_KAFKA_RETVAL=$?
-
-# Run isort-custreamz and get results/return code
-ISORT_CUSTREAMZ=`isort python/custreamz --check-only --settings-path=python/custreamz/setup.cfg 2>&1`
-ISORT_CUSTREAMZ_RETVAL=$?
-
-# Run isort-dask-cudf and get results/return code
-ISORT_DASK_CUDF=`isort python/dask_cudf --check-only --settings-path=python/dask_cudf/setup.cfg 2>&1`
-ISORT_DASK_CUDF_RETVAL=$?
-
-# Run black and get results/return code
-BLACK=`black --check python 2>&1`
-BLACK_RETVAL=$?
-
-# Run flake8 and get results/return code
-FLAKE=`flake8 --config=python/.flake8 python 2>&1`
-FLAKE_RETVAL=$?
-
-# Run flake8-cython and get results/return code
-FLAKE_CYTHON=`flake8 --config=python/.flake8.cython 2>&1`
-FLAKE_CYTHON_RETVAL=$?
-
-# Run mypy and get results/return code
-MYPY_CUDF=`mypy --config=python/cudf/setup.cfg python/cudf/cudf 2>&1`
-MYPY_CUDF_RETVAL=$?
-
-# Run pydocstyle and get results/return code
-PYDOCSTYLE=`pydocstyle --config=python/.flake8 python 2>&1`
-PYDOCSTYLE_RETVAL=$?
+pre-commit run --hook-stage manual --all-files
+PRE_COMMIT_RETVAL=$?
 
 # Run clang-format and check for a consistent code format
 CLANG_FORMAT=`python cpp/scripts/run-clang-format.py 2>&1`
 CLANG_FORMAT_RETVAL=$?
 
-# Output results if failure otherwise show pass
-if [ "$ISORT_CUDF_RETVAL" != "0" ]; then
-  echo -e "\n\n>>>> FAILED: isort-cudf style check; begin output\n\n"
-  echo -e "$ISORT_CUDF"
-  echo -e "\n\n>>>> FAILED: isort-cudf style check; end output\n\n"
-else
-  echo -e "\n\n>>>> PASSED: isort-cudf style check\n\n"
-fi
-
-if [ "$ISORT_CUDF_KAFKA_RETVAL" != "0" ]; then
-  echo -e "\n\n>>>> FAILED: isort-cudf-kafka style check; begin output\n\n"
-  echo -e "$ISORT_CUDF_KAFKA"
-  echo -e "\n\n>>>> FAILED: isort-cudf-kafka style check; end output\n\n"
-else
-  echo -e "\n\n>>>> PASSED: isort-cudf-kafka style check\n\n"
-fi
-
-if [ "$ISORT_CUSTREAMZ_RETVAL" != "0" ]; then
-  echo -e "\n\n>>>> FAILED: isort-custreamz style check; begin output\n\n"
-  echo -e "$ISORT_CUSTREAMZ"
-  echo -e "\n\n>>>> FAILED: isort-custreamz style check; end output\n\n"
-else
-  echo -e "\n\n>>>> PASSED: isort-custreamz style check\n\n"
-fi
-
-if [ "$ISORT_DASK_CUDF_RETVAL" != "0" ]; then
-  echo -e "\n\n>>>> FAILED: isort-dask-cudf style check; begin output\n\n"
-  echo -e "$ISORT_DASK_CUDF"
-  echo -e "\n\n>>>> FAILED: isort-dask-cudf style check; end output\n\n"
-else
-  echo -e "\n\n>>>> PASSED: isort-dask-cudf style check\n\n"
-fi
-
-if [ "$BLACK_RETVAL" != "0" ]; then
-  echo -e "\n\n>>>> FAILED: black style check; begin output\n\n"
-  echo -e "$BLACK"
-  echo -e "\n\n>>>> FAILED: black style check; end output\n\n"
-else
-  echo -e "\n\n>>>> PASSED: black style check\n\n"
-fi
-
-if [ "$FLAKE_RETVAL" != "0" ]; then
-  echo -e "\n\n>>>> FAILED: flake8 style check; begin output\n\n"
-  echo -e "$FLAKE"
-  echo -e "\n\n>>>> FAILED: flake8 style check; end output\n\n"
-else
-  echo -e "\n\n>>>> PASSED: flake8 style check\n\n"
-fi
-
-if [ "$FLAKE_CYTHON_RETVAL" != "0" ]; then
-  echo -e "\n\n>>>> FAILED: flake8-cython style check; begin output\n\n"
-  echo -e "$FLAKE_CYTHON"
-  echo -e "\n\n>>>> FAILED: flake8-cython style check; end output\n\n"
-else
-  echo -e "\n\n>>>> PASSED: flake8-cython style check\n\n"
-fi
-
-if [ "$MYPY_CUDF_RETVAL" != "0" ]; then
-  echo -e "\n\n>>>> FAILED: mypy style check; begin output\n\n"
-  echo -e "$MYPY_CUDF"
-  echo -e "\n\n>>>> FAILED: mypy style check; end output\n\n"
-else
-  echo -e "\n\n>>>> PASSED: mypy style check\n\n"
-fi
-
-if [ "$PYDOCSTYLE_RETVAL" != "0" ]; then
-  echo -e "\n\n>>>> FAILED: pydocstyle style check; begin output\n\n"
-  echo -e "$PYDOCSTYLE"
-  echo -e "\n\n>>>> FAILED: pydocstyle style check; end output\n\n"
-else
-  echo -e "\n\n>>>> PASSED: pydocstyle style check\n\n"
-fi
-
 if [ "$CLANG_FORMAT_RETVAL" != "0" ]; then
   echo -e "\n\n>>>> FAILED: clang format check; begin output\n\n"
   echo -e "$CLANG_FORMAT"
@@ -141,9 +35,7 @@ HEADER_META_RETVAL=$?
 echo -e "$HEADER_META"
 
 RETVALS=(
-  $ISORT_CUDF_RETVAL $ISORT_CUDF_KAFKA_RETVAL $ISORT_CUSTREAMZ_RETVAL $ISORT_DASK_CUDF_RETVAL
-  $BLACK_RETVAL $FLAKE_RETVAL $FLAKE_CYTHON_RETVAL $PYDOCSTYLE_RETVAL $CLANG_FORMAT_RETVAL
-  $HEADER_META_RETVAL $MYPY_CUDF_RETVAL
+  $PRE_COMMIT_RETVAL $CLANG_FORMAT_RETVAL
 )
 IFS=$'\n'
 RETVAL=`echo "${RETVALS[*]}" | sort -nr | head -n1`

conda/environments/cudf_dev_cuda11.0.yml

@@ -34,12 +34,8 @@ dependencies:
   - pandoc=<2.0.0
   - cudatoolkit=11.0
   - pip
-  - flake8=3.8.3
-  - black=19.10
-  - isort=5.6.4
-  - mypy=0.782
   - typing_extensions
-  - pre_commit
+  - pre-commit
   - dask>=2021.09.1
   - distributed>=2021.09.1
   - streamz

conda/environments/cudf_dev_cuda11.2.yml

@@ -34,12 +34,8 @@ dependencies:
   - pandoc=<2.0.0
   - cudatoolkit=11.2
   - pip
-  - flake8=3.8.3
-  - black=19.10
-  - isort=5.6.4
-  - mypy=0.782
   - typing_extensions
-  - pre_commit
+  - pre-commit
   - dask>=2021.09.1
   - distributed>=2021.09.1
   - streamz