Make gpuCI and pre-commit style configurations consistent

.pre-commit-config.yaml

@@ -1,23 +1,45 @@
 repos:
-      - repo: https://github.com/timothycrosley/isort
-        rev: 5.0.7
+      - repo: https://github.com/pycqa/isort
+        rev: 5.6.4
         hooks:
               - id: isort
+                alias: isort-cudf
+                name: isort-cudf
+                args: ["--settings-path=python/cudf/setup.cfg"]
+                files: python/cudf/.*
+                exclude: __init__.py$
+                types: [text]
+                types_or: [python, cython, pyi]
+              - id: isort
+                alias: isort-cudf-kafka
+                name: isort-cudf-kafka
+                args: ["--settings-path=python/cudf_kafka/setup.cfg"]
+                files: python/cudf_kafka/.*
+                types: [text]
+                types_or: [python, cython]
+              - id: isort
+                alias: isort-custreamz
+                name: isort-custreamz
+                args: ["--settings-path=python/custreamz/setup.cfg"]
+                files: python/custreamz/.*
+              - id: isort
+                alias: isort-dask-cudf
+                name: isort-dask-cudf
+                args: ["--settings-path=python/dask_cudf/setup.cfg"]
+                files: python/dask_cudf/.*
       - repo: https://github.com/ambv/black
         rev: 19.10b0
         hooks:
               - id: black
+                files: python/.*
       - repo: https://gitlab.com/pycqa/flake8
         rev: 3.8.3
         hooks:
               - id: flake8
                 alias: flake8
                 name: flake8
                 args: ["--config=python/.flake8"]
-                types: [python]
-      - repo: https://gitlab.com/pycqa/flake8
-        rev: 3.8.3
-        hooks:
+                files: python/.*\.py$
               - id: flake8
                 alias: flake8-cython
                 name: flake8-cython

ci/checks/style.sh

@@ -13,41 +13,77 @@ LANG=C.UTF-8
 # Activate common conda env
 source activate gdf
 
-# Run isort and get results/return code
-ISORT=`isort --check-only python/**/*.py`
-ISORT_RETVAL=$?
+# Run isort-cudf and get results/return code
+ISORT_CUDF=`isort python/cudf --check-only --skip-glob *.pyx --settings-path=python/cudf/setup.cfg 2>&1`
+ISORT_CUDF_RETVAL=$?
+
+# Run isort-cudf-kafka and get results/return code
+ISORT_CUDF_KAFKA=`isort python/cudf_kafka --check-only --settings-path=python/cudf_kafka/setup.cfg 2>&1`
+ISORT_CUDF_KAFKA_RETVAL=$?
+
+# Run isort-custreamz and get results/return code
+ISORT_CUSTREAMZ=`isort python/custreamz --check-only --settings-path=python/custreamz/setup.cfg 2>&1`
+ISORT_CUSTREAMZ_RETVAL=$?
+
+# Run isort-dask-cudf and get results/return code
+ISORT_DASK_CUDF=`isort python/dask_cudf --check-only --settings-path=python/dask_cudf/setup.cfg 2>&1`
+ISORT_DASK_CUDF_RETVAL=$?
 
 # Run black and get results/return code
-BLACK=`black --check python`
+BLACK=`black --check python 2>&1`
 BLACK_RETVAL=$?
 
 # Run flake8 and get results/return code
-FLAKE=`flake8 --config=python/.flake8 python`
+FLAKE=`flake8 --config=python/.flake8 python 2>&1`
 FLAKE_RETVAL=$?
 
 # Run flake8-cython and get results/return code
-FLAKE_CYTHON=`flake8 --config=python/.flake8.cython`
+FLAKE_CYTHON=`flake8 --config=python/.flake8.cython 2>&1`
 FLAKE_CYTHON_RETVAL=$?
 
 # Run mypy and get results/return code
-MYPY_CUDF=`mypy --config=python/cudf/setup.cfg python/cudf/cudf`
+MYPY_CUDF=`mypy --config=python/cudf/setup.cfg python/cudf/cudf 2>&1`
 MYPY_CUDF_RETVAL=$?
 
 # Run pydocstyle and get results/return code
-PYDOCSTYLE=`pydocstyle --config=python/.flake8 python`
+PYDOCSTYLE=`pydocstyle --config=python/.flake8 python 2>&1`
 PYDOCSTYLE_RETVAL=$?
 
 # Run clang-format and check for a consistent code format
 CLANG_FORMAT=`python cpp/scripts/run-clang-format.py 2>&1`
 CLANG_FORMAT_RETVAL=$?
 
 # Output results if failure otherwise show pass
-if [ "$ISORT_RETVAL" != "0" ]; then
-  echo -e "\n\n>>>> FAILED: isort style check; begin output\n\n"
-  echo -e "$ISORT"
-  echo -e "\n\n>>>> FAILED: isort style check; end output\n\n"
+if [ "$ISORT_CUDF_RETVAL" != "0" ]; then
+  echo -e "\n\n>>>> FAILED: isort-cudf style check; begin output\n\n"
+  echo -e "$ISORT_CUDF"
+  echo -e "\n\n>>>> FAILED: isort-cudf style check; end output\n\n"
+else
+  echo -e "\n\n>>>> PASSED: isort-cudf style check\n\n"
+fi
+
+if [ "$ISORT_CUDF_KAFKA_RETVAL" != "0" ]; then
+  echo -e "\n\n>>>> FAILED: isort-cudf-kafka style check; begin output\n\n"
+  echo -e "$ISORT_CUDF_KAFKA"
+  echo -e "\n\n>>>> FAILED: isort-cudf-kafka style check; end output\n\n"
+else
+  echo -e "\n\n>>>> PASSED: isort-cudf-kafka style check\n\n"
+fi
+
+if [ "$ISORT_CUSTREAMZ_RETVAL" != "0" ]; then
+  echo -e "\n\n>>>> FAILED: isort-custreamz style check; begin output\n\n"
+  echo -e "$ISORT_CUSTREAMZ"
+  echo -e "\n\n>>>> FAILED: isort-custreamz style check; end output\n\n"
+else
+  echo -e "\n\n>>>> PASSED: isort-custreamz style check\n\n"
+fi
+
+if [ "$ISORT_DASK_CUDF_RETVAL" != "0" ]; then
+  echo -e "\n\n>>>> FAILED: isort-dask-cudf style check; begin output\n\n"
+  echo -e "$ISORT_DASK_CUDF"
+  echo -e "\n\n>>>> FAILED: isort-dask-cudf style check; end output\n\n"
 else
-  echo -e "\n\n>>>> PASSED: isort style check\n\n"
+  echo -e "\n\n>>>> PASSED: isort-dask-cudf style check\n\n"
 fi
 
 if [ "$BLACK_RETVAL" != "0" ]; then
@@ -103,7 +139,11 @@ HEADER_META=`ci/checks/headers_test.sh`
 HEADER_META_RETVAL=$?
 echo -e "$HEADER_META"
 
-RETVALS=($ISORT_RETVAL $BLACK_RETVAL $FLAKE_RETVAL $FLAKE_CYTHON_RETVAL $PYDOCSTYLE_RETVAL $CLANG_FORMAT_RETVAL $HEADER_META_RETVAL $MYPY_CUDF_RETVAL)
+RETVALS=(
+  $ISORT_CUDF_RETVAL $ISORT_CUDF_KAFKA_RETVAL $ISORT_CUSTREAMZ_RETVAL $ISORT_DASK_CUDF_RETVAL
+  $BLACK_RETVAL $FLAKE_RETVAL $FLAKE_CYTHON_RETVAL $PYDOCSTYLE_RETVAL $CLANG_FORMAT_RETVAL
+  $HEADER_META_RETVAL $MYPY_CUDF_RETVAL
+)
 IFS=$'\n'
 RETVAL=`echo "${RETVALS[*]}" | sort -nr | head -n1`
 

cpp/libcudf_kafka/tests/kafka_consumer_tests.cpp

@@ -20,8 +20,8 @@
 #include <string>
 #include "cudf_kafka/kafka_consumer.hpp"
 
-#include <cudf/io/datasource.hpp>
 #include <cudf/io/csv.hpp>
+#include <cudf/io/datasource.hpp>
 
 namespace kafka = cudf::io::external::kafka;
 

cpp/scripts/run-clang-format.py

@@ -29,10 +29,10 @@
 DEFAULT_DIRS = [
     "cpp/benchmarks",
     "cpp/include",
-    "cpp/include/cudf",
-    "cpp/include/nvtext",
+    "cpp/libcudf_kafka",
     "cpp/src",
     "cpp/tests",
+    "java/src/main/native",
 ]
 
 

java/src/main/native/include/jni_utils.hpp

@@ -243,21 +243,13 @@ template <typename N_TYPE, typename J_ARRAY_TYPE, typename ACCESSOR> class nativ
     return data_ptr;
   }
 
-  const N_TYPE *const begin() const {
-    return data();
-  }
+  const N_TYPE *const begin() const { return data(); }
 
-  N_TYPE *begin() {
-    return data();
-  }
+  N_TYPE *begin() { return data(); }
 
-  const N_TYPE *const end() const {
-    return data() + size();
-  }
+  const N_TYPE *const end() const { return data() + size(); }
 
-  N_TYPE *end() {
-    return data() + size();
-  }
+  N_TYPE *end() { return data() + size(); }
 
   const J_ARRAY_TYPE get_jArray() const { return orig; }
 
@@ -315,7 +307,7 @@ template <typename T> class native_jpointerArray {
 
   int size() const noexcept { return wrapped.size(); }
 
-  T *operator[](int index) const { 
+  T *operator[](int index) const {
     if (data() == NULL) {
       throw_java_exception(env, NPE_CLASS, "pointer is NULL");
     }
@@ -754,14 +746,13 @@ inline void jni_cuda_check(JNIEnv *const env, cudaError_t cuda_status) {
     if (cudaErrorMemoryAllocation == cudaPeekAtLastError()) {                                      \
       cudaGetLastError();                                                                          \
     }                                                                                              \
-    auto what = std::string("Could not allocate native memory: ") +                                \
-      (e.what() == nullptr ? "" : e.what());                                                       \
+    auto what =                                                                                    \
+        std::string("Could not allocate native memory: ") + (e.what() == nullptr ? "" : e.what()); \
     JNI_CHECK_THROW_NEW(env, cudf::jni::OOM_CLASS, what.c_str(), ret_val);                         \
   }                                                                                                \
   catch (const std::exception &e) {                                                                \
     /* If jni_exception caught then a Java exception is pending and this will not overwrite it. */ \
     JNI_CHECK_THROW_NEW(env, class_name, e.what(), ret_val);                                       \
   }
 
-#define CATCH_STD(env, ret_val)                                                                    \
-    CATCH_STD_CLASS(env, cudf::jni::CUDF_ERROR_CLASS, ret_val)
+#define CATCH_STD(env, ret_val) CATCH_STD_CLASS(env, cudf::jni::CUDF_ERROR_CLASS, ret_val)

java/src/main/native/src/AggregationJni.cpp

@@ -20,8 +20,7 @@
 
 extern "C" {
 
-JNIEXPORT void JNICALL Java_ai_rapids_cudf_Aggregation_close(JNIEnv *env,
-                                                             jclass class_object,
+JNIEXPORT void JNICALL Java_ai_rapids_cudf_Aggregation_close(JNIEnv *env, jclass class_object,
                                                              jlong ptr) {
   try {
     cudf::jni::auto_set_device(env);
@@ -51,7 +50,7 @@ JNIEXPORT jlong JNICALL Java_ai_rapids_cudf_Aggregation_createNoParamAgg(JNIEnv
       case 3: // MAX
         ret = cudf::make_max_aggregation();
         break;
-      //case 4 COUNT
+      // case 4 COUNT
       case 5: // ANY
         ret = cudf::make_any_aggregation();
         break;
@@ -84,9 +83,7 @@ JNIEXPORT jlong JNICALL Java_ai_rapids_cudf_Aggregation_createNoParamAgg(JNIEnv
       // case 18: COLLECT_LIST
       // case 19: COLLECT_SET
       // case 20: MERGE_LISTS
-      case 20:
-        ret = cudf::make_merge_lists_aggregation();
-        break;
+      case 20: ret = cudf::make_merge_lists_aggregation(); break;
       // case 21: MERGE_SETS
       // case 22: LEAD
       // case 23: LAG
@@ -107,18 +104,16 @@ JNIEXPORT jlong JNICALL Java_ai_rapids_cudf_Aggregation_createNthAgg(JNIEnv *env
   try {
     cudf::jni::auto_set_device(env);
 
-    std::unique_ptr<cudf::aggregation> ret = 
-        cudf::make_nth_element_aggregation(offset,
-                include_nulls ? cudf::null_policy::INCLUDE : cudf::null_policy::EXCLUDE);
+    std::unique_ptr<cudf::aggregation> ret = cudf::make_nth_element_aggregation(
+        offset, include_nulls ? cudf::null_policy::INCLUDE : cudf::null_policy::EXCLUDE);
     return reinterpret_cast<jlong>(ret.release());
   }
   CATCH_STD(env, 0);
 }
 
 JNIEXPORT jlong JNICALL Java_ai_rapids_cudf_Aggregation_createDdofAgg(JNIEnv *env,
                                                                       jclass class_object,
-                                                                      jint kind,
-                                                                      jint ddof) {
+                                                                      jint kind, jint ddof) {
   try {
     cudf::jni::auto_set_device(env);
 
@@ -184,8 +179,7 @@ JNIEXPORT jlong JNICALL Java_ai_rapids_cudf_Aggregation_createQuantAgg(JNIEnv *e
 
 JNIEXPORT jlong JNICALL Java_ai_rapids_cudf_Aggregation_createLeadLagAgg(JNIEnv *env,
                                                                          jclass class_object,
-                                                                         jint kind,
-                                                                         jint offset) {
+                                                                         jint kind, jint offset) {
   try {
     cudf::jni::auto_set_device(env);
 
@@ -205,9 +199,8 @@ JNIEXPORT jlong JNICALL Java_ai_rapids_cudf_Aggregation_createLeadLagAgg(JNIEnv
   CATCH_STD(env, 0);
 }
 
-JNIEXPORT jlong JNICALL Java_ai_rapids_cudf_Aggregation_createCollectListAgg(JNIEnv *env,
-                                                                             jclass class_object,
-                                                                             jboolean include_nulls) {
+JNIEXPORT jlong JNICALL Java_ai_rapids_cudf_Aggregation_createCollectListAgg(
+    JNIEnv *env, jclass class_object, jboolean include_nulls) {
   try {
     cudf::jni::auto_set_device(env);
     cudf::null_policy policy =
@@ -231,9 +224,8 @@ JNIEXPORT jlong JNICALL Java_ai_rapids_cudf_Aggregation_createCollectSetAgg(JNIE
         nulls_equal ? cudf::null_equality::EQUAL : cudf::null_equality::UNEQUAL;
     cudf::nan_equality nan_equality =
         nans_equal ? cudf::nan_equality::ALL_EQUAL : cudf::nan_equality::UNEQUAL;
-    std::unique_ptr<cudf::aggregation> ret = cudf::make_collect_set_aggregation(null_policy,
-                                                                                null_equality,
-                                                                                nan_equality);
+    std::unique_ptr<cudf::aggregation> ret =
+        cudf::make_collect_set_aggregation(null_policy, null_equality, nan_equality);
     return reinterpret_cast<jlong>(ret.release());
   }
   CATCH_STD(env, 0);
@@ -249,8 +241,8 @@ JNIEXPORT jlong JNICALL Java_ai_rapids_cudf_Aggregation_createMergeSetsAgg(JNIEn
         nulls_equal ? cudf::null_equality::EQUAL : cudf::null_equality::UNEQUAL;
     cudf::nan_equality nan_equality =
         nans_equal ? cudf::nan_equality::ALL_EQUAL : cudf::nan_equality::UNEQUAL;
-    std::unique_ptr<cudf::aggregation> ret = cudf::make_merge_sets_aggregation(null_equality,
-                                                                               nan_equality);
+    std::unique_ptr<cudf::aggregation> ret =
+        cudf::make_merge_sets_aggregation(null_equality, nan_equality);
     return reinterpret_cast<jlong>(ret.release());
   }
   CATCH_STD(env, 0);