Merge branch 'branch-21.12' of https://github.com/rapidsai/cudf into …

…9177

.pre-commit-config.yaml

@@ -1,5 +1,5 @@
 repos:
-      - repo: https://github.com/pycqa/isort
+      - repo: https://github.com/PyCQA/isort
         rev: 5.6.4
         hooks:
               - id: isort
@@ -27,12 +27,12 @@ repos:
                 name: isort-dask-cudf
                 args: ["--settings-path=python/dask_cudf/setup.cfg"]
                 files: python/dask_cudf/.*
-      - repo: https://github.com/ambv/black
+      - repo: https://github.com/psf/black
         rev: 19.10b0
         hooks:
               - id: black
                 files: python/.*
-      - repo: https://gitlab.com/pycqa/flake8
+      - repo: https://github.com/PyCQA/flake8
         rev: 3.8.3
         hooks:
               - id: flake8
@@ -45,27 +45,49 @@ repos:
                 name: flake8-cython
                 args: ["--config=python/.flake8.cython"]
                 types: [cython]
-      - repo: local
-        hooks:
-              - id: clang-format
-                name: clang-format
-                description: Format files with ClangFormat.
-                entry: clang-format -i
-                language: system
-                files: \.(cu|cuh|h|hpp|cpp|inl)$
-                args: ['-fallback-style=none']
       - repo: https://github.com/pre-commit/mirrors-mypy
         rev: 'v0.782'
         hooks:
               - id: mypy
                 args: ["--config-file=python/cudf/setup.cfg", "python/cudf/cudf"]
                 pass_filenames: false
-      - repo: https://github.com/pycqa/pydocstyle
-        rev: 6.0.0
+      - repo: https://github.com/PyCQA/pydocstyle
+        rev: 6.1.1
         hooks:
               - id: pydocstyle
                 args: ["--config=python/.flake8"]
-
+      - repo: local
+        hooks:
+              - id: clang-format
+                # Using the pre-commit stage to simplify invocation of all
+                # other hooks simultaneously (via any other hook stage).  This
+                # can be removed if we also move to running clang-format
+                # entirely through pre-commit.
+                stages: [commit]
+                name: clang-format
+                description: Format files with ClangFormat.
+                entry: clang-format -i
+                language: system
+                files: \.(cu|cuh|h|hpp|cpp|inl)$
+                args: ['-fallback-style=none']
+              - id: cmake-format
+                name: cmake-format
+                entry: bash cpp/scripts/run-cmake-format.sh cmake-format
+                language: python
+                types: [cmake]
+                # Note that pre-commit autoupdate does not update the versions
+                # of dependencies, so we'll have to update this manually.
+                additional_dependencies:
+                  - cmake-format==0.6.11
+              - id: cmake-lint
+                name: cmake-lint
+                entry: bash cpp/scripts/run-cmake-format.sh cmake-lint
+                language: python
+                types: [cmake]
+                # Note that pre-commit autoupdate does not update the versions
+                # of dependencies, so we'll have to update this manually.
+                additional_dependencies:
+                  - cmake-format==0.6.11
 
 default_language_version:
       python: python3

build.sh

@@ -18,7 +18,7 @@ ARGS=$*
 REPODIR=$(cd $(dirname $0); pwd)
 
 VALIDARGS="clean libcudf cudf dask_cudf benchmarks tests libcudf_kafka cudf_kafka custreamz -v -g -n -l --allgpuarch --disable_nvtx --show_depr_warn --ptds -h"
-HELP="$0 [clean] [libcudf] [cudf] [dask_cudf] [benchmarks] [tests] [libcudf_kafka] [cudf_kafka] [custreamz] [-v] [-g] [-n] [-h] [-l] [--cmake-args=\"<args>\"]
+HELP="$0 [clean] [libcudf] [cudf] [dask_cudf] [benchmarks] [tests] [libcudf_kafka] [cudf_kafka] [custreamz] [-v] [-g] [-n] [-h] [-l] [--cmake-args=\\\"<args>\\\"]
    clean                         - remove all existing build artifacts and configuration (start
                                    over)
    libcudf                       - build the cudf C++ code only

ci/checks/style.sh

@@ -14,119 +14,18 @@ LANG=C.UTF-8
 . /opt/conda/etc/profile.d/conda.sh
 conda activate rapids
 
-# Run isort-cudf and get results/return code
-ISORT_CUDF=`isort python/cudf --check-only --settings-path=python/cudf/setup.cfg 2>&1`
-ISORT_CUDF_RETVAL=$?
+FORMAT_FILE_URL=https://raw.githubusercontent.com/rapidsai/rapids-cmake/main/cmake-format-rapids-cmake.json
+export RAPIDS_CMAKE_FORMAT_FILE=/tmp/rapids_cmake_ci/cmake-formats-rapids-cmake.json
+mkdir -p $(dirname ${RAPIDS_CMAKE_FORMAT_FILE})
+wget -O ${RAPIDS_CMAKE_FORMAT_FILE} ${FORMAT_FILE_URL}
 
-# Run isort-cudf-kafka and get results/return code
-ISORT_CUDF_KAFKA=`isort python/cudf_kafka --check-only --settings-path=python/cudf_kafka/setup.cfg 2>&1`
-ISORT_CUDF_KAFKA_RETVAL=$?
-
-# Run isort-custreamz and get results/return code
-ISORT_CUSTREAMZ=`isort python/custreamz --check-only --settings-path=python/custreamz/setup.cfg 2>&1`
-ISORT_CUSTREAMZ_RETVAL=$?
-
-# Run isort-dask-cudf and get results/return code
-ISORT_DASK_CUDF=`isort python/dask_cudf --check-only --settings-path=python/dask_cudf/setup.cfg 2>&1`
-ISORT_DASK_CUDF_RETVAL=$?
-
-# Run black and get results/return code
-BLACK=`black --check python 2>&1`
-BLACK_RETVAL=$?
-
-# Run flake8 and get results/return code
-FLAKE=`flake8 --config=python/.flake8 python 2>&1`
-FLAKE_RETVAL=$?
-
-# Run flake8-cython and get results/return code
-FLAKE_CYTHON=`flake8 --config=python/.flake8.cython 2>&1`
-FLAKE_CYTHON_RETVAL=$?
-
-# Run mypy and get results/return code
-MYPY_CUDF=`mypy --config=python/cudf/setup.cfg python/cudf/cudf 2>&1`
-MYPY_CUDF_RETVAL=$?
-
-# Run pydocstyle and get results/return code
-PYDOCSTYLE=`pydocstyle --config=python/.flake8 python 2>&1`
-PYDOCSTYLE_RETVAL=$?
+pre-commit run --hook-stage manual --all-files
+PRE_COMMIT_RETVAL=$?
 
 # Run clang-format and check for a consistent code format
 CLANG_FORMAT=`python cpp/scripts/run-clang-format.py 2>&1`
 CLANG_FORMAT_RETVAL=$?
 
-# Output results if failure otherwise show pass
-if [ "$ISORT_CUDF_RETVAL" != "0" ]; then
-  echo -e "\n\n>>>> FAILED: isort-cudf style check; begin output\n\n"
-  echo -e "$ISORT_CUDF"
-  echo -e "\n\n>>>> FAILED: isort-cudf style check; end output\n\n"
-else
-  echo -e "\n\n>>>> PASSED: isort-cudf style check\n\n"
-fi
-
-if [ "$ISORT_CUDF_KAFKA_RETVAL" != "0" ]; then
-  echo -e "\n\n>>>> FAILED: isort-cudf-kafka style check; begin output\n\n"
-  echo -e "$ISORT_CUDF_KAFKA"
-  echo -e "\n\n>>>> FAILED: isort-cudf-kafka style check; end output\n\n"
-else
-  echo -e "\n\n>>>> PASSED: isort-cudf-kafka style check\n\n"
-fi
-
-if [ "$ISORT_CUSTREAMZ_RETVAL" != "0" ]; then
-  echo -e "\n\n>>>> FAILED: isort-custreamz style check; begin output\n\n"
-  echo -e "$ISORT_CUSTREAMZ"
-  echo -e "\n\n>>>> FAILED: isort-custreamz style check; end output\n\n"
-else
-  echo -e "\n\n>>>> PASSED: isort-custreamz style check\n\n"
-fi
-
-if [ "$ISORT_DASK_CUDF_RETVAL" != "0" ]; then
-  echo -e "\n\n>>>> FAILED: isort-dask-cudf style check; begin output\n\n"
-  echo -e "$ISORT_DASK_CUDF"
-  echo -e "\n\n>>>> FAILED: isort-dask-cudf style check; end output\n\n"
-else
-  echo -e "\n\n>>>> PASSED: isort-dask-cudf style check\n\n"
-fi
-
-if [ "$BLACK_RETVAL" != "0" ]; then
-  echo -e "\n\n>>>> FAILED: black style check; begin output\n\n"
-  echo -e "$BLACK"
-  echo -e "\n\n>>>> FAILED: black style check; end output\n\n"
-else
-  echo -e "\n\n>>>> PASSED: black style check\n\n"
-fi
-
-if [ "$FLAKE_RETVAL" != "0" ]; then
-  echo -e "\n\n>>>> FAILED: flake8 style check; begin output\n\n"
-  echo -e "$FLAKE"
-  echo -e "\n\n>>>> FAILED: flake8 style check; end output\n\n"
-else
-  echo -e "\n\n>>>> PASSED: flake8 style check\n\n"
-fi
-
-if [ "$FLAKE_CYTHON_RETVAL" != "0" ]; then
-  echo -e "\n\n>>>> FAILED: flake8-cython style check; begin output\n\n"
-  echo -e "$FLAKE_CYTHON"
-  echo -e "\n\n>>>> FAILED: flake8-cython style check; end output\n\n"
-else
-  echo -e "\n\n>>>> PASSED: flake8-cython style check\n\n"
-fi
-
-if [ "$MYPY_CUDF_RETVAL" != "0" ]; then
-  echo -e "\n\n>>>> FAILED: mypy style check; begin output\n\n"
-  echo -e "$MYPY_CUDF"
-  echo -e "\n\n>>>> FAILED: mypy style check; end output\n\n"
-else
-  echo -e "\n\n>>>> PASSED: mypy style check\n\n"
-fi
-
-if [ "$PYDOCSTYLE_RETVAL" != "0" ]; then
-  echo -e "\n\n>>>> FAILED: pydocstyle style check; begin output\n\n"
-  echo -e "$PYDOCSTYLE"
-  echo -e "\n\n>>>> FAILED: pydocstyle style check; end output\n\n"
-else
-  echo -e "\n\n>>>> PASSED: pydocstyle style check\n\n"
-fi
-
 if [ "$CLANG_FORMAT_RETVAL" != "0" ]; then
   echo -e "\n\n>>>> FAILED: clang format check; begin output\n\n"
   echo -e "$CLANG_FORMAT"
@@ -141,9 +40,7 @@ HEADER_META_RETVAL=$?
 echo -e "$HEADER_META"
 
 RETVALS=(
-  $ISORT_CUDF_RETVAL $ISORT_CUDF_KAFKA_RETVAL $ISORT_CUSTREAMZ_RETVAL $ISORT_DASK_CUDF_RETVAL
-  $BLACK_RETVAL $FLAKE_RETVAL $FLAKE_CYTHON_RETVAL $PYDOCSTYLE_RETVAL $CLANG_FORMAT_RETVAL
-  $HEADER_META_RETVAL $MYPY_CUDF_RETVAL
+  $PRE_COMMIT_RETVAL $CLANG_FORMAT_RETVAL
 )
 IFS=$'\n'
 RETVAL=`echo "${RETVALS[*]}" | sort -nr | head -n1`

ci/gpu/build.sh

@@ -83,7 +83,7 @@ gpuci_mamba_retry install -y \
                   "rapids-notebook-env=$MINOR_VERSION.*" \
                   "dask-cuda=${MINOR_VERSION}" \
                   "rmm=$MINOR_VERSION.*" \
-                  "ucx-py=0.22.*"
+                  "ucx-py=0.23.*"
 
 # https://docs.rapids.ai/maintainers/depmgmt/
 # gpuci_mamba_retry remove --force rapids-build-env rapids-notebook-env

ci/gpu/java.sh

@@ -80,7 +80,7 @@ gpuci_conda_retry install -y \
                   "rapids-notebook-env=$MINOR_VERSION.*" \
                   "dask-cuda=${MINOR_VERSION}" \
                   "rmm=$MINOR_VERSION.*" \
-                  "ucx-py=0.22.*" \
+                  "ucx-py=0.23.*" \
                   "openjdk=8.*" \
                   "maven"
 

ci/release/update-version.sh

@@ -31,6 +31,7 @@ function sed_runner() {
 
 # cpp update
 sed_runner 's/'"CUDF VERSION .* LANGUAGES"'/'"CUDF VERSION ${NEXT_FULL_TAG} LANGUAGES"'/g' cpp/CMakeLists.txt
+sed_runner 's/'"branch-.*\/RAPIDS.cmake"'/'"branch-${NEXT_SHORT_TAG}\/RAPIDS.cmake"'/g' cpp/CMakeLists.txt
 
 # cpp libcudf_kafka update
 sed_runner 's/'"CUDA_KAFKA VERSION .* LANGUAGES"'/'"CUDA_KAFKA VERSION ${NEXT_FULL_TAG} LANGUAGES"'/g' cpp/libcudf_kafka/CMakeLists.txt

conda/environments/cudf_dev_cuda11.0.yml

@@ -38,8 +38,9 @@ dependencies:
   - black=19.10
   - isort=5.6.4
   - mypy=0.782
+  - pydocstyle=6.1.1
   - typing_extensions
-  - pre_commit
+  - pre-commit
   - dask>=2021.09.1
   - distributed>=2021.09.1
   - streamz
@@ -58,6 +59,8 @@ dependencies:
   - cachetools
   - transformers<=4.10.3
   - pydata-sphinx-theme
+  - librdkafka=1.7.0
+  - python-confluent-kafka=1.7.0
   - pip:
       - git+https://github.com/dask/dask.git@main
       - git+https://github.com/dask/distributed.git@main

conda/environments/cudf_dev_cuda11.2.yml

@@ -38,8 +38,9 @@ dependencies:
   - black=19.10
   - isort=5.6.4
   - mypy=0.782
+  - pydocstyle=6.1.1
   - typing_extensions
-  - pre_commit
+  - pre-commit
   - dask>=2021.09.1
   - distributed>=2021.09.1
   - streamz
@@ -58,6 +59,8 @@ dependencies:
   - cachetools
   - transformers<=4.10.3
   - pydata-sphinx-theme
+  - librdkafka=1.7.0
+  - python-confluent-kafka=1.7.0
   - pip:
       - git+https://github.com/dask/dask.git@main
       - git+https://github.com/dask/distributed.git@main

conda/environments/cudf_dev_cuda11.5.yml

@@ -0,0 +1,68 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+
+name: cudf_dev
+channels:
+  - rapidsai
+  - nvidia
+  - rapidsai-nightly
+  - conda-forge
+dependencies:
+  - clang=11.0.0
+  - clang-tools=11.0.0
+  - cupy>7.1.0,<10.0.0a0
+  - rmm=21.12.*
+  - cmake>=3.20.1
+  - cmake_setuptools>=0.1.3
+  - python>=3.7,<3.9
+  - numba>=0.53.1
+  - numpy
+  - pandas>=1.0,<1.4.0dev0
+  - pyarrow=5.0.0=*cuda
+  - fastavro>=0.22.9
+  - python-snappy>=0.6.0
+  - notebook>=0.5.0
+  - cython>=0.29,<0.30
+  - fsspec>=0.6.0
+  - pytest
+  - pytest-benchmark
+  - pytest-xdist
+  - sphinx
+  - sphinxcontrib-websupport
+  - nbsphinx
+  - numpydoc
+  - ipython
+  - pandoc=<2.0.0
+  - cudatoolkit=11.5
+  - pip
+  - flake8=3.8.3
+  - black=19.10
+  - isort=5.6.4
+  - mypy=0.782
+  - pydocstyle=6.1.1
+  - typing_extensions
+  - pre-commit
+  - dask>=2021.09.1
+  - distributed>=2021.09.1
+  - streamz
+  - arrow-cpp=5.0.0
+  - dlpack>=0.5,<0.6.0a0
+  - arrow-cpp-proc * cuda
+  - double-conversion
+  - rapidjson
+  - hypothesis
+  - sphinx-markdown-tables
+  - sphinx-copybutton
+  - mimesis<4.1
+  - packaging
+  - protobuf
+  - nvtx>=0.2.1
+  - cachetools
+  - transformers<=4.10.3
+  - pydata-sphinx-theme
+  - librdkafka=1.7.0
+  - python-confluent-kafka=1.7.0
+  - pip:
+      - git+https://github.com/dask/dask.git@main
+      - git+https://github.com/dask/distributed.git@main
+      - git+https://github.com/python-streamz/streamz.git@master
+      - pyorc

conda/recipes/dask-cudf/meta.yaml

@@ -14,7 +14,7 @@ source:
 
 build:
   number: {{ GIT_DESCRIBE_NUMBER }}
-  string: py{{ py_version }}_{{ GIT_DESCRIBE_HASH }}_{{ GIT_DESCRIBE_NUMBER }}
+  string: cuda_{{ cuda_version }}_py{{ py_version }}_{{ GIT_DESCRIBE_HASH }}_{{ GIT_DESCRIBE_NUMBER }}
   script_env:
     - VERSION_SUFFIX
     - PARALLEL_LEVEL
@@ -28,11 +28,13 @@ requirements:
     - cudf {{ version }}
     - dask>=2021.09.1
     - distributed>=2021.09.1
+    - cudatoolkit {{ cuda_version }}
   run:
     - python
     - cudf {{ version }}
     - dask>=2021.09.1
     - distributed>=2021.09.1
+    - {{ pin_compatible('cudatoolkit', max_pin='x.x') }}
 
 test:                                   # [linux64]
   requires:                             # [linux64]

conda/recipes/libcudf/build.sh

@@ -2,7 +2,7 @@
 
 if [[ -z "$PROJECT_FLASH" || "$PROJECT_FLASH" == "0" ]]; then
     # This assumes the script is executed from the root of the repo directory
-    ./build.sh -v libcudf --allgpuarch
+    ./build.sh -v libcudf --allgpuarch --cmake-args=\"-DCMAKE_INSTALL_LIBDIR=lib\"
 else
-    ./build.sh -v libcudf tests --allgpuarch
-fi
+    ./build.sh -v libcudf tests --allgpuarch --cmake-args=\"-DCMAKE_INSTALL_LIBDIR=lib\"
+fi