[RELEASE] cudf v21.12

.pre-commit-config.yaml

@@ -1,5 +1,5 @@
 repos:
-      - repo: https://github.com/pycqa/isort
+      - repo: https://github.com/PyCQA/isort
         rev: 5.6.4
         hooks:
               - id: isort
@@ -27,12 +27,12 @@ repos:
                 name: isort-dask-cudf
                 args: ["--settings-path=python/dask_cudf/setup.cfg"]
                 files: python/dask_cudf/.*
-      - repo: https://github.com/ambv/black
+      - repo: https://github.com/psf/black
         rev: 19.10b0
         hooks:
               - id: black
                 files: python/.*
-      - repo: https://gitlab.com/pycqa/flake8
+      - repo: https://github.com/PyCQA/flake8
         rev: 3.8.3
         hooks:
               - id: flake8
@@ -45,30 +45,49 @@ repos:
                 name: flake8-cython
                 args: ["--config=python/.flake8.cython"]
                 types: [cython]
+      - repo: https://github.com/pre-commit/mirrors-mypy
+        rev: 'v0.782'
+        hooks:
+              - id: mypy
+                args: ["--config-file=python/cudf/setup.cfg", "python/cudf/cudf"]
+                pass_filenames: false
+      - repo: https://github.com/PyCQA/pydocstyle
+        rev: 6.1.1
+        hooks:
+              - id: pydocstyle
+                args: ["--config=python/.flake8"]
       - repo: local
         hooks:
               - id: clang-format
+                # Using the pre-commit stage to simplify invocation of all
+                # other hooks simultaneously (via any other hook stage).  This
+                # can be removed if we also move to running clang-format
+                # entirely through pre-commit.
+                stages: [commit]
                 name: clang-format
                 description: Format files with ClangFormat.
                 entry: clang-format -i
                 language: system
                 files: \.(cu|cuh|h|hpp|cpp|inl)$
                 args: ['-fallback-style=none']
-      - repo: local
-        hooks:
-              - id: mypy
-                name: mypy
-                description: mypy
-                pass_filenames: false
-                entry: mypy --config-file=python/cudf/setup.cfg python/cudf/cudf
-                language: system
-                types: [python]
-      -   repo: https://github.com/pycqa/pydocstyle
-          rev: 6.0.0
-          hooks:
-              - id: pydocstyle
-                args: ["--config=python/.flake8"]
-
+              - id: cmake-format
+                name: cmake-format
+                entry: bash cpp/scripts/run-cmake-format.sh cmake-format
+                language: python
+                types: [cmake]
+                # Note that pre-commit autoupdate does not update the versions
+                # of dependencies, so we'll have to update this manually.
+                additional_dependencies:
+                  - cmake-format==0.6.11
+              - id: cmake-lint
+                name: cmake-lint
+                entry: bash cpp/scripts/run-cmake-format.sh cmake-lint
+                language: python
+                types: [cmake]
+                # Note that pre-commit autoupdate does not update the versions
+                # of dependencies, so we'll have to update this manually.
+                additional_dependencies:
+                  - cmake-format==0.6.11
 
 default_language_version:
       python: python3

CHANGELOG.md

@@ -1,3 +1,7 @@
+# cuDF 21.12.00 (Date TBD)
+
+Please see https://github.com/rapidsai/cudf/releases/tag/v21.12.00a for the latest changes to this development branch.
+
 # cuDF 21.10.00 (Date TBD)
 
 Please see https://github.com/rapidsai/cudf/releases/tag/v21.10.00a for the latest changes to this development branch.

CONTRIBUTING.md

@@ -62,12 +62,12 @@ The following instructions are for developers and contributors to cuDF OSS devel
 Compilers:
 
 * `gcc`     version 9.3+
-* `nvcc`    version 11.0+
+* `nvcc`    version 11.5+
 * `cmake`   version 3.20.1+
 
 CUDA/GPU:
 
-* CUDA 11.0+
+* CUDA 11.5+
 * NVIDIA driver 450.80.02+
 * Pascal architecture or better
 

build.sh

@@ -18,7 +18,7 @@ ARGS=$*
 REPODIR=$(cd $(dirname $0); pwd)
 
 VALIDARGS="clean libcudf cudf dask_cudf benchmarks tests libcudf_kafka cudf_kafka custreamz -v -g -n -l --allgpuarch --disable_nvtx --show_depr_warn --ptds -h"
-HELP="$0 [clean] [libcudf] [cudf] [dask_cudf] [benchmarks] [tests] [libcudf_kafka] [cudf_kafka] [custreamz] [-v] [-g] [-n] [-h] [-l] [--cmake-args=\"<args>\"]
+HELP="$0 [clean] [libcudf] [cudf] [dask_cudf] [benchmarks] [tests] [libcudf_kafka] [cudf_kafka] [custreamz] [-v] [-g] [-n] [-h] [-l] [--cmake-args=\\\"<args>\\\"]
    clean                         - remove all existing build artifacts and configuration (start
                                    over)
    libcudf                       - build the cudf C++ code only

ci/benchmark/build.sh

@@ -37,7 +37,7 @@ export GBENCH_BENCHMARKS_DIR="$WORKSPACE/cpp/build/gbenchmarks/"
 export LIBCUDF_KERNEL_CACHE_PATH="$HOME/.jitify-cache"
 
 # Dask & Distributed git tag
-export DASK_DISTRIBUTED_GIT_TAG='2021.09.1'
+export DASK_DISTRIBUTED_GIT_TAG='main'
 
 function remove_libcudf_kernel_cache_dir {
     EXITCODE=$?

ci/checks/style.sh

@@ -14,119 +14,18 @@ LANG=C.UTF-8
 . /opt/conda/etc/profile.d/conda.sh
 conda activate rapids
 
-# Run isort-cudf and get results/return code
-ISORT_CUDF=`isort python/cudf --check-only --settings-path=python/cudf/setup.cfg 2>&1`
-ISORT_CUDF_RETVAL=$?
+FORMAT_FILE_URL=https://raw.githubusercontent.com/rapidsai/rapids-cmake/main/cmake-format-rapids-cmake.json
+export RAPIDS_CMAKE_FORMAT_FILE=/tmp/rapids_cmake_ci/cmake-formats-rapids-cmake.json
+mkdir -p $(dirname ${RAPIDS_CMAKE_FORMAT_FILE})
+wget -O ${RAPIDS_CMAKE_FORMAT_FILE} ${FORMAT_FILE_URL}
 
-# Run isort-cudf-kafka and get results/return code
-ISORT_CUDF_KAFKA=`isort python/cudf_kafka --check-only --settings-path=python/cudf_kafka/setup.cfg 2>&1`
-ISORT_CUDF_KAFKA_RETVAL=$?
-
-# Run isort-custreamz and get results/return code
-ISORT_CUSTREAMZ=`isort python/custreamz --check-only --settings-path=python/custreamz/setup.cfg 2>&1`
-ISORT_CUSTREAMZ_RETVAL=$?
-
-# Run isort-dask-cudf and get results/return code
-ISORT_DASK_CUDF=`isort python/dask_cudf --check-only --settings-path=python/dask_cudf/setup.cfg 2>&1`
-ISORT_DASK_CUDF_RETVAL=$?
-
-# Run black and get results/return code
-BLACK=`black --check python 2>&1`
-BLACK_RETVAL=$?
-
-# Run flake8 and get results/return code
-FLAKE=`flake8 --config=python/.flake8 python 2>&1`
-FLAKE_RETVAL=$?
-
-# Run flake8-cython and get results/return code
-FLAKE_CYTHON=`flake8 --config=python/.flake8.cython 2>&1`
-FLAKE_CYTHON_RETVAL=$?
-
-# Run mypy and get results/return code
-MYPY_CUDF=`mypy --config=python/cudf/setup.cfg python/cudf/cudf 2>&1`
-MYPY_CUDF_RETVAL=$?
-
-# Run pydocstyle and get results/return code
-PYDOCSTYLE=`pydocstyle --config=python/.flake8 python 2>&1`
-PYDOCSTYLE_RETVAL=$?
+pre-commit run --hook-stage manual --all-files
+PRE_COMMIT_RETVAL=$?
 
 # Run clang-format and check for a consistent code format
 CLANG_FORMAT=`python cpp/scripts/run-clang-format.py 2>&1`
 CLANG_FORMAT_RETVAL=$?
 
-# Output results if failure otherwise show pass
-if [ "$ISORT_CUDF_RETVAL" != "0" ]; then
-  echo -e "\n\n>>>> FAILED: isort-cudf style check; begin output\n\n"
-  echo -e "$ISORT_CUDF"
-  echo -e "\n\n>>>> FAILED: isort-cudf style check; end output\n\n"
-else
-  echo -e "\n\n>>>> PASSED: isort-cudf style check\n\n"
-fi
-
-if [ "$ISORT_CUDF_KAFKA_RETVAL" != "0" ]; then
-  echo -e "\n\n>>>> FAILED: isort-cudf-kafka style check; begin output\n\n"
-  echo -e "$ISORT_CUDF_KAFKA"
-  echo -e "\n\n>>>> FAILED: isort-cudf-kafka style check; end output\n\n"
-else
-  echo -e "\n\n>>>> PASSED: isort-cudf-kafka style check\n\n"
-fi
-
-if [ "$ISORT_CUSTREAMZ_RETVAL" != "0" ]; then
-  echo -e "\n\n>>>> FAILED: isort-custreamz style check; begin output\n\n"
-  echo -e "$ISORT_CUSTREAMZ"
-  echo -e "\n\n>>>> FAILED: isort-custreamz style check; end output\n\n"
-else
-  echo -e "\n\n>>>> PASSED: isort-custreamz style check\n\n"
-fi
-
-if [ "$ISORT_DASK_CUDF_RETVAL" != "0" ]; then
-  echo -e "\n\n>>>> FAILED: isort-dask-cudf style check; begin output\n\n"
-  echo -e "$ISORT_DASK_CUDF"
-  echo -e "\n\n>>>> FAILED: isort-dask-cudf style check; end output\n\n"
-else
-  echo -e "\n\n>>>> PASSED: isort-dask-cudf style check\n\n"
-fi
-
-if [ "$BLACK_RETVAL" != "0" ]; then
-  echo -e "\n\n>>>> FAILED: black style check; begin output\n\n"
-  echo -e "$BLACK"
-  echo -e "\n\n>>>> FAILED: black style check; end output\n\n"
-else
-  echo -e "\n\n>>>> PASSED: black style check\n\n"
-fi
-
-if [ "$FLAKE_RETVAL" != "0" ]; then
-  echo -e "\n\n>>>> FAILED: flake8 style check; begin output\n\n"
-  echo -e "$FLAKE"
-  echo -e "\n\n>>>> FAILED: flake8 style check; end output\n\n"
-else
-  echo -e "\n\n>>>> PASSED: flake8 style check\n\n"
-fi
-
-if [ "$FLAKE_CYTHON_RETVAL" != "0" ]; then
-  echo -e "\n\n>>>> FAILED: flake8-cython style check; begin output\n\n"
-  echo -e "$FLAKE_CYTHON"
-  echo -e "\n\n>>>> FAILED: flake8-cython style check; end output\n\n"
-else
-  echo -e "\n\n>>>> PASSED: flake8-cython style check\n\n"
-fi
-
-if [ "$MYPY_CUDF_RETVAL" != "0" ]; then
-  echo -e "\n\n>>>> FAILED: mypy style check; begin output\n\n"
-  echo -e "$MYPY_CUDF"
-  echo -e "\n\n>>>> FAILED: mypy style check; end output\n\n"
-else
-  echo -e "\n\n>>>> PASSED: mypy style check\n\n"
-fi
-
-if [ "$PYDOCSTYLE_RETVAL" != "0" ]; then
-  echo -e "\n\n>>>> FAILED: pydocstyle style check; begin output\n\n"
-  echo -e "$PYDOCSTYLE"
-  echo -e "\n\n>>>> FAILED: pydocstyle style check; end output\n\n"
-else
-  echo -e "\n\n>>>> PASSED: pydocstyle style check\n\n"
-fi
-
 if [ "$CLANG_FORMAT_RETVAL" != "0" ]; then
   echo -e "\n\n>>>> FAILED: clang format check; begin output\n\n"
   echo -e "$CLANG_FORMAT"
@@ -141,9 +40,7 @@ HEADER_META_RETVAL=$?
 echo -e "$HEADER_META"
 
 RETVALS=(
-  $ISORT_CUDF_RETVAL $ISORT_CUDF_KAFKA_RETVAL $ISORT_CUSTREAMZ_RETVAL $ISORT_DASK_CUDF_RETVAL
-  $BLACK_RETVAL $FLAKE_RETVAL $FLAKE_CYTHON_RETVAL $PYDOCSTYLE_RETVAL $CLANG_FORMAT_RETVAL
-  $HEADER_META_RETVAL $MYPY_CUDF_RETVAL
+  $PRE_COMMIT_RETVAL $CLANG_FORMAT_RETVAL
 )
 IFS=$'\n'
 RETVAL=`echo "${RETVALS[*]}" | sort -nr | head -n1`

ci/gpu/build.sh

@@ -31,7 +31,7 @@ export GIT_DESCRIBE_TAG=`git describe --tags`
 export MINOR_VERSION=`echo $GIT_DESCRIBE_TAG | grep -o -E '([0-9]+\.[0-9]+)'`
 
 # Dask & Distributed git tag
-export DASK_DISTRIBUTED_GIT_TAG='2021.09.1'
+export DASK_DISTRIBUTED_GIT_TAG='main'
 
 ################################################################################
 # TRAP - Setup trap for removing jitify cache
@@ -83,7 +83,7 @@ gpuci_mamba_retry install -y \
                   "rapids-notebook-env=$MINOR_VERSION.*" \
                   "dask-cuda=${MINOR_VERSION}" \
                   "rmm=$MINOR_VERSION.*" \
-                  "ucx-py=0.22.*"
+                  "ucx-py=0.23.*"
 
 # https://docs.rapids.ai/maintainers/depmgmt/
 # gpuci_mamba_retry remove --force rapids-build-env rapids-notebook-env

ci/gpu/java.sh

@@ -80,7 +80,7 @@ gpuci_conda_retry install -y \
                   "rapids-notebook-env=$MINOR_VERSION.*" \
                   "dask-cuda=${MINOR_VERSION}" \
                   "rmm=$MINOR_VERSION.*" \
-                  "ucx-py=0.22.*" \
+                  "ucx-py=0.23.*" \
                   "openjdk=8.*" \
                   "maven"
 

ci/release/update-version.sh

@@ -38,6 +38,9 @@ sed_runner 's/'"CUDA_KAFKA VERSION .* LANGUAGES"'/'"CUDA_KAFKA VERSION ${NEXT_FU
 # cpp cudf_jni update
 sed_runner 's/'"CUDF_JNI VERSION .* LANGUAGES"'/'"CUDF_JNI VERSION ${NEXT_FULL_TAG} LANGUAGES"'/g' java/src/main/native/CMakeLists.txt
 
+# rapids-cmake version
+sed_runner 's/'"branch-.*\/RAPIDS.cmake"'/'"branch-${NEXT_SHORT_TAG}\/RAPIDS.cmake"'/g' fetch_rapids.cmake
+
 # doxyfile update
 sed_runner 's/PROJECT_NUMBER         = .*/PROJECT_NUMBER         = '${NEXT_FULL_TAG}'/g' cpp/doxygen/Doxyfile
 

conda/environments/cudf_dev_cuda11.0.yml

@@ -7,10 +7,10 @@ channels:
   - rapidsai-nightly
   - conda-forge
 dependencies:
-  - clang=11.0.0
-  - clang-tools=11.0.0
-  - cupy>7.1.0,<10.0.0a0
-  - rmm=21.10.*
+  - clang=11.1.0
+  - clang-tools=11.1.0
+  - cupy>=9.5.0,<10.0.0a0
+  - rmm=21.12.*
   - cmake>=3.20.1
   - cmake_setuptools>=0.1.3
   - python>=3.7,<3.9
@@ -19,6 +19,7 @@ dependencies:
   - pandas>=1.0,<1.4.0dev0
   - pyarrow=5.0.0=*cuda
   - fastavro>=0.22.9
+  - python-snappy>=0.6.0
   - notebook>=0.5.0
   - cython>=0.29,<0.30
   - fsspec>=0.6.0
@@ -37,10 +38,11 @@ dependencies:
   - black=19.10
   - isort=5.6.4
   - mypy=0.782
+  - pydocstyle=6.1.1
   - typing_extensions
-  - pre_commit
-  - dask=2021.09.1
-  - distributed=2021.09.1
+  - pre-commit
+  - dask>=2021.09.1
+  - distributed>=2021.09.1
   - streamz
   - arrow-cpp=5.0.0
   - dlpack>=0.5,<0.6.0a0
@@ -57,8 +59,10 @@ dependencies:
   - cachetools
   - transformers<=4.10.3
   - pydata-sphinx-theme
+  - librdkafka=1.7.0
+  - python-confluent-kafka=1.7.0
   - pip:
-      - git+https://github.com/dask/dask.git@2021.09.1
-      - git+https://github.com/dask/distributed.git@2021.09.1
+      - git+https://github.com/dask/dask.git@main
+      - git+https://github.com/dask/distributed.git@main
       - git+https://github.com/python-streamz/streamz.git@master
       - pyorc