[RELEASE] cudf v0.14

.gitignore

@@ -23,12 +23,10 @@ cudf.egg-info/
 python/build
 python/*/build
 python/cudf/cudf-coverage.xml
-python/cudf/*/_lib/**/*.cpp
+python/cudf/*/_lib/**/*\.cpp
 python/cudf/*/_lib/**/*.h
 python/cudf/*/_lib/.nfs*
-python/cudf/*/_libxx/**/*.cpp
-python/cudf/*/_libxx/**/*.h
-python/cudf/*/_libxx/.nfs*
+python/cudf/*/_cuda/*\.cpp
 python/cudf/*.ipynb
 python/cudf/.ipynb_checkpoints
 python/nvstrings/nvstrings-coverage.xml

.gitmodules

@@ -5,7 +5,7 @@
 [submodule "thirdparty/jitify"]
 	path = thirdparty/jitify
 	url = https://github.com/rapidsai/jitify.git
-	branch = cudf
+	branch = cudf_0.14
 [submodule "thirdparty/libcudacxx"]
 	path = thirdparty/libcudacxx
 	url = https://github.com/rapidsai/thirdparty-freestanding.git

.pre-commit-config.yaml

@@ -8,16 +8,30 @@ repos:
         hooks:
               - id: black
       - repo: https://gitlab.com/pycqa/flake8
-        rev: 3.7.7
+        rev: 3.8.1
         hooks:
               - id: flake8
+                alias: flake8
+                name: flake8
+                args: ["--config=python/.flake8"]
+                types: [python]
       - repo: https://gitlab.com/pycqa/flake8
-        rev: 3.7.7
+        rev: 3.8.1
         hooks:
               - id: flake8
                 alias: flake8-cython
                 name: flake8-cython
-                args: ["--config=python/cudf/.flake8.cython"]
+                args: ["--config=python/.flake8.cython"]
                 types: [cython]
+      - repo: local
+        hooks:
+              - id: clang-format
+                name: clang-format
+                description: Format files with ClangFormat.
+                entry: clang-format -i
+                language: system
+                files: \.(cu|cuh|h|hpp|cpp|inl)$
+                args: ['-fallback-style=none']
+
 default_language_version:
       python: python3

CODE_OF_CONDUCT.md

@@ -0,0 +1 @@
+This project has adopted the [Contributor Covenant Code of Conduct](https://docs.rapids.ai/resources/conduct/). 

CONTRIBUTING.md

@@ -89,12 +89,25 @@ flake8 --config=python/cudf/.flake8.cython
 Additionally, many editors have plugins that will apply `isort` and `Black` as
 you edit files, as well as use `flake8` to report any style / syntax issues.
 
+#### C++/CUDA
+
+cuDF uses [`clang-format`](https://clang.llvm.org/docs/ClangFormat.html)
+
+In order to format the C++/CUDA files, navigate to the root (`cudf`) directory and run:
+```
+python3 ./cpp/scripts/run-clang-format.py -inplace
+```
+
+Additionally, many editors have plugins or extensions that you can set up to automatically run `clang-format` either manually or on file save.
+
+#### Pre-commit hooks
+
 Optionally, you may wish to setup [pre-commit hooks](https://pre-commit.com/)
-to automatically run `isort`, `Black`, and `flake8` when you make a git commit.
+to automatically run `isort`, `Black`, `flake8` and `clang-format` when you make a git commit.
 This can be done by installing `pre-commit` via `conda` or `pip`:
 
 ```bash
-conda install pre_commit
+conda install -c conda-forge pre_commit
 ```
 
 ```bash
@@ -107,21 +120,21 @@ and then running:
 pre-commit install
 ```
 
-from the root of the cuDF repository. Now `isort`, `Black`, and `flake8` will be
+from the root of the cuDF repository. Now `isort`, `Black`, `flake8` and `clang-format` will be
 run each time you commit changes.
 
 ### Get libcudf Dependencies
 
 Compiler requirements:
 
 * `gcc`     version 5.4+
-* `nvcc`    version 9.2+
+* `nvcc`    version 10.0+
 * `cmake`   version 3.12.4+
 
 CUDA/GPU requirements:
 
-* CUDA 9.2+
-* NVIDIA driver 396.44+
+* CUDA 10.0+
+* NVIDIA driver 410.48+
 * Pascal architecture or better
 
 You can obtain CUDA from [https://developer.nvidia.com/cuda-downloads](https://developer.nvidia.com/cuda-downloads).
@@ -157,11 +170,12 @@ git submodule update --init --remote --recursive
 - Create the conda development environment `cudf_dev`:
 ```bash
 # create the conda environment (assuming in base `cudf` directory)
+# note: RAPIDS currently doesn't support `channel_priority: strict`; use `channel_priority: flexible` instead
 conda env create --name cudf_dev --file conda/environments/cudf_dev_cuda10.0.yml
 # activate the environment
 conda activate cudf_dev
 ```
-- If using CUDA 9.2, create the environment with `conda env create --name cudf_dev --file conda/environments/cudf_dev_cuda9.2.yml` instead.
+- If using CUDA 10.0, create the environment with `conda env create --name cudf_dev --file conda/environments/cudf_dev_cuda10.0.yml` instead.
 - For other CUDA versions, check the corresponding cudf_dev_cuda*.yml file in conda/environments
 
 - Build and install `libcudf` after its dependencies. CMake depends on the `nvcc` executable being on your path or defined in `$CUDACXX`.
@@ -290,8 +304,8 @@ A Dockerfile is provided with a preconfigured conda environment for building and
 ### Prerequisites
 
 * Install [nvidia-docker2](https://github.com/nvidia/nvidia-docker/wiki/Installation-(version-2.0)) for Docker + GPU support
-* Verify NVIDIA driver is `396.44` or higher
-* Ensure CUDA 9.2+ is installed
+* Verify NVIDIA driver is `410.48` or higher
+* Ensure CUDA 10.0+ is installed
 
 ### Usage
 
@@ -318,7 +332,7 @@ flag. Below is a list of the available arguments and their purpose:
 
 | Build Argument | Default Value | Other Value(s) | Purpose |
 | --- | --- | --- | --- |
-| `CUDA_VERSION` | 9.2 | 10.0 | set CUDA version |
+| `CUDA_VERSION` | 10.0 | 10.1, 10.2 | set CUDA version |
 | `LINUX_VERSION` | ubuntu16.04 | ubuntu18.04 | set Ubuntu version |
 | `CC` & `CXX` | 5 | 7 | set gcc/g++ version; **NOTE:** gcc7 requires Ubuntu 18.04 |
 | `CUDF_REPO` | This repo | Forks of cuDF | set git URL to use for `git clone` |

Dockerfile

@@ -1,5 +1,5 @@
 # An integration test & dev container which builds and installs cuDF from master
-ARG CUDA_VERSION=9.2
+ARG CUDA_VERSION=10.0
 ARG CUDA_SHORT_VERSION=${CUDA_VERSION}
 ARG LINUX_VERSION=ubuntu16.04
 FROM nvidia/cuda:${CUDA_VERSION}-devel-${LINUX_VERSION}

README.md

@@ -43,34 +43,46 @@ Please see the [Demo Docker Repository](https://hub.docker.com/r/rapidsai/rapids
 
 ## Installation
 
+
+### CUDA/GPU requirements
+
+* CUDA 10.0+
+* NVIDIA driver 410.48+
+* Pascal architecture or better (Compute Capability >=6.0)
+
 ### Conda
 
 cuDF can be installed with conda ([miniconda](https://conda.io/miniconda.html), or the full [Anaconda distribution](https://www.anaconda.com/download)) from the `rapidsai` channel:
 
-For `cudf version == 0.10` :
+For `cudf version == 0.13` :
 ```bash
-# for CUDA 9.2
+# for CUDA 10.0
 conda install -c rapidsai -c nvidia -c numba -c conda-forge \
-    cudf=0.10 python=3.6 cudatoolkit=9.2
+    cudf=0.13 python=3.6 cudatoolkit=10.0
 
-# or, for CUDA 10.0
+# or, for CUDA 10.1
 conda install -c rapidsai -c nvidia -c numba -c conda-forge \
-    cudf=0.10 python=3.6 cudatoolkit=10.0
+    cudf=0.13 python=3.6 cudatoolkit=10.1
 
-# or, for CUDA 10.1
+# or, for CUDA 10.2
 conda install -c rapidsai -c nvidia -c numba -c conda-forge \
-    cudf=0.10 python=3.6 cudatoolkit=10.1
+    cudf=0.13 python=3.6 cudatoolkit=10.2
+
 ```
 
 For the nightly version of `cudf` :
 ```bash
-# for CUDA 9.2
+# for CUDA 10.0
 conda install -c rapidsai-nightly -c nvidia -c numba -c conda-forge \
-    cudf python=3.6 cudatoolkit=9.2
+    cudf python=3.6 cudatoolkit=10.0
 
-# or, for CUDA 10.0
+# or, for CUDA 10.1
 conda install -c rapidsai-nightly -c nvidia -c numba -c conda-forge \
-    cudf python=3.6 cudatoolkit=10.0
+    cudf python=3.6 cudatoolkit=10.1
+
+# or, for CUDA 10.2
+conda install -c rapidsai-nightly -c nvidia -c numba -c conda-forge \
+    cudf python=3.6 cudatoolkit=10.2
 ```
 
 Note: cuDF is supported only on Linux, and with Python versions 3.6 or 3.7.

build.sh

@@ -51,7 +51,7 @@ BUILD_DIRS="${LIB_BUILD_DIR} ${NVSTRINGS_BUILD_DIR} ${CUDF_BUILD_DIR} ${DASK_CUD
 VERBOSE=""
 BUILD_TYPE=Release
 INSTALL_TARGET=install
-BENCHMARKS=OFF
+BUILD_BENCHMARKS=OFF
 BUILD_ALL_GPU_ARCH=0
 BUILD_NVTX=ON
 BUILD_TESTS=OFF
@@ -106,7 +106,7 @@ if hasArg --allgpuarch; then
     BUILD_ALL_GPU_ARCH=1
 fi
 if hasArg benchmarks; then
-    BENCHMARKS="ON"
+    BUILD_BENCHMARKS=ON
 fi
 if hasArg tests; then
     BUILD_TESTS=ON
@@ -151,7 +151,7 @@ if buildAll || hasArg libnvstrings || hasArg libcudf; then
           -DCMAKE_CXX11_ABI=ON \
           ${GPU_ARCH} \
           -DUSE_NVTX=${BUILD_NVTX} \
-          -DBUILD_BENCHMARKS=${BENCHMARKS} \
+          -DBUILD_BENCHMARKS=${BUILD_BENCHMARKS} \
           -DBUILD_LEGACY_TESTS=${BUILD_LEGACY_TESTS} \
           -DDISABLE_DEPRECATION_WARNING=${BUILD_DISABLE_DEPRECATION_WARNING} \
           -DCMAKE_BUILD_TYPE=${BUILD_TYPE} ..
@@ -196,6 +196,10 @@ if buildAll || hasArg libcudf; then
     if [[ ${BUILD_TESTS} == "ON" ]]; then
         make -j${PARALLEL_LEVEL} build_tests_cudf VERBOSE=${VERBOSE}
     fi
+
+    if [[ ${BUILD_BENCHMARKS} == "ON" ]]; then
+        make -j${PARALLEL_LEVEL} build_benchmarks_cudf VERBOSE=${VERBOSE}
+    fi
 fi
 
 # Build and install the cudf Python package

ci/checks/headers_test.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+# Copyright (c) 2020, NVIDIA CORPORATION.
+#####################################
+# conda existence test for headers  #
+#####################################
+
+RETVAL=0
+for LIBNAME in cudf nvstrings; do 
+  # existence tests for lib${LIBNAME}
+  HEADERS=`cd cpp && find include/${LIBNAME}/ -type f \( -iname "*.h" -o  -iname "*.hpp" \) -printf "    - test -f \\\$PREFIX/%p\n" | sort`
+  META_TESTS=`grep "test -f .*\.h\(pp\)\?" conda/recipes/lib${LIBNAME}/meta.yaml | sort`
+  HEADER_DIFF=`diff <(echo "$HEADERS") <(echo "$META_TESTS")`
+  LIB_RETVAL=$?
+
+  if [ "$LIB_RETVAL" != "0" ]; then
+    echo -e "\n\n>>>> FAILED: lib${LIBNAME} header existence conda/recipes/lib${LIBNAME}/meta.yaml check; begin output\n\n"
+    echo -e "$HEADER_DIFF"
+    echo -e "\n\n>>>> FAILED: lib${LIBNAME} header existence conda/recipes/lib${LIBNAME}/meta.yaml check; end output\n\n"
+    RETVAL=1
+  else
+    echo -e "\n\n>>>> PASSED: lib${LIBNAME} header existence conda/recipes/lib${LIBNAME}/meta.yaml check\n\n"
+  fi
+done
+
+exit $RETVAL

ci/checks/style.sh

@@ -22,13 +22,17 @@ BLACK=`black --check python`
 BLACK_RETVAL=$?
 
 # Run flake8 and get results/return code
-FLAKE=`flake8 python`
+FLAKE=`flake8 --config=python/.flake8 python`
 FLAKE_RETVAL=$?
 
 # Run flake8-cython and get results/return code
-FLAKE_CYTHON=`flake8 --config=python/cudf/.flake8.cython`
+FLAKE_CYTHON=`flake8 --config=python/.flake8.cython`
 FLAKE_CYTHON_RETVAL=$?
 
+# Run clang-format and check for a consistent code format
+CLANG_FORMAT=`python cpp/scripts/run-clang-format.py 2>&1`
+CLANG_FORMAT_RETVAL=$?
+
 # Output results if failure otherwise show pass
 if [ "$ISORT_RETVAL" != "0" ]; then
   echo -e "\n\n>>>> FAILED: isort style check; begin output\n\n"
@@ -62,7 +66,20 @@ else
   echo -e "\n\n>>>> PASSED: flake8-cython style check\n\n"
 fi
 
-RETVALS=($ISORT_RETVAL $BLACK_RETVAL $FLAKE_RETVAL $FLAKE_CYTHON_RETVAL)
+if [ "$CLANG_FORMAT_RETVAL" != "0" ]; then
+  echo -e "\n\n>>>> FAILED: clang format check; begin output\n\n"
+  echo -e "$CLANG_FORMAT"
+  echo -e "\n\n>>>> FAILED: clang format check; end output\n\n"
+else
+  echo -e "\n\n>>>> PASSED: clang format check\n\n"
+fi
+
+# Run header meta.yml check and get results/return code
+HEADER_META=`ci/checks/headers_test.sh`
+HEADER_META_RETVAL=$?
+echo -e "$HEADER_META"
+
+RETVALS=($ISORT_RETVAL $BLACK_RETVAL $FLAKE_RETVAL $FLAKE_CYTHON_RETVAL $CLANG_FORMAT_RETVAL $HEADER_META_RETVAL)
 IFS=$'\n'
 RETVAL=`echo "${RETVALS[*]}" | sort -nr | head -n1`
 

ci/cpu/upload_anaconda.sh

@@ -32,12 +32,12 @@ if [ "$UPLOAD_LIBCUDF" == "1" ]; then
   test -e ${LIBNVSTRINGS_FILE}
   echo "Upload libNVStrings"
   echo ${LIBNVSTRINGS_FILE}
-  anaconda -t ${MY_UPLOAD_KEY} upload -u ${CONDA_USERNAME:-rapidsai} ${LABEL_OPTION} --force ${LIBNVSTRINGS_FILE}
+  anaconda -t ${MY_UPLOAD_KEY} upload -u ${CONDA_USERNAME:-rapidsai} ${LABEL_OPTION} --skip-existing ${LIBNVSTRINGS_FILE}
 
   test -e ${LIBCUDF_FILE}
   echo "Upload libcudf"
   echo ${LIBCUDF_FILE}
-  anaconda -t ${MY_UPLOAD_KEY} upload -u ${CONDA_USERNAME:-rapidsai} ${LABEL_OPTION} --force ${LIBCUDF_FILE}
+  anaconda -t ${MY_UPLOAD_KEY} upload -u ${CONDA_USERNAME:-rapidsai} ${LABEL_OPTION} --skip-existing ${LIBCUDF_FILE}
 fi
 
 if [ "$UPLOAD_CUDF" == "1" ]; then
@@ -47,20 +47,20 @@ if [ "$UPLOAD_CUDF" == "1" ]; then
   test -e ${NVSTRINGS_FILE}
   echo "Upload nvstrings"
   echo ${NVSTRINGS_FILE}
-  anaconda -t ${MY_UPLOAD_KEY} upload -u ${CONDA_USERNAME:-rapidsai} ${LABEL_OPTION} --force ${NVSTRINGS_FILE}
+  anaconda -t ${MY_UPLOAD_KEY} upload -u ${CONDA_USERNAME:-rapidsai} ${LABEL_OPTION} --skip-existing ${NVSTRINGS_FILE}
 
   test -e ${CUDF_FILE}
   echo "Upload cudf"
   echo ${CUDF_FILE}
-  anaconda -t ${MY_UPLOAD_KEY} upload -u ${CONDA_USERNAME:-rapidsai} ${LABEL_OPTION} --force ${CUDF_FILE}
+  anaconda -t ${MY_UPLOAD_KEY} upload -u ${CONDA_USERNAME:-rapidsai} ${LABEL_OPTION} --skip-existing ${CUDF_FILE}
 
   test -e ${DASK_CUDF_FILE}
   echo "Upload dask-cudf"
   echo ${DASK_CUDF_FILE}
-  anaconda -t ${MY_UPLOAD_KEY} upload -u ${CONDA_USERNAME:-rapidsai} ${LABEL_OPTION} --force ${DASK_CUDF_FILE}
+  anaconda -t ${MY_UPLOAD_KEY} upload -u ${CONDA_USERNAME:-rapidsai} ${LABEL_OPTION} --skip-existing ${DASK_CUDF_FILE}
 
   test -e ${CUSTREAMZ_FILE}
   echo "Upload custreamz"
   echo ${CUSTREAMZ_FILE}
-  anaconda -t ${MY_UPLOAD_KEY} upload -u ${CONDA_USERNAME:-rapidsai} ${LABEL_OPTION} --force ${CUSTREAMZ_FILE}
+  anaconda -t ${MY_UPLOAD_KEY} upload -u ${CONDA_USERNAME:-rapidsai} ${LABEL_OPTION} --skip-existing ${CUSTREAMZ_FILE}
 fi