Merge branch 'branch-0.18' into cs_nested_empty_strings

CHANGELOG.md

@@ -17,10 +17,13 @@
 - PR #6275 Update to official libcu++ on Github
 - PR #6838 Fix `columns` & `index` handling in dataframe constructor
 - PR #6750 Remove **kwargs from string/categorical methods
+- PR #6585 Add dictionary support to libcudf groupby functions
 - PR #6909 Support reading byte array backed decimal columns from parquet files
 - PR #6939 Use simplified `rmm::exec_policy`
 - PR #6512 Refactor rolling.cu to reduce compile time
 - PR #6982 Disable some pragma unroll statements in thrust `sort.h`
+- PR #7051 Verify decimal cast in java package
+- PR #7120 Verify window operations on decimal in java package
 
 ## Bug Fixes
 

ci/cpu/upload.sh

@@ -40,14 +40,14 @@ export CUSTREAMZ_FILE=`conda build conda/recipes/custreamz --python=$PYTHON --ou
 ################################################################################
 
 gpuci_logger "Starting conda uploads"
-if [ "$UPLOAD_LIBCUDF" == "1" ]; then
+if [[ "$BUILD_LIBCUDF" == "1" && "$UPLOAD_LIBCUDF" == "1" ]]; then
   test -e ${LIBCUDF_FILE}
   echo "Upload libcudf"
   echo ${LIBCUDF_FILE}
   gpuci_retry anaconda -t ${MY_UPLOAD_KEY} upload -u ${CONDA_USERNAME:-rapidsai} ${LABEL_OPTION} --skip-existing ${LIBCUDF_FILE}
 fi
 
-if [ "$UPLOAD_CUDF" == "1" ]; then
+if [[ "$BUILD_CUDF" == "1" && "$UPLOAD_CUDF" == "1" ]]; then
   test -e ${CUDF_FILE}
   echo "Upload cudf"
   echo ${CUDF_FILE}
@@ -64,14 +64,14 @@ if [ "$UPLOAD_CUDF" == "1" ]; then
   gpuci_retry anaconda -t ${MY_UPLOAD_KEY} upload -u ${CONDA_USERNAME:-rapidsai} ${LABEL_OPTION} --skip-existing ${CUSTREAMZ_FILE}
 fi
 
-if [ "$UPLOAD_LIBCUDF_KAFKA" == "1" ]; then
+if [[ "$BUILD_LIBCUDF" == "1" && "$UPLOAD_LIBCUDF_KAFKA" == "1" ]]; then
   test -e ${LIBCUDF_KAFKA_FILE}
   echo "Upload libcudf_kafka"
   echo ${LIBCUDF_KAFKA_FILE}
   gpuci_retry anaconda -t ${MY_UPLOAD_KEY} upload -u ${CONDA_USERNAME:-rapidsai} ${LABEL_OPTION} --skip-existing ${LIBCUDF_KAFKA_FILE}
 fi
 
-if [ "$UPLOAD_CUDF_KAFKA" == "1" ]; then
+if [[ "$BUILD_CUDF" == "1" && "$UPLOAD_CUDF_KAFKA" == "1" ]]; then
   test -e ${CUDF_KAFKA_FILE}
   echo "Upload cudf_kafka"
   echo ${CUDF_KAFKA_FILE}

conda/environments/cudf_dev_cuda10.1.yml

@@ -1,3 +1,5 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+
 name: cudf_dev
 channels:
   - rapidsai
@@ -60,3 +62,4 @@ dependencies:
       - git+https://github.com/dask/dask.git@master
       - git+https://github.com/dask/distributed.git@master
       - git+https://github.com/python-streamz/streamz.git
+      - pyorc

conda/environments/cudf_dev_cuda10.2.yml

@@ -1,3 +1,5 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+
 name: cudf_dev
 channels:
   - rapidsai
@@ -60,3 +62,4 @@ dependencies:
       - git+https://github.com/dask/dask.git@master
       - git+https://github.com/dask/distributed.git@master
       - git+https://github.com/python-streamz/streamz.git
+      - pyorc

conda/environments/cudf_dev_cuda11.0.yml

@@ -1,3 +1,5 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+
 name: cudf_dev
 channels:
   - rapidsai
@@ -60,3 +62,4 @@ dependencies:
       - git+https://github.com/dask/dask.git@master
       - git+https://github.com/dask/distributed.git@master
       - git+https://github.com/python-streamz/streamz.git
+      - pyorc

conda/recipes/libcudf/meta.yaml

@@ -123,6 +123,7 @@ test:
     - test -f $PREFIX/include/cudf/lists/detail/concatenate.hpp
     - test -f $PREFIX/include/cudf/lists/detail/copying.hpp
     - test -f $PREFIX/include/cudf/lists/extract.hpp
+    - test -f $PREFIX/include/cudf/lists/gather.hpp
     - test -f $PREFIX/include/cudf/lists/lists_column_view.hpp
     - test -f $PREFIX/include/cudf/merge.hpp
     - test -f $PREFIX/include/cudf/null_mask.hpp

cpp/include/cudf/aggregation.hpp

@@ -73,7 +73,7 @@ class aggregation {
     ARGMIN,          ///< Index of min element
     NUNIQUE,         ///< count number of unique elements
     NTH_ELEMENT,     ///< get the nth element
-    ROW_NUMBER,      ///< get row-number of element
+    ROW_NUMBER,      ///< get row-number of current index (relative to rolling window)
     COLLECT,         ///< collect values into a list
     LEAD,            ///< window function, accesses row at specified offset following current row
     LAG,             ///< window function, accesses row at specified offset preceding current row

cpp/include/cudf/detail/aggregation/aggregation.cuh

@@ -21,6 +21,7 @@
 #include <cudf/detail/aggregation/aggregation.hpp>
 #include <cudf/detail/utilities/device_atomics.cuh>
 #include <cudf/detail/utilities/release_assert.cuh>
+#include <cudf/dictionary/dictionary_column_view.hpp>
 #include <cudf/table/table_device_view.cuh>
 
 #include <rmm/cuda_stream_view.hpp>
@@ -172,6 +173,64 @@ struct update_target_element<
   }
 };
 
+/**
+ * @brief Function object to update a single element in a target column using
+ * the dictionary key addressed by the specific index.
+ *
+ * `target[target_index] = d_dictionary.keys[d_dictionary.indices[source_index]]`
+ */
+struct update_target_from_dictionary {
+  template <typename KeyType,
+            std::enable_if_t<is_fixed_width<KeyType>() && !is_fixed_point<KeyType>()>* = nullptr>
+  __device__ void operator()(mutable_column_device_view& target,
+                             size_type target_index,
+                             column_device_view& d_dictionary,
+                             size_type source_index) const noexcept
+  {
+// This code will segfault in nvcc/ptxas 10.2 only
+// https://nvbugswb.nvidia.com/NvBugs5/SWBug.aspx?bugid=3186317
+#if (__CUDACC_VER_MAJOR__ != 10) or (__CUDACC_VER_MINOR__ != 2)
+    auto const keys  = d_dictionary.child(cudf::dictionary_column_view::keys_column_index);
+    auto const value = keys.element<KeyType>(
+      static_cast<cudf::size_type>(d_dictionary.element<dictionary32>(source_index)));
+    using Target = target_type_t<KeyType, aggregation::SUM>;
+    atomicAdd(&target.element<Target>(target_index), static_cast<Target>(value));
+#endif
+  }
+  template <typename KeyType,
+            std::enable_if_t<!is_fixed_width<KeyType>() || is_fixed_point<KeyType>()>* = nullptr>
+  __device__ void operator()(mutable_column_device_view& target,
+                             size_type target_index,
+                             column_device_view& d_dictionary,
+                             size_type source_index) const noexcept {};
+};
+
+/**
+ * @brief Specialization function for dictionary type and aggregation SUM.
+ *
+ * @tparam target_has_nulls Indicates presence of null elements in `target`
+ * @tparam source_has_nulls Indicates presence of null elements in `source`.
+ */
+template <bool target_has_nulls, bool source_has_nulls>
+struct update_target_element<dictionary32, aggregation::SUM, target_has_nulls, source_has_nulls> {
+  __device__ void operator()(mutable_column_device_view target,
+                             size_type target_index,
+                             column_device_view source,
+                             size_type source_index) const noexcept
+  {
+    if (source_has_nulls and source.is_null(source_index)) { return; }
+
+    type_dispatcher(source.child(cudf::dictionary_column_view::keys_column_index).type(),
+                    update_target_from_dictionary{},
+                    target,
+                    target_index,
+                    source,
+                    source_index);
+
+    if (target_has_nulls and target.is_null(target_index)) { target.set_valid(target_index); }
+  }
+};
+
 template <typename Source, bool target_has_nulls, bool source_has_nulls>
 struct update_target_element<
   Source,

cpp/include/cudf/detail/hashing.hpp

@@ -46,6 +46,7 @@ std::unique_ptr<column> md5_hash(
   rmm::cuda_stream_view stream        = rmm::cuda_stream_default,
   rmm::mr::device_memory_resource* mr = rmm::mr::get_current_device_resource());
 
+template <template <typename> class hash_function>
 std::unique_ptr<column> serial_murmur_hash3_32(
   table_view const& input,
   uint32_t seed                       = 0,