From 8860baf33fb8f24f08380cda39027a88f0b9b1e0 Mon Sep 17 00:00:00 2001
From: rwlee <rwlee@users.noreply.github.com>
Date: Mon, 4 Jan 2021 13:22:25 -0800
Subject: [PATCH 01/20] Spark Murmur3 hash functionality(#7024)

Resolves #6863

Expands existing murmur3 hashing functionality to match Spark's murmur3 hashing algorithm by modifying tail processing for unaligned bytes and processing booleans as 32bit integers rather than singular bytes.

Authors:
  - Ryan Lee <ryanlee@nvidia.com>
  - rwlee <rwlee@users.noreply.github.com>

Approvers:
  - Jake Hemstad
  - null
  - Robert (Bobby) Evans
  - GALI PREM SAGAR

URL: https://github.com/rapidsai/cudf/pull/7024
---
 cpp/include/cudf/detail/hashing.hpp           |   1 +
 .../cudf/detail/utilities/hash_functions.cuh  | 234 ++++++++++++++----
 cpp/include/cudf/types.hpp                    |   9 +-
 cpp/src/hash/hashing.cu                       |  10 +-
 cpp/tests/hashing/hash_test.cpp               |  65 ++++-
 .../java/ai/rapids/cudf/ColumnVector.java     |  47 +++-
 .../main/java/ai/rapids/cudf/HashType.java    |   3 +-
 .../java/ai/rapids/cudf/ColumnVectorTest.java | 137 +++++++---
 python/cudf/cudf/_lib/cpp/types.pxd           |   1 +
 9 files changed, 419 insertions(+), 88 deletions(-)

diff --git a/cpp/include/cudf/detail/hashing.hpp b/cpp/include/cudf/detail/hashing.hpp
index 18f97f00f21..06f523c2320 100644
--- a/cpp/include/cudf/detail/hashing.hpp
+++ b/cpp/include/cudf/detail/hashing.hpp
@@ -46,6 +46,7 @@ std::unique_ptr<column> md5_hash(
   rmm::cuda_stream_view stream        = rmm::cuda_stream_default,
   rmm::mr::device_memory_resource* mr = rmm::mr::get_current_device_resource());
 
+template <template <typename> class hash_function>
 std::unique_ptr<column> serial_murmur_hash3_32(
   table_view const& input,
   uint32_t seed                       = 0,
diff --git a/cpp/include/cudf/detail/utilities/hash_functions.cuh b/cpp/include/cudf/detail/utilities/hash_functions.cuh
index 08d88aee137..e9d66d125dd 100644
--- a/cpp/include/cudf/detail/utilities/hash_functions.cuh
+++ b/cpp/include/cudf/detail/utilities/hash_functions.cuh
@@ -111,7 +111,7 @@ void CUDA_DEVICE_CALLABLE md5_process(TKey const& key, md5_intermediate_data* ha
 /**
  * Normalization of floating point NANs and zeros helper
  */
-template <typename T, typename std::enable_if_t<std::is_floating_point<T>::value>* = nullptr>
+template <typename T, std::enable_if_t<std::is_floating_point<T>::value>* = nullptr>
 T CUDA_DEVICE_CALLABLE normalize_nans_and_zeros_helper(T key)
 {
   if (isnan(key)) {
@@ -148,7 +148,7 @@ void CUDA_DEVICE_CALLABLE uint32ToLowercaseHexString(uint32_t num, char* destina
 }
 
 struct MD5ListHasher {
-  template <typename T, typename std::enable_if_t<is_chrono<T>()>* = nullptr>
+  template <typename T, std::enable_if_t<is_chrono<T>()>* = nullptr>
   void __device__ operator()(column_device_view data_col,
                              size_type offset_begin,
                              size_type offset_end,
@@ -157,7 +157,7 @@ struct MD5ListHasher {
     release_assert(false && "MD5 Unsupported chrono type column");
   }
 
-  template <typename T, typename std::enable_if_t<!is_fixed_width<T>()>* = nullptr>
+  template <typename T, std::enable_if_t<!is_fixed_width<T>()>* = nullptr>
   void __device__ operator()(column_device_view data_col,
                              size_type offset_begin,
                              size_type offset_end,
@@ -166,7 +166,7 @@ struct MD5ListHasher {
     release_assert(false && "MD5 Unsupported non-fixed-width type column");
   }
 
-  template <typename T, typename std::enable_if_t<is_floating_point<T>()>* = nullptr>
+  template <typename T, std::enable_if_t<is_floating_point<T>()>* = nullptr>
   void __device__ operator()(column_device_view data_col,
                              size_type offset_begin,
                              size_type offset_end,
@@ -179,9 +179,9 @@ struct MD5ListHasher {
     }
   }
 
-  template <typename T,
-            typename std::enable_if_t<is_fixed_width<T>() && !is_floating_point<T>() &&
-                                      !is_chrono<T>()>* = nullptr>
+  template <
+    typename T,
+    std::enable_if_t<is_fixed_width<T>() && !is_floating_point<T>() && !is_chrono<T>()>* = nullptr>
   void CUDA_DEVICE_CALLABLE operator()(column_device_view data_col,
                                        size_type offset_begin,
                                        size_type offset_end,
@@ -268,7 +268,7 @@ struct MD5Hash {
       uint32ToLowercaseHexString(hash_state->hash_value[i], result_location + (8 * i));
   }
 
-  template <typename T, typename std::enable_if_t<is_chrono<T>()>* = nullptr>
+  template <typename T, std::enable_if_t<is_chrono<T>()>* = nullptr>
   void __device__ operator()(column_device_view col,
                              size_type row_index,
                              md5_intermediate_data* hash_state) const
@@ -276,7 +276,7 @@ struct MD5Hash {
     release_assert(false && "MD5 Unsupported chrono type column");
   }
 
-  template <typename T, typename std::enable_if_t<!is_fixed_width<T>()>* = nullptr>
+  template <typename T, std::enable_if_t<!is_fixed_width<T>()>* = nullptr>
   void __device__ operator()(column_device_view col,
                              size_type row_index,
                              md5_intermediate_data* hash_state) const
@@ -284,7 +284,7 @@ struct MD5Hash {
     release_assert(false && "MD5 Unsupported non-fixed-width type column");
   }
 
-  template <typename T, typename std::enable_if_t<is_floating_point<T>()>* = nullptr>
+  template <typename T, std::enable_if_t<is_floating_point<T>()>* = nullptr>
   void __device__ operator()(column_device_view col,
                              size_type row_index,
                              md5_intermediate_data* hash_state) const
@@ -292,9 +292,9 @@ struct MD5Hash {
     md5_process(normalize_nans_and_zeros_helper<T>(col.element<T>(row_index)), hash_state);
   }
 
-  template <typename T,
-            typename std::enable_if_t<is_fixed_width<T>() && !is_floating_point<T>() &&
-                                      !is_chrono<T>()>* = nullptr>
+  template <
+    typename T,
+    std::enable_if_t<is_fixed_width<T>() && !is_floating_point<T>() && !is_chrono<T>()>* = nullptr>
   void CUDA_DEVICE_CALLABLE operator()(column_device_view col,
                                        size_type row_index,
                                        md5_intermediate_data* hash_state) const
@@ -370,15 +370,15 @@ struct MurmurHash3_32 {
   using argument_type = Key;
   using result_type   = hash_value_type;
 
-  CUDA_HOST_DEVICE_CALLABLE MurmurHash3_32() : m_seed(0) {}
+  MurmurHash3_32() = default;
   CUDA_HOST_DEVICE_CALLABLE MurmurHash3_32(uint32_t seed) : m_seed(seed) {}
 
-  CUDA_HOST_DEVICE_CALLABLE uint32_t rotl32(uint32_t x, int8_t r) const
+  CUDA_DEVICE_CALLABLE uint32_t rotl32(uint32_t x, int8_t r) const
   {
     return (x << r) | (x >> (32 - r));
   }
 
-  CUDA_HOST_DEVICE_CALLABLE uint32_t fmix32(uint32_t h) const
+  CUDA_DEVICE_CALLABLE uint32_t fmix32(uint32_t h) const
   {
     h ^= h >> 16;
     h *= 0x85ebca6b;
@@ -399,7 +399,7 @@ struct MurmurHash3_32 {
    *
    * @returns A hash value that intelligently combines the lhs and rhs hash values
    */
-  CUDA_HOST_DEVICE_CALLABLE result_type hash_combine(result_type lhs, result_type rhs)
+  CUDA_DEVICE_CALLABLE result_type hash_combine(result_type lhs, result_type rhs)
   {
     result_type combined{lhs};
 
@@ -408,11 +408,11 @@ struct MurmurHash3_32 {
     return combined;
   }
 
-  result_type CUDA_HOST_DEVICE_CALLABLE operator()(Key const& key) const { return compute(key); }
+  result_type CUDA_DEVICE_CALLABLE operator()(Key const& key) const { return compute(key); }
 
   // compute wrapper for floating point types
-  template <typename T, typename std::enable_if_t<std::is_floating_point<T>::value>* = nullptr>
-  hash_value_type CUDA_HOST_DEVICE_CALLABLE compute_floating_point(T const& key) const
+  template <typename T, std::enable_if_t<std::is_floating_point<T>::value>* = nullptr>
+  hash_value_type CUDA_DEVICE_CALLABLE compute_floating_point(T const& key) const
   {
     if (key == T{0.0}) {
       return compute(T{0.0});
@@ -425,10 +425,10 @@ struct MurmurHash3_32 {
   }
 
   template <typename TKey>
-  result_type CUDA_HOST_DEVICE_CALLABLE compute(TKey const& key) const
+  result_type CUDA_DEVICE_CALLABLE compute(TKey const& key) const
   {
     constexpr int len         = sizeof(argument_type);
-    const uint8_t* const data = (const uint8_t*)&key;
+    uint8_t const* const data = reinterpret_cast<uint8_t const*>(&key);
     constexpr int nblocks     = len / 4;
 
     uint32_t h1           = m_seed;
@@ -436,7 +436,7 @@ struct MurmurHash3_32 {
     constexpr uint32_t c2 = 0x1b873593;
     //----------
     // body
-    const uint32_t* const blocks = (const uint32_t*)(data + nblocks * 4);
+    uint32_t const* const blocks = reinterpret_cast<uint32_t const*>(data + nblocks * 4);
     for (int i = -nblocks; i; i++) {
       uint32_t k1 = blocks[i];  // getblock32(blocks,i);
       k1 *= c1;
@@ -448,7 +448,7 @@ struct MurmurHash3_32 {
     }
     //----------
     // tail
-    const uint8_t* tail = (const uint8_t*)(data + nblocks * 4);
+    uint8_t const* tail = reinterpret_cast<uint8_t const*>(data + nblocks * 4);
     uint32_t k1         = 0;
     switch (len & 3) {
       case 3: k1 ^= tail[2] << 16;
@@ -468,11 +468,11 @@ struct MurmurHash3_32 {
   }
 
  private:
-  uint32_t m_seed;
+  uint32_t m_seed{0};
 };
 
 template <>
-hash_value_type CUDA_HOST_DEVICE_CALLABLE MurmurHash3_32<bool>::operator()(bool const& key) const
+hash_value_type CUDA_DEVICE_CALLABLE MurmurHash3_32<bool>::operator()(bool const& key) const
 {
   return this->compute(static_cast<uint8_t>(key));
 }
@@ -481,28 +481,24 @@ hash_value_type CUDA_HOST_DEVICE_CALLABLE MurmurHash3_32<bool>::operator()(bool
  * @brief Specialization of MurmurHash3_32 operator for strings.
  */
 template <>
-hash_value_type CUDA_HOST_DEVICE_CALLABLE
+hash_value_type CUDA_DEVICE_CALLABLE
 MurmurHash3_32<cudf::string_view>::operator()(cudf::string_view const& key) const
 {
-  const int len         = (int)key.size_bytes();
-  const uint8_t* data   = (const uint8_t*)key.data();
-  const int nblocks     = len / 4;
+  auto const len        = key.size_bytes();
+  uint8_t const* data   = reinterpret_cast<uint8_t const*>(key.data());
+  int const nblocks     = len / 4;
   result_type h1        = m_seed;
   constexpr uint32_t c1 = 0xcc9e2d51;
   constexpr uint32_t c2 = 0x1b873593;
-  auto getblock32       = [] __host__ __device__(const uint32_t* p, int i) -> uint32_t {
-  // Individual byte reads for unaligned accesses (very likely)
-#ifndef __CUDA_ARCH__
-    CUDF_FAIL("Hashing a string in host code is not supported.");
-#else
-    auto q = (const uint8_t*)(p + i);
+  auto getblock32       = [] __device__(uint32_t const* p, int i) -> uint32_t {
+    // Individual byte reads for unaligned accesses (very likely)
+    auto q = (uint8_t const*)(p + i);
     return q[0] | (q[1] << 8) | (q[2] << 16) | (q[3] << 24);
-#endif
   };
 
   //----------
   // body
-  const uint32_t* const blocks = (const uint32_t*)(data + nblocks * 4);
+  uint32_t const* const blocks = reinterpret_cast<uint32_t const*>(data + nblocks * 4);
   for (int i = -nblocks; i; i++) {
     uint32_t k1 = getblock32(blocks, i);
     k1 *= c1;
@@ -514,7 +510,7 @@ MurmurHash3_32<cudf::string_view>::operator()(cudf::string_view const& key) cons
   }
   //----------
   // tail
-  const uint8_t* tail = (const uint8_t*)(data + nblocks * 4);
+  uint8_t const* tail = reinterpret_cast<uint8_t const*>(data + nblocks * 4);
   uint32_t k1         = 0;
   switch (len & 3) {
     case 3: k1 ^= tail[2] << 16;
@@ -534,14 +530,164 @@ MurmurHash3_32<cudf::string_view>::operator()(cudf::string_view const& key) cons
 }
 
 template <>
-hash_value_type CUDA_HOST_DEVICE_CALLABLE MurmurHash3_32<float>::operator()(float const& key) const
+hash_value_type CUDA_DEVICE_CALLABLE MurmurHash3_32<float>::operator()(float const& key) const
+{
+  return this->compute_floating_point(key);
+}
+
+template <>
+hash_value_type CUDA_DEVICE_CALLABLE MurmurHash3_32<double>::operator()(double const& key) const
+{
+  return this->compute_floating_point(key);
+}
+
+template <typename Key>
+struct SparkMurmurHash3_32 {
+  using argument_type = Key;
+  using result_type   = hash_value_type;
+
+  SparkMurmurHash3_32() = default;
+  CUDA_HOST_DEVICE_CALLABLE SparkMurmurHash3_32(uint32_t seed) : m_seed(seed) {}
+
+  CUDA_DEVICE_CALLABLE uint32_t rotl32(uint32_t x, int8_t r) const
+  {
+    return (x << r) | (x >> (32 - r));
+  }
+
+  CUDA_DEVICE_CALLABLE uint32_t fmix32(uint32_t h) const
+  {
+    h ^= h >> 16;
+    h *= 0x85ebca6b;
+    h ^= h >> 13;
+    h *= 0xc2b2ae35;
+    h ^= h >> 16;
+    return h;
+  }
+
+  result_type CUDA_DEVICE_CALLABLE operator()(Key const& key) const { return compute(key); }
+
+  // compute wrapper for floating point types
+  template <typename T, std::enable_if_t<std::is_floating_point<T>::value>* = nullptr>
+  hash_value_type CUDA_DEVICE_CALLABLE compute_floating_point(T const& key) const
+  {
+    if (key == T{0.0}) {
+      return compute(T{0.0});
+    } else if (isnan(key)) {
+      T nan = std::numeric_limits<T>::quiet_NaN();
+      return compute(nan);
+    } else {
+      return compute(key);
+    }
+  }
+
+  template <typename TKey>
+  result_type CUDA_DEVICE_CALLABLE compute(TKey const& key) const
+  {
+    constexpr int len        = sizeof(TKey);
+    int8_t const* const data = reinterpret_cast<int8_t const*>(&key);
+    constexpr int nblocks    = len / 4;
+
+    uint32_t h1           = m_seed;
+    constexpr uint32_t c1 = 0xcc9e2d51;
+    constexpr uint32_t c2 = 0x1b873593;
+    //----------
+    // body
+    uint32_t const* const blocks = reinterpret_cast<uint32_t const*>(data + nblocks * 4);
+    for (int i = -nblocks; i; i++) {
+      uint32_t k1 = blocks[i];
+      k1 *= c1;
+      k1 = rotl32(k1, 15);
+      k1 *= c2;
+      h1 ^= k1;
+      h1 = rotl32(h1, 13);
+      h1 = h1 * 5 + 0xe6546b64;
+    }
+    //----------
+    // byte by byte tail processing
+    for (int i = nblocks * 4; i < len; i++) {
+      int32_t k1 = data[i];
+      k1 *= c1;
+      k1 = rotl32(k1, 15);
+      k1 *= c2;
+      h1 ^= k1;
+      h1 = rotl32(h1, 13);
+      h1 = h1 * 5 + 0xe6546b64;
+    }
+    //----------
+    // finalization
+    h1 ^= len;
+    h1 = fmix32(h1);
+    return h1;
+  }
+
+ private:
+  uint32_t m_seed{0};
+};
+
+template <>
+hash_value_type CUDA_DEVICE_CALLABLE SparkMurmurHash3_32<bool>::operator()(bool const& key) const
+{
+  return this->compute<uint32_t>(key);
+}
+
+/**
+ * @brief Specialization of MurmurHash3_32 operator for strings.
+ */
+template <>
+hash_value_type CUDA_DEVICE_CALLABLE
+SparkMurmurHash3_32<cudf::string_view>::operator()(cudf::string_view const& key) const
+{
+  auto const len        = key.size_bytes();
+  int8_t const* data    = reinterpret_cast<int8_t const*>(key.data());
+  int const nblocks     = len / 4;
+  result_type h1        = m_seed;
+  constexpr uint32_t c1 = 0xcc9e2d51;
+  constexpr uint32_t c2 = 0x1b873593;
+  auto getblock32       = [] __device__(uint32_t const* p, int i) -> uint32_t {
+    // Individual byte reads for unaligned accesses (very likely)
+    auto q = (const uint8_t*)(p + i);
+    return q[0] | (q[1] << 8) | (q[2] << 16) | (q[3] << 24);
+  };
+
+  //----------
+  // body
+  uint32_t const* const blocks = reinterpret_cast<uint32_t const*>(data + nblocks * 4);
+  for (int i = -nblocks; i; i++) {
+    uint32_t k1 = getblock32(blocks, i);
+    k1 *= c1;
+    k1 = rotl32(k1, 15);
+    k1 *= c2;
+    h1 ^= k1;
+    h1 = rotl32(h1, 13);
+    h1 = h1 * 5 + 0xe6546b64;
+  }
+  //----------
+  // Spark's byte by byte tail processing
+  for (int i = nblocks * 4; i < len; i++) {
+    int32_t k1 = data[i];
+    k1 *= c1;
+    k1 = rotl32(k1, 15);
+    k1 *= c2;
+    h1 ^= k1;
+    h1 = rotl32(h1, 13);
+    h1 = h1 * 5 + 0xe6546b64;
+  }
+  //----------
+  // finalization
+  h1 ^= len;
+  h1 = fmix32(h1);
+  return h1;
+}
+
+template <>
+hash_value_type CUDA_DEVICE_CALLABLE SparkMurmurHash3_32<float>::operator()(float const& key) const
 {
   return this->compute_floating_point(key);
 }
 
 template <>
-hash_value_type CUDA_HOST_DEVICE_CALLABLE
-MurmurHash3_32<double>::operator()(double const& key) const
+hash_value_type CUDA_DEVICE_CALLABLE
+SparkMurmurHash3_32<double>::operator()(double const& key) const
 {
   return this->compute_floating_point(key);
 }
@@ -576,7 +722,7 @@ struct IdentityHash {
 
   template <typename return_type = result_type>
   CUDA_HOST_DEVICE_CALLABLE std::enable_if_t<!std::is_arithmetic<Key>::value, return_type>
-  operator()(const Key& key) const
+  operator()(Key const& key) const
   {
     release_assert(false && "IdentityHash does not support this data type");
     return 0;
@@ -584,7 +730,7 @@ struct IdentityHash {
 
   template <typename return_type = result_type>
   CUDA_HOST_DEVICE_CALLABLE std::enable_if_t<std::is_arithmetic<Key>::value, return_type>
-  operator()(const Key& key) const
+  operator()(Key const& key) const
   {
     return static_cast<result_type>(key);
   }
diff --git a/cpp/include/cudf/types.hpp b/cpp/include/cudf/types.hpp
index a08ae366f68..48e5d9543b8 100644
--- a/cpp/include/cudf/types.hpp
+++ b/cpp/include/cudf/types.hpp
@@ -324,10 +324,11 @@ std::size_t size_of(data_type t);
  *  @brief Identifies the hash function to be used
  */
 enum class hash_id {
-  HASH_IDENTITY = 0,   ///< Identity hash function that simply returns the key to be hashed
-  HASH_MURMUR3,        ///< Murmur3 hash function
-  HASH_MD5,            ///< MD5 hash function
-  HASH_SERIAL_MURMUR3  ///< Serial Murmur3 hash function
+  HASH_IDENTITY = 0,    ///< Identity hash function that simply returns the key to be hashed
+  HASH_MURMUR3,         ///< Murmur3 hash function
+  HASH_MD5,             ///< MD5 hash function
+  HASH_SERIAL_MURMUR3,  ///< Serial Murmur3 hash function
+  HASH_SPARK_MURMUR3    ///< Spark Murmur3 hash function
 };
 
 /** @} */
diff --git a/cpp/src/hash/hashing.cu b/cpp/src/hash/hashing.cu
index 13ee340ba27..16efb666b3e 100644
--- a/cpp/src/hash/hashing.cu
+++ b/cpp/src/hash/hashing.cu
@@ -52,7 +52,10 @@ std::unique_ptr<column> hash(table_view const& input,
   switch (hash_function) {
     case (hash_id::HASH_MURMUR3): return murmur_hash3_32(input, initial_hash, stream, mr);
     case (hash_id::HASH_MD5): return md5_hash(input, stream, mr);
-    case (hash_id::HASH_SERIAL_MURMUR3): return serial_murmur_hash3_32(input, seed, stream, mr);
+    case (hash_id::HASH_SERIAL_MURMUR3):
+      return serial_murmur_hash3_32<MurmurHash3_32>(input, seed, stream, mr);
+    case (hash_id::HASH_SPARK_MURMUR3):
+      return serial_murmur_hash3_32<SparkMurmurHash3_32>(input, seed, stream, mr);
     default: return nullptr;
   }
 }
@@ -119,6 +122,7 @@ std::unique_ptr<column> md5_hash(table_view const& input,
                              mr);
 }
 
+template <template <typename> class hash_function>
 std::unique_ptr<column> serial_murmur_hash3_32(table_view const& input,
                                                uint32_t seed,
                                                rmm::cuda_stream_view stream,
@@ -145,7 +149,7 @@ std::unique_ptr<column> serial_murmur_hash3_32(table_view const& input,
                          [rindex = row_index] __device__(auto hash, auto column) {
                            return cudf::type_dispatcher(
                              column.type(),
-                             element_hasher_with_seed<MurmurHash3_32, true>{hash, hash},
+                             element_hasher_with_seed<hash_function, true>{hash, hash},
                              column,
                              rindex);
                          });
@@ -163,7 +167,7 @@ std::unique_ptr<column> serial_murmur_hash3_32(table_view const& input,
                          [rindex = row_index] __device__(auto hash, auto column) {
                            return cudf::type_dispatcher(
                              column.type(),
-                             element_hasher_with_seed<MurmurHash3_32, false>{hash, hash},
+                             element_hasher_with_seed<hash_function, false>{hash, hash},
                              column,
                              rindex);
                          });
diff --git a/cpp/tests/hashing/hash_test.cpp b/cpp/tests/hashing/hash_test.cpp
index 12cea8bc0de..88415d820d6 100644
--- a/cpp/tests/hashing/hash_test.cpp
+++ b/cpp/tests/hashing/hash_test.cpp
@@ -115,6 +115,12 @@ TEST_F(HashTest, MultiValueNulls)
 
   EXPECT_EQ(input1.num_rows(), serial_output1->size());
   CUDF_TEST_EXPECT_COLUMNS_EQUAL(serial_output1->view(), serial_output2->view());
+
+  auto const spark_output1 = cudf::hash(input1, cudf::hash_id::HASH_SPARK_MURMUR3, {}, 0);
+  auto const spark_output2 = cudf::hash(input2, cudf::hash_id::HASH_SPARK_MURMUR3);
+
+  EXPECT_EQ(input1.num_rows(), spark_output1->size());
+  CUDF_TEST_EXPECT_COLUMNS_EQUAL(spark_output1->view(), spark_output2->view());
 }
 
 template <typename T>
@@ -140,6 +146,12 @@ TYPED_TEST(HashTestTyped, Equality)
 
   EXPECT_EQ(input.num_rows(), serial_output1->size());
   CUDF_TEST_EXPECT_COLUMNS_EQUAL(serial_output1->view(), serial_output2->view());
+
+  auto const spark_output1 = cudf::hash(input, cudf::hash_id::HASH_SPARK_MURMUR3, {}, 0);
+  auto const spark_output2 = cudf::hash(input, cudf::hash_id::HASH_SPARK_MURMUR3);
+
+  EXPECT_EQ(input.num_rows(), spark_output1->size());
+  CUDF_TEST_EXPECT_COLUMNS_EQUAL(spark_output1->view(), spark_output2->view());
 }
 
 TYPED_TEST(HashTestTyped, EqualityNulls)
@@ -164,6 +176,12 @@ TYPED_TEST(HashTestTyped, EqualityNulls)
 
   EXPECT_EQ(input1.num_rows(), serial_output1->size());
   CUDF_TEST_EXPECT_COLUMNS_EQUAL(serial_output1->view(), serial_output2->view());
+
+  auto const spark_output1 = cudf::hash(input1, cudf::hash_id::HASH_SPARK_MURMUR3, {}, 0);
+  auto const spark_output2 = cudf::hash(input2, cudf::hash_id::HASH_SPARK_MURMUR3);
+
+  EXPECT_EQ(input1.num_rows(), spark_output1->size());
+  CUDF_TEST_EXPECT_COLUMNS_EQUAL(spark_output1->view(), spark_output2->view());
 }
 
 template <typename T>
@@ -196,6 +214,11 @@ TYPED_TEST(HashTestFloatTyped, TestExtremes)
   auto const serial_output2 = cudf::hash(input2, cudf::hash_id::HASH_SERIAL_MURMUR3);
 
   CUDF_TEST_EXPECT_COLUMNS_EQUAL(serial_output1->view(), serial_output2->view());
+
+  auto const spark_output1 = cudf::hash(input1, cudf::hash_id::HASH_SPARK_MURMUR3, {}, 0);
+  auto const spark_output2 = cudf::hash(input2, cudf::hash_id::HASH_SPARK_MURMUR3);
+
+  CUDF_TEST_EXPECT_COLUMNS_EQUAL(spark_output1->view(), spark_output2->view());
 }
 
 class SerialMurmurHash3Test : public cudf::test::BaseFixture {
@@ -204,7 +227,7 @@ class SerialMurmurHash3Test : public cudf::test::BaseFixture {
 TEST_F(SerialMurmurHash3Test, MultiValueWithSeeds)
 {
   fixed_width_column_wrapper<int32_t> const strings_col_result(
-    {1467149710, -680899318, -1620282500, 91106683, -1165223012});
+    {1467149710, -680899318, -1620282500, 91106683, -1564993834});
   fixed_width_column_wrapper<int32_t> const ints_col_result(
     {933211791, 751823303, -1080202046, 723455942, 133916647});
 
@@ -212,7 +235,7 @@ TEST_F(SerialMurmurHash3Test, MultiValueWithSeeds)
                                             "The quick brown fox",
                                             "jumps over the lazy dog.",
                                             "All work and no play makes Jack a dull boy",
-                                            "!\"#$%&\'()*+,-./0123456789:;<=>?@[\\]^_`{|}~"});
+                                            "!\"#$%&\'()*+,-./]:;<=>?@[\\]^_`{|}~\ud720\ud721"});
 
   using limits = std::numeric_limits<int32_t>;
   fixed_width_column_wrapper<int32_t> const ints_col({0, 100, -100, limits::min(), limits::max()});
@@ -236,6 +259,44 @@ TEST_F(SerialMurmurHash3Test, MultiValueWithSeeds)
   CUDF_TEST_EXPECT_COLUMNS_EQUAL(hashed_output3->view(), hashed_output4->view(), true);
 }
 
+class SparkMurmurHash3Test : public cudf::test::BaseFixture {
+};
+
+TEST_F(SparkMurmurHash3Test, MultiValueWithSeeds)
+{
+  fixed_width_column_wrapper<int32_t> const strings_col_result(
+    {1467149710, 723257560, -1620282500, -2001858707, 1588473657});
+  fixed_width_column_wrapper<int32_t> const ints_col_result(
+    {933211791, 751823303, -1080202046, 723455942, 133916647});
+
+  strings_column_wrapper const strings_col({"",
+                                            "The quick brown fox",
+                                            "jumps over the lazy dog.",
+                                            "All work and no play makes Jack a dull boy",
+                                            "!\"#$%&\'()*+,-./:;<=>?@[\\]^_`{|}~\ud720\ud721"});
+
+  using limits = std::numeric_limits<int32_t>;
+  fixed_width_column_wrapper<int32_t> const ints_col({0, 100, -100, limits::min(), limits::max()});
+
+  fixed_width_column_wrapper<bool> const bools_col1({0, 1, 1, 1, 0});
+  fixed_width_column_wrapper<bool> const bools_col2({0, 1, 2, 255, 0});
+
+  auto const input1 = cudf::table_view({strings_col});
+  auto const input2 = cudf::table_view({ints_col});
+  auto const input3 = cudf::table_view({strings_col, ints_col, bools_col1});
+  auto const input4 = cudf::table_view({strings_col, ints_col, bools_col2});
+
+  auto const hashed_output1 = cudf::hash(input1, cudf::hash_id::HASH_SPARK_MURMUR3, {}, 314);
+  auto const hashed_output2 = cudf::hash(input2, cudf::hash_id::HASH_SPARK_MURMUR3, {}, 42);
+  auto const hashed_output3 = cudf::hash(input3, cudf::hash_id::HASH_SPARK_MURMUR3, {});
+  auto const hashed_output4 = cudf::hash(input4, cudf::hash_id::HASH_SPARK_MURMUR3, {});
+
+  CUDF_TEST_EXPECT_COLUMNS_EQUAL(hashed_output1->view(), strings_col_result, true);
+  CUDF_TEST_EXPECT_COLUMNS_EQUAL(hashed_output2->view(), ints_col_result, true);
+  EXPECT_EQ(input3.num_rows(), hashed_output3->size());
+  CUDF_TEST_EXPECT_COLUMNS_EQUAL(hashed_output3->view(), hashed_output4->view(), true);
+}
+
 class MD5HashTest : public cudf::test::BaseFixture {
 };
 
diff --git a/java/src/main/java/ai/rapids/cudf/ColumnVector.java b/java/src/main/java/ai/rapids/cudf/ColumnVector.java
index 926fba30418..34a2e2fe374 100644
--- a/java/src/main/java/ai/rapids/cudf/ColumnVector.java
+++ b/java/src/main/java/ai/rapids/cudf/ColumnVector.java
@@ -435,15 +435,15 @@ public static ColumnVector md5Hash(ColumnView... columns) {
   }
 
   /**
-   * Create a new vector containing the MD5 hash of each row in the table.
+   * Create a new vector containing the murmur3 hash of each row in the table.
    *
    * @param seed integer seed for the murmur3 hash function
    * @param columns array of columns to hash, must have identical number of rows.
-   * @return the new ColumnVector of 32 character hex strings representing each row's hash value.
+   * @return the new ColumnVector of 32-bit values representing each row's hash value.
    */
   public static ColumnVector serial32BitMurmurHash3(int seed, ColumnView columns[]) {
     if (columns.length < 1) {
-      throw new IllegalArgumentException("MD5 hashing requires at least 1 column of input");
+      throw new IllegalArgumentException("Murmur3 hashing requires at least 1 column of input");
     }
     long[] columnViews = new long[columns.length];
     long size = columns[0].getRowCount();
@@ -460,15 +460,52 @@ public static ColumnVector serial32BitMurmurHash3(int seed, ColumnView columns[]
   }
 
   /**
-   * Create a new vector containing the MD5 hash of each row in the table, seed defaulted to 0.
+   * Create a new vector containing the murmur3 hash of each row in the table, seed defaulted to 0.
    *
    * @param columns array of columns to hash, must have identical number of rows.
-   * @return the new ColumnVector of 32 character hex strings representing each row's hash value.
+   * @return the new ColumnVector of 32-bit values representing each row's hash value.
    */
   public static ColumnVector serial32BitMurmurHash3(ColumnView columns[]) {
     return serial32BitMurmurHash3(0, columns);
   }
 
+  /**
+   * Create a new vector containing spark's 32-bit murmur3 hash of each row in the table.
+   * Spark's murmur3 hash uses a different tail processing algorithm.
+   *
+   * @param seed integer seed for the murmur3 hash function
+   * @param columns array of columns to hash, must have identical number of rows.
+   * @return the new ColumnVector of 32-bit values representing each row's hash value.
+   */
+  public static ColumnVector spark32BitMurmurHash3(int seed, ColumnView columns[]) {
+    if (columns.length < 1) {
+      throw new IllegalArgumentException("Murmur3 hashing requires at least 1 column of input");
+    }
+    long[] columnViews = new long[columns.length];
+    long size = columns[0].getRowCount();
+
+    for(int i = 0; i < columns.length; i++) {
+      assert columns[i] != null : "Column vectors passed may not be null";
+      assert columns[i].getRowCount() == size : "Row count mismatch, all columns must be the same size";
+      assert !columns[i].getType().isDurationType() : "Unsupported column type Duration";
+      assert !columns[i].getType().isTimestampType() : "Unsupported column type Timestamp";
+      assert !columns[i].getType().isNestedType() : "Unsupported column of nested type";
+      columnViews[i] = columns[i].getNativeView();
+    }
+    return new ColumnVector(hash(columnViews, HashType.HASH_SPARK_MURMUR3.getNativeId(), new int[0], seed));
+  }
+
+  /**
+   * Create a new vector containing spark's 32-bit murmur3 hash of each row in the table with the
+   * seed set to 0. Spark's murmur3 hash uses a different tail processing algorithm.
+   *
+   * @param columns array of columns to hash, must have identical number of rows.
+   * @return the new ColumnVector of 32-bit values representing each row's hash value.
+   */
+  public static ColumnVector spark32BitMurmurHash3(ColumnView columns[]) {
+    return spark32BitMurmurHash3(0, columns);
+  }
+
   /**
    * Generic method to cast ColumnVector
    * When casting from a Date, Timestamp, or Boolean to a numerical type the underlying numerical
diff --git a/java/src/main/java/ai/rapids/cudf/HashType.java b/java/src/main/java/ai/rapids/cudf/HashType.java
index 9266545bd94..b521bc5c42c 100644
--- a/java/src/main/java/ai/rapids/cudf/HashType.java
+++ b/java/src/main/java/ai/rapids/cudf/HashType.java
@@ -25,7 +25,8 @@ public enum HashType {
   // TODO IDENTITY(0),
   // TODO MURMUR3(1),
   HASH_MD5(2),
-  HASH_SERIAL_MURMUR3(3);
+  HASH_SERIAL_MURMUR3(3),
+  HASH_SPARK_MURMUR3(4);
 
   private static final HashType[] HASH_TYPES = HashType.values();
   final int nativeId;
diff --git a/java/src/test/java/ai/rapids/cudf/ColumnVectorTest.java b/java/src/test/java/ai/rapids/cudf/ColumnVectorTest.java
index e01b0b8ef1b..d3cde7f77cc 100644
--- a/java/src/test/java/ai/rapids/cudf/ColumnVectorTest.java
+++ b/java/src/test/java/ai/rapids/cudf/ColumnVectorTest.java
@@ -284,7 +284,7 @@ void testNormalizeNANsAndZeros() {
   @Test
   void testMD5HashStrings() {
     try (ColumnVector v0 = ColumnVector.fromStrings(
-          "a", "B\n", "dE\"\u0100\t\u0101 \u0500\u0501",
+          "a", "B\n", "dE\"\u0100\t\u0101 \ud720\ud721",
           "A very long (greater than 128 bytes/char string) to test a multi hash-step data point " +
           "in the MD5 hash function. This string needed to be longer.",
           null, null);
@@ -293,7 +293,7 @@ void testMD5HashStrings() {
            "A 60 character string to test MD5's message padding algorithm",
            "hiJ\ud720\ud721\ud720\ud721", null);
          ColumnVector v2 = ColumnVector.fromStrings(
-           "a", "B\nc",  "dE\"\u0100\t\u0101 \u0500\u0501\\Fg2\'",
+           "a", "B\nc",  "dE\"\u0100\t\u0101 \ud720\ud721\\Fg2\'",
            "A very long (greater than 128 bytes/char string) to test a multi hash-step data point " +
            "in the MD5 hash function. This string needed to be longer.A 60 character string to " +
            "test MD5's message padding algorithm",
@@ -302,7 +302,7 @@ void testMD5HashStrings() {
          ColumnVector result2 = ColumnVector.md5Hash(v2);
          ColumnVector expected = ColumnVector.fromStrings(
           "0cc175b9c0f1b6a831c399e269772661", "f5112705c2f6dc7d3fc6bd496df6c2e8",
-          "675c30ce6d1b27dcb5009b01be42e9bd", "8fa29148f63c1fe9248fdc4644e3a193",
+          "d49db62680847e0e7107e0937d29668e", "8fa29148f63c1fe9248fdc4644e3a193",
           "1bc221b25e6c4825929e884092f4044f", "d41d8cd98f00b204e9800998ecf8427e")) {
       assertColumnsAreEqual(result01, expected);
       assertColumnsAreEqual(result2, expected);
@@ -318,7 +318,7 @@ void testMD5HashInts() {
           "7dea362b3fac8e00956a4952a3d4f474", "cdc824bf721df654130ed7447fb878ac",
           "7fb89558e395330c6a10ab98915fcafb", "d41d8cd98f00b204e9800998ecf8427e",
           "152afd7bf4dbe26f85032eee0269201a", "ed0622e1179e101cf7edc0b952cc5262")) {
-      assertColumnsAreEqual(result, expected);
+      assertColumnsAreEqual(expected, result);
     }
   }
 
@@ -337,7 +337,7 @@ void testMD5HashDoubles() {
           "bf26d90b64827fdbc58da0aa195156fe", "bf26d90b64827fdbc58da0aa195156fe",
           "bf26d90b64827fdbc58da0aa195156fe", "bf26d90b64827fdbc58da0aa195156fe",
           "73c82437c94e197f7e35e14f0140497a", "740660a5f71e7a264fca45330c34da31")) {
-      assertColumnsAreEqual(result, expected);
+      assertColumnsAreEqual(expected, result);
     }
   }
 
@@ -356,7 +356,7 @@ void testMD5HashFloats() {
           "d6fd2bac25776d9a7269ca0e24b21b36", "d6fd2bac25776d9a7269ca0e24b21b36",
           "d6fd2bac25776d9a7269ca0e24b21b36", "d6fd2bac25776d9a7269ca0e24b21b36",
           "55e3a4db046ad9065bd7d64243de408f", "33b552ad34a825b275f5f2b59778b5c3")){
-      assertColumnsAreEqual(result, expected);
+      assertColumnsAreEqual(expected, result);
     }
   }
 
@@ -369,14 +369,14 @@ void testMD5HashBools() {
           "d41d8cd98f00b204e9800998ecf8427e", "249ba6277758050695e8f5909bacd6d3",
           "c4103f122d27677c9db144cae1394a66", "55a54008ad1ba589aa210d2629c1df41",
           "93b885adfe0da089cdf634904fd59f71", "441077cc9e57554dd476bdfb8b8b8102")) {
-      assertColumnsAreEqual(result, expected);
+      assertColumnsAreEqual(expected, result);
     }
   }
 
   @Test
   void testMD5HashMixed() {
     try (ColumnVector strings = ColumnVector.fromStrings(
-          "a", "B\n", "dE\"\u0100\t\u0101 \u0500\u0501",
+          "a", "B\n", "dE\"\u0100\t\u0101 \ud720\ud721",
           "A very long (greater than 128 bytes/char string) to test a multi hash-step data point " +
           "in the MD5 hash function. This string needed to be longer.",
           null, null);
@@ -389,9 +389,9 @@ void testMD5HashMixed() {
          ColumnVector result = ColumnVector.md5Hash(strings, integers, doubles, floats, bools);
          ColumnVector expected = ColumnVector.fromStrings(
           "c12c8638819fdd8377bbf537a4ebf0b4", "abad86357c1ae28eeb89f4b59700946a",
-          "8c176e687e677a752868fe9d319d9d5c", "2f64d6a1d5b730fd97115924cf9aa486",
+          "7e376255c6354716cd63418208dc7b90", "2f64d6a1d5b730fd97115924cf9aa486",
           "9f9d26bb5d25d56453a91f0558370fa4", "d41d8cd98f00b204e9800998ecf8427e")) {
-      assertColumnsAreEqual(result, expected);
+      assertColumnsAreEqual(expected, result);
     }
   }
 
@@ -411,23 +411,22 @@ void testMD5HashLists() {
       assertColumnsAreEqual(expected, result);
     }
   }
-
   @Test
-  void test32BitMurmur3HashStrings() {
+  void testSerial32BitMurmur3HashStrings() {
     try (ColumnVector v0 = ColumnVector.fromStrings(
-           "a", "B\nc",  "dE\"\u0100\t\u0101 \u0500\u0501\\Fg2\'",
+           "a", "B\nc",  "dE\"\u0100\t\u0101 \ud720\ud721\\Fg2\'",
            "A very long (greater than 128 bytes/char string) to test a multi hash-step data point " +
            "in the MD5 hash function. This string needed to be longer.A 60 character string to " +
            "test MD5's message padding algorithm",
            "hiJ\ud720\ud721\ud720\ud721", null);
          ColumnVector result = ColumnVector.serial32BitMurmurHash3(42, new ColumnVector[]{v0});
-         ColumnVector expected = ColumnVector.fromBoxedInts(-1293573533, 1163854319, 1247767417, 1504480835, 1249086584, 42)) {
+         ColumnVector expected = ColumnVector.fromBoxedInts(-1293573533, 1163854319, 1423943036, 1504480835, 1249086584, 42)) {
       assertColumnsAreEqual(expected, result);
     }
   }
 
   @Test
-  void test32BitMurmur3HashInts() {
+  void testSerial32BitMurmur3HashInts() {
     try (ColumnVector v0 = ColumnVector.fromBoxedInts(0, 100, null, null, Integer.MIN_VALUE, null);
          ColumnVector v1 = ColumnVector.fromBoxedInts(0, null, -100, null, null, Integer.MAX_VALUE);
          ColumnVector result = ColumnVector.serial32BitMurmurHash3(42, new ColumnVector[]{v0, v1});
@@ -437,7 +436,7 @@ void test32BitMurmur3HashInts() {
   }
 
   @Test
-  void test32BitMurmur3HashDoubles() {
+  void testSerial32BitMurmur3HashDoubles() {
     try (ColumnVector v = ColumnVector.fromBoxedDoubles(
           0.0, null, 100.0, -100.0, Double.MIN_NORMAL, Double.MAX_VALUE,
           POSITIVE_DOUBLE_NAN_UPPER_RANGE, POSITIVE_DOUBLE_NAN_LOWER_RANGE,
@@ -450,7 +449,7 @@ void test32BitMurmur3HashDoubles() {
   }
 
   @Test
-  void test32BitMurmur3HashFloats() {
+  void testSerialBitMurmur3HashFloats() {
     try (ColumnVector v = ColumnVector.fromBoxedFloats(
           0f, 100f, -100f, Float.MIN_NORMAL, Float.MAX_VALUE, null,
           POSITIVE_FLOAT_NAN_LOWER_RANGE, POSITIVE_FLOAT_NAN_UPPER_RANGE,
@@ -463,7 +462,7 @@ void test32BitMurmur3HashFloats() {
   }
 
   @Test
-  void test32BitMurmur3HashBools() {
+  void testSerial32BitMurmur3HashBools() {
     try (ColumnVector v0 = ColumnVector.fromBoxedBooleans(null, true, false, true, null, false);
          ColumnVector v1 = ColumnVector.fromBoxedBooleans(null, true, false, null, false, true);
          ColumnVector result = ColumnVector.serial32BitMurmurHash3(0, new ColumnVector[]{v0, v1});
@@ -473,9 +472,9 @@ void test32BitMurmur3HashBools() {
   }
 
   @Test
-  void test32BitMurmur3HashMixed() {
+  void testSerial32BitMurmur3HashMixed() {
     try (ColumnVector strings = ColumnVector.fromStrings(
-          "a", "B\n", "dE\"\u0100\t\u0101 \u0500\u0501",
+          "a", "B\n", "dE\"\u0100\t\u0101 \ud720\ud721",
           "A very long (greater than 128 bytes/char string) to test a multi hash-step data point " +
           "in the MD5 hash function. This string needed to be longer.",
           null, null);
@@ -486,7 +485,87 @@ void test32BitMurmur3HashMixed() {
           0f, 100f, -100f, NEGATIVE_FLOAT_NAN_LOWER_RANGE, NEGATIVE_FLOAT_NAN_UPPER_RANGE, null);
          ColumnVector bools = ColumnVector.fromBoxedBooleans(true, false, null, false, true, null);
          ColumnVector result = ColumnVector.serial32BitMurmurHash3(1868, new ColumnVector[]{strings, integers, doubles, floats, bools});
-         ColumnVector expected = ColumnVector.fromBoxedInts(387200465, 1988790727, 1843539474, 814731646, -1073686048, 1868)) {
+         ColumnVector expected = ColumnVector.fromBoxedInts(387200465, 1988790727, 774895031, 814731646, -1073686048, 1868)) {
+      assertColumnsAreEqual(expected, result);
+    }
+  }
+
+  @Test
+  void testSpark32BitMurmur3HashStrings() {
+    try (ColumnVector v0 = ColumnVector.fromStrings(
+           "a", "B\nc",  "dE\"\u0100\t\u0101 \ud720\ud721\\Fg2\'",
+           "A very long (greater than 128 bytes/char string) to test a multi hash-step data point " +
+           "in the MD5 hash function. This string needed to be longer.A 60 character string to " +
+           "test MD5's message padding algorithm",
+           "hiJ\ud720\ud721\ud720\ud721", null);
+         ColumnVector result = ColumnVector.spark32BitMurmurHash3(42, new ColumnVector[]{v0});
+         ColumnVector expected = ColumnVector.fromBoxedInts(1485273170, 1709559900, 1423943036, 176121990, 1199621434, 42)) {
+      assertColumnsAreEqual(expected, result);
+    }
+  }
+
+  @Test
+  void testSpark32BitMurmur3HashInts() {
+    try (ColumnVector v0 = ColumnVector.fromBoxedInts(0, 100, null, null, Integer.MIN_VALUE, null);
+         ColumnVector v1 = ColumnVector.fromBoxedInts(0, null, -100, null, null, Integer.MAX_VALUE);
+         ColumnVector result = ColumnVector.spark32BitMurmurHash3(42, new ColumnVector[]{v0, v1});
+         ColumnVector expected = ColumnVector.fromBoxedInts(59727262, 751823303, -1080202046, 42, 723455942, 133916647)) {
+      assertColumnsAreEqual(expected, result);
+    }
+  }
+
+  @Test
+  void testSpark32BitMurmur3HashDoubles() {
+    try (ColumnVector v = ColumnVector.fromBoxedDoubles(
+          0.0, null, 100.0, -100.0, Double.MIN_NORMAL, Double.MAX_VALUE,
+          POSITIVE_DOUBLE_NAN_UPPER_RANGE, POSITIVE_DOUBLE_NAN_LOWER_RANGE,
+          NEGATIVE_DOUBLE_NAN_UPPER_RANGE, NEGATIVE_DOUBLE_NAN_LOWER_RANGE,
+          Double.POSITIVE_INFINITY, Double.NEGATIVE_INFINITY);
+         ColumnVector result = ColumnVector.spark32BitMurmurHash3(new ColumnVector[]{v});
+         ColumnVector expected = ColumnVector.fromBoxedInts(1669671676, 0, -544903190, -1831674681, 150502665, 474144502, 1428788237, 1428788237, 1428788237, 1428788237, 420913893, 1915664072)) {
+      assertColumnsAreEqual(expected, result);
+    }
+  }
+
+  @Test
+  void testSpark32BitMurmur3HashFloats() {
+    try (ColumnVector v = ColumnVector.fromBoxedFloats(
+          0f, 100f, -100f, Float.MIN_NORMAL, Float.MAX_VALUE, null,
+          POSITIVE_FLOAT_NAN_LOWER_RANGE, POSITIVE_FLOAT_NAN_UPPER_RANGE,
+          NEGATIVE_FLOAT_NAN_LOWER_RANGE, NEGATIVE_FLOAT_NAN_UPPER_RANGE,
+          Float.POSITIVE_INFINITY, Float.NEGATIVE_INFINITY);
+         ColumnVector result = ColumnVector.spark32BitMurmurHash3(411, new ColumnVector[]{v});
+         ColumnVector expected = ColumnVector.fromBoxedInts(-235179434, 1812056886, 2028471189, 1775092689, -1531511762, 411, -1053523253, -1053523253, -1053523253, -1053523253, -1526256646, 930080402)){
+      assertColumnsAreEqual(expected, result);
+    }
+  }
+
+  @Test
+  void testSpark32BitMurmur3HashBools() {
+    try (ColumnVector v0 = ColumnVector.fromBoxedBooleans(null, true, false, true, null, false);
+         ColumnVector v1 = ColumnVector.fromBoxedBooleans(null, true, false, null, false, true);
+         ColumnVector result = ColumnVector.spark32BitMurmurHash3(0, new ColumnVector[]{v0, v1});
+         ColumnVector expected = ColumnVector.fromBoxedInts(0, -1589400010, -239939054, -68075478, 593689054, -1194558265)) {
+      assertColumnsAreEqual(expected, result);
+
+    }
+  }
+
+  @Test
+  void testSpark32BitMurmur3HashMixed() {
+    try (ColumnVector strings = ColumnVector.fromStrings(
+          "a", "B\n", "dE\"\u0100\t\u0101 \ud720\ud721",
+          "A very long (greater than 128 bytes/char string) to test a multi hash-step data point " +
+          "in the MD5 hash function. This string needed to be longer.",
+          null, null);
+         ColumnVector integers = ColumnVector.fromBoxedInts(0, 100, -100, Integer.MIN_VALUE, Integer.MAX_VALUE, null);
+         ColumnVector doubles = ColumnVector.fromBoxedDoubles(
+          0.0, 100.0, -100.0, POSITIVE_DOUBLE_NAN_LOWER_RANGE, POSITIVE_DOUBLE_NAN_UPPER_RANGE, null);
+         ColumnVector floats = ColumnVector.fromBoxedFloats(
+          0f, 100f, -100f, NEGATIVE_FLOAT_NAN_LOWER_RANGE, NEGATIVE_FLOAT_NAN_UPPER_RANGE, null);
+         ColumnVector bools = ColumnVector.fromBoxedBooleans(true, false, null, false, true, null);
+         ColumnVector result = ColumnVector.spark32BitMurmurHash3(1868, new ColumnVector[]{strings, integers, doubles, floats, bools});
+         ColumnVector expected = ColumnVector.fromBoxedInts(1936985022, 720652989, 339312041, 1400354989, 769988643, 1868)) {
       assertColumnsAreEqual(expected, result);
     }
   }
@@ -511,7 +590,7 @@ void isNotNullTestEmptyColumn() {
     try (ColumnVector v = ColumnVector.fromBoxedInts();
          ColumnVector expected = ColumnVector.fromBoxedBooleans();
          ColumnVector result = v.isNotNull()) {
-      assertColumnsAreEqual(result, expected);
+      assertColumnsAreEqual(expected, result);
     }
   }
 
@@ -520,7 +599,7 @@ void isNotNullTest() {
     try (ColumnVector v = ColumnVector.fromBoxedInts(1, 2, null, 4, null, 6);
          ColumnVector expected = ColumnVector.fromBoxedBooleans(true, true, false, true, false, true);
          ColumnVector result = v.isNotNull()) {
-      assertColumnsAreEqual(result, expected);
+      assertColumnsAreEqual(expected, result);
     }
   }
 
@@ -529,7 +608,7 @@ void isNotNullTestAllNulls() {
     try (ColumnVector v = ColumnVector.fromBoxedInts(null, null, null, null, null, null);
          ColumnVector expected = ColumnVector.fromBoxedBooleans(false, false, false, false, false, false);
          ColumnVector result = v.isNotNull()) {
-      assertColumnsAreEqual(result, expected);
+      assertColumnsAreEqual(expected, result);
     }
   }
 
@@ -1232,7 +1311,7 @@ void testReplaceEmptyColumn() {
          ColumnVector expected = ColumnVector.fromBoxedBooleans();
          Scalar s = Scalar.fromBool(false);
          ColumnVector result = input.replaceNulls(s)) {
-      assertColumnsAreEqual(result, expected);
+      assertColumnsAreEqual(expected, result);
     }
   }
 
@@ -1242,7 +1321,7 @@ void testReplaceNullBoolsWithAllNulls() {
          ColumnVector expected = ColumnVector.fromBoxedBooleans(false, false, false, false);
          Scalar s = Scalar.fromBool(false);
          ColumnVector result = input.replaceNulls(s)) {
-      assertColumnsAreEqual(result, expected);
+      assertColumnsAreEqual(expected, result);
     }
   }
 
@@ -1252,7 +1331,7 @@ void testReplaceSomeNullBools() {
          ColumnVector expected = ColumnVector.fromBoxedBooleans(false, true, true, false);
          Scalar s = Scalar.fromBool(true);
          ColumnVector result = input.replaceNulls(s)) {
-      assertColumnsAreEqual(result, expected);
+      assertColumnsAreEqual(expected, result);
     }
   }
 
@@ -1262,7 +1341,7 @@ void testReplaceNullIntegersWithAllNulls() {
          ColumnVector expected = ColumnVector.fromBoxedInts(0, 0, 0, 0);
          Scalar s = Scalar.fromInt(0);
          ColumnVector result = input.replaceNulls(s)) {
-      assertColumnsAreEqual(result, expected);
+      assertColumnsAreEqual(expected, result);
     }
   }
 
@@ -1272,7 +1351,7 @@ void testReplaceSomeNullIntegers() {
          ColumnVector expected = ColumnVector.fromBoxedInts(1, 2, 999, 4, 999);
          Scalar s = Scalar.fromInt(999);
          ColumnVector result = input.replaceNulls(s)) {
-      assertColumnsAreEqual(result, expected);
+      assertColumnsAreEqual(expected, result);
     }
   }
 
diff --git a/python/cudf/cudf/_lib/cpp/types.pxd b/python/cudf/cudf/_lib/cpp/types.pxd
index 86f4e03741d..cf86076f8d6 100644
--- a/python/cudf/cudf/_lib/cpp/types.pxd
+++ b/python/cudf/cudf/_lib/cpp/types.pxd
@@ -79,6 +79,7 @@ cdef extern from "cudf/types.hpp" namespace "cudf" nogil:
         HASH_MURMUR3 "cudf::hash_id::HASH_MURMUR3"
         HASH_MD5 "cudf::hash_id::HASH_MD5"
         HASH_SERIAL_MURMUR3 "cudf::hash_id::HASH_SERIAL_MURMUR3"
+        HASH_SPARK_MURMUR3 "cudf::hash_id::HASH_SPARK_MURMUR3"
 
     cdef cppclass data_type:
         data_type() except +

From d641688f1857a366421514274264d9cee13c07e2 Mon Sep 17 00:00:00 2001
From: Rong Ou <rong.ou@gmail.com>
Date: Mon, 4 Jan 2021 15:37:25 -0800
Subject: [PATCH 02/20] Upgrade nvcomp to 1.2.1(#7069)

This version is more friendly to ccache:
```console
ccache -C  # clear the cache
time mvn clean package -DskipTests

real	4m43.015s
user	11m18.426s
sys	0m21.891s

time mvn clean package -DskipTests  # everything is now cached

real	0m20.265s
user	0m45.810s
sys	0m3.670s
```

Not sure about the ABI flag, but leaving it in causes the .so to not load:
```console
/usr/lib/jvm/java-8-openjdk-amd64/jre/bin/java: symbol lookup error: /tmp/nvcomp5478764208255606671.so: undefined symbol: _ZN6nvcomp5Check8not_nullEPKvRKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEESA_i
```

@jlowe

Authors:
  - Rong Ou <rong.ou@gmail.com>

Approvers:
  - Jason Lowe

URL: https://github.com/rapidsai/cudf/pull/7069
---
 .../main/native/cmake/Modules/ConfigureNvcomp.cmake    | 10 ----------
 .../native/cmake/Templates/Nvcomp.CMakeLists.txt.cmake |  2 +-
 2 files changed, 1 insertion(+), 11 deletions(-)

diff --git a/java/src/main/native/cmake/Modules/ConfigureNvcomp.cmake b/java/src/main/native/cmake/Modules/ConfigureNvcomp.cmake
index aa30420c003..0bef79116af 100644
--- a/java/src/main/native/cmake/Modules/ConfigureNvcomp.cmake
+++ b/java/src/main/native/cmake/Modules/ConfigureNvcomp.cmake
@@ -18,16 +18,6 @@ set(NVCOMP_ROOT "${CMAKE_BINARY_DIR}/nvcomp")
 
 set(NVCOMP_CMAKE_ARGS "-DUSE_RMM=ON -DCUB_DIR=${CUB_INCLUDE}")
 
-if(NOT CMAKE_CXX11_ABI)
-    message(STATUS "NVCOMP: Disabling the GLIBCXX11 ABI")
-    list(APPEND NVCOMP_CMAKE_ARGS " -DCMAKE_C_FLAGS=-D_GLIBCXX_USE_CXX11_ABI=0")
-    list(APPEND NVCOMP_CMAKE_ARGS " -DCMAKE_CXX_FLAGS=-D_GLIBCXX_USE_CXX11_ABI=0")
-elseif(CMAKE_CXX11_ABI)
-    message(STATUS "NVCOMP: Enabling the GLIBCXX11 ABI")
-    list(APPEND NVCOMP_CMAKE_ARGS " -DCMAKE_C_FLAGS=-D_GLIBCXX_USE_CXX11_ABI=1")
-    list(APPEND NVCOMP_CMAKE_ARGS " -DCMAKE_CXX_FLAGS=-D_GLIBCXX_USE_CXX11_ABI=1")
-endif(NOT CMAKE_CXX11_ABI)
-
 configure_file("${CMAKE_SOURCE_DIR}/cmake/Templates/Nvcomp.CMakeLists.txt.cmake"
                "${NVCOMP_ROOT}/CMakeLists.txt")
 
diff --git a/java/src/main/native/cmake/Templates/Nvcomp.CMakeLists.txt.cmake b/java/src/main/native/cmake/Templates/Nvcomp.CMakeLists.txt.cmake
index f244df84058..dca0a174e54 100644
--- a/java/src/main/native/cmake/Templates/Nvcomp.CMakeLists.txt.cmake
+++ b/java/src/main/native/cmake/Templates/Nvcomp.CMakeLists.txt.cmake
@@ -22,7 +22,7 @@ include(ExternalProject)
 
 ExternalProject_Add(nvcomp
     GIT_REPOSITORY  https://github.com/NVIDIA/nvcomp.git
-    GIT_TAG         v1.1.0
+    GIT_TAG         v1.2.1
     GIT_SHALLOW     true
     SOURCE_DIR      "${NVCOMP_ROOT}/nvcomp"
     BINARY_DIR      "${NVCOMP_ROOT}/build"

From 31c0d29a37f5073cdc8a6f78307d6ddfbb1f7144 Mon Sep 17 00:00:00 2001
From: Mike Wilson <hyperbolic2346@users.noreply.github.com>
Date: Mon, 4 Jan 2021 19:52:25 -0500
Subject: [PATCH 03/20] Adding decimal writing support to parquet(#7017)

Since cudf doesn't support precision, the precision must be passed in as a write option. This is handled as a vector of uint8's that indicates the precision of each flattened column in order to support nested types.

Partially closes #6474

Authors:
  - Mike Wilson <knobby@burntsheep.com>
  - Mike Wilson <hyperbolic2346@users.noreply.github.com>

Approvers:
  - Vukasin Milovanovic
  - Mark Harris

URL: https://github.com/rapidsai/cudf/pull/7017
---
 cpp/include/cudf/fixed_point/fixed_point.hpp | 18 ++--
 cpp/include/cudf/io/detail/parquet.hpp       | 10 +-
 cpp/include/cudf/io/parquet.hpp              | 28 ++++++
 cpp/include/cudf_test/column_wrapper.hpp     |  4 +-
 cpp/src/io/functions.cpp                     |  7 +-
 cpp/src/io/parquet/chunked_state.hpp         | 15 ++-
 cpp/src/io/parquet/writer_impl.cu            | 56 ++++++++++--
 cpp/src/io/parquet/writer_impl.hpp           |  2 +-
 cpp/tests/io/parquet_test.cpp                | 96 +++++++++++++++++++-
 9 files changed, 205 insertions(+), 31 deletions(-)

diff --git a/cpp/include/cudf/fixed_point/fixed_point.hpp b/cpp/include/cudf/fixed_point/fixed_point.hpp
index c8f9a73a1d6..d71f7ce005f 100644
--- a/cpp/include/cudf/fixed_point/fixed_point.hpp
+++ b/cpp/include/cudf/fixed_point/fixed_point.hpp
@@ -520,12 +520,18 @@ class fixed_point {
    */
   explicit operator std::string() const
   {
-    int const n          = std::pow(10, -_scale);
-    int const f          = _value % n;
-    auto const num_zeros = std::max(0, (-_scale - static_cast<int32_t>(std::to_string(f).size())));
-    auto const zeros     = num_zeros <= 0 ? std::string("") : std::string(num_zeros, '0');
-    return std::to_string(_value / n) + std::string(".") + zeros +
-           std::to_string(std::abs(_value) % n);
+    if (_scale < 0) {
+      int const n = std::pow(10, -_scale);
+      int const f = _value % n;
+      auto const num_zeros =
+        std::max(0, (-_scale - static_cast<int32_t>(std::to_string(f).size())));
+      auto const zeros = std::string(num_zeros, '0');
+      return std::to_string(_value / n) + std::string(".") + zeros +
+             std::to_string(std::abs(_value) % n);
+    } else {
+      auto const zeros = std::string(_scale, '0');
+      return std::to_string(_value) + zeros;
+    }
   }
 };  // namespace numeric
 
diff --git a/cpp/include/cudf/io/detail/parquet.hpp b/cpp/include/cudf/io/detail/parquet.hpp
index 1769c72e1c8..163d8c9d735 100644
--- a/cpp/include/cudf/io/detail/parquet.hpp
+++ b/cpp/include/cudf/io/detail/parquet.hpp
@@ -113,11 +113,11 @@ class writer {
    */
   std::unique_ptr<std::vector<uint8_t>> write(
     table_view const& table,
-    const table_metadata* metadata            = nullptr,
-    bool return_filemetadata                  = false,
-    const std::string column_chunks_file_path = "",
-    bool int96_timestamps                     = false,
-    rmm::cuda_stream_view stream              = rmm::cuda_stream_default);
+    const table_metadata* metadata                = nullptr,
+    bool return_filemetadata                      = false,
+    const std::string column_chunks_file_path     = "",
+    std::vector<uint8_t> const& decimal_precision = {},
+    rmm::cuda_stream_view stream                  = rmm::cuda_stream_default);
 
   /**
    * @brief Begins the chunked/streamed write process.
diff --git a/cpp/include/cudf/io/parquet.hpp b/cpp/include/cudf/io/parquet.hpp
index e6eebec42d8..177c2da8a9d 100644
--- a/cpp/include/cudf/io/parquet.hpp
+++ b/cpp/include/cudf/io/parquet.hpp
@@ -401,6 +401,9 @@ class parquet_writer_options {
   bool _write_timestamps_as_int96 = false;
   // Column chunks file path to be set in the raw output metadata
   std::string _column_chunks_file_path;
+  /// vector of precision values for decimal writing. Exactly one entry
+  /// per decimal column. Optional unless decimals are being written.
+  std::vector<uint8_t> _decimal_precision;
 
   /**
    * @brief Constructor from sink and table.
@@ -480,6 +483,11 @@ class parquet_writer_options {
    */
   std::string get_column_chunks_file_path() const { return _column_chunks_file_path; }
 
+  /**
+   * @brief Returns a constant reference to the decimal precision vector.
+   */
+  std::vector<uint8_t> const& get_decimal_precision() const { return _decimal_precision; }
+
   /**
    * @brief Sets metadata.
    *
@@ -525,6 +533,11 @@ class parquet_writer_options {
   {
     _column_chunks_file_path.assign(file_path);
   }
+
+  /**
+   * @brief Sets the decimal precision vector data.
+   */
+  void set_decimal_precision(std::vector<uint8_t> dp) { _decimal_precision = std::move(dp); }
 };
 
 class parquet_writer_options_builder {
@@ -687,6 +700,8 @@ class chunked_parquet_writer_options {
   const table_metadata_with_nullability* _nullable_metadata = nullptr;
   // Parquet writes can write INT96 or TIMESTAMP_MICROS. Defaults to TIMESTAMP_MICROS.
   bool _write_timestamps_as_int96 = false;
+  // Optional decimal precision data - must be present if writing decimals
+  std::vector<uint8_t> _decimal_precision = {};
 
   /**
    * @brief Constructor from sink.
@@ -728,6 +743,11 @@ class chunked_parquet_writer_options {
     return _nullable_metadata;
   }
 
+  /**
+   * @brief Returns decimal precision pointer.
+   */
+  std::vector<uint8_t> const& get_decimal_precision() const { return _decimal_precision; }
+
   /**
    * @brief Returns `true` if timestamps will be written as INT96
    */
@@ -743,6 +763,14 @@ class chunked_parquet_writer_options {
     _nullable_metadata = metadata;
   }
 
+  /**
+   * @brief Sets decimal precision data.
+   *
+   * @param v Vector of precision data flattened with exactly one entry per
+   *          decimal column.
+   */
+  void set_decimal_precision_data(std::vector<uint8_t> const& v) { _decimal_precision = v; }
+
   /**
    * @brief Sets the level of statistics in parquet_writer_options.
    *
diff --git a/cpp/include/cudf_test/column_wrapper.hpp b/cpp/include/cudf_test/column_wrapper.hpp
index a97f4d042ce..54a51f08798 100644
--- a/cpp/include/cudf_test/column_wrapper.hpp
+++ b/cpp/include/cudf_test/column_wrapper.hpp
@@ -511,9 +511,9 @@ class fixed_point_column_wrapper : public detail::column_wrapper {
    *
    * Example:
    * @code{.cpp}
-   * // Creates a non-nullable column of INT32 elements with 5 elements: {0, 2, 4, 6, 8}
+   * // Creates a non-nullable column of DECIMAL32 elements with 5 elements: {0, 2, 4, 6, 8}
    * auto elements = make_counting_transform_iterator(0, [](auto i) { return i * 2;});
-   * auto w = fixed_width_column_wrapper<int32_t>(elements, elements + 5, scale_type{0});
+   * auto w = fixed_point_column_wrapper<int32_t>(elements, elements + 5, scale_type{0});
    * @endcode
    *
    * @tparam FixedPointRepIterator Iterator for fixed_point::rep
diff --git a/cpp/src/io/functions.cpp b/cpp/src/io/functions.cpp
index a4504e12c2f..69dd69e5505 100644
--- a/cpp/src/io/functions.cpp
+++ b/cpp/src/io/functions.cpp
@@ -360,7 +360,7 @@ std::unique_ptr<std::vector<uint8_t>> write_parquet(parquet_writer_options const
                        options.get_metadata(),
                        options.is_enabled_return_filemetadata(),
                        options.get_column_chunks_file_path(),
-                       options.is_enabled_int96_timestamps());
+                       options.get_decimal_precision());
 }
 
 /**
@@ -394,8 +394,9 @@ std::shared_ptr<pq_chunked_state> write_parquet_chunked_begin(
     state->user_metadata_with_nullability = *op.get_nullable_metadata();
     state->user_metadata                  = &state->user_metadata_with_nullability;
   }
-  state->int96_timestamps = op.is_enabled_int96_timestamps();
-  state->stream           = 0;
+  state->int96_timestamps   = op.is_enabled_int96_timestamps();
+  state->_decimal_precision = op.get_decimal_precision();
+  state->stream             = 0;
   state->wp->write_chunked_begin(*state);
   return state;
 }
diff --git a/cpp/src/io/parquet/chunked_state.hpp b/cpp/src/io/parquet/chunked_state.hpp
index 5bbc5366f70..d6758efe417 100644
--- a/cpp/src/io/parquet/chunked_state.hpp
+++ b/cpp/src/io/parquet/chunked_state.hpp
@@ -53,19 +53,24 @@ struct pq_chunked_state {
   /// only used in the write_chunked() case. copied from the (optionally) user supplied
   /// argument to write_parquet_chunked_begin()
   bool single_write_mode;
-  ///  timestamps should be written as int96 types
+  /// timestamps should be written as int96 types
   bool int96_timestamps;
+  /// vector of precision values for decimal writing. Exactly one entry
+  /// per decimal column.
+  std::vector<uint8_t> _decimal_precision;
 
   pq_chunked_state() = default;
 
   pq_chunked_state(table_metadata const* metadata,
-                   SingleWriteMode mode         = SingleWriteMode::NO,
-                   bool write_int96_timestamps  = false,
-                   rmm::cuda_stream_view stream = rmm::cuda_stream_default)
+                   SingleWriteMode mode                          = SingleWriteMode::NO,
+                   bool write_int96_timestamps                   = false,
+                   std::vector<uint8_t> const& decimal_precision = {},
+                   rmm::cuda_stream_view stream                  = rmm::cuda_stream_default)
     : stream{stream},
       user_metadata{metadata},
       single_write_mode{mode == SingleWriteMode::YES},
-      int96_timestamps(write_int96_timestamps)
+      int96_timestamps(write_int96_timestamps),
+      _decimal_precision(decimal_precision)
   {
   }
 };
diff --git a/cpp/src/io/parquet/writer_impl.cu b/cpp/src/io/parquet/writer_impl.cu
index afb7489bfa9..f84145a44c7 100644
--- a/cpp/src/io/parquet/writer_impl.cu
+++ b/cpp/src/io/parquet/writer_impl.cu
@@ -164,6 +164,8 @@ class parquet_column_view {
                                std::vector<bool> const &nullability,
                                const table_metadata *metadata,
                                bool int96_timestamps,
+                               std::vector<uint8_t> const &decimal_precision,
+                               uint &decimal_precision_idx,
                                rmm::cuda_stream_view stream)
     : _col(col),
       _leaf_col(get_leaf_col(col)),
@@ -295,6 +297,28 @@ class parquet_column_view {
         _converted_type = ConvertedType::UTF8;
         _stats_dtype    = statistics_dtype::dtype_string;
         break;
+      case cudf::type_id::DECIMAL32:
+        _physical_type  = Type::INT32;
+        _converted_type = ConvertedType::DECIMAL;
+        _stats_dtype    = statistics_dtype::dtype_int32;
+        _decimal_scale  = -_leaf_col.type().scale();  // parquet and cudf disagree about scale signs
+        CUDF_EXPECTS(decimal_precision.size() > decimal_precision_idx,
+                     "Not enough decimal precision values passed for data!");
+        CUDF_EXPECTS(decimal_precision[decimal_precision_idx] > _decimal_scale,
+                     "Precision must be greater than scale!");
+        _decimal_precision = decimal_precision[decimal_precision_idx++];
+        break;
+      case cudf::type_id::DECIMAL64:
+        _physical_type  = Type::INT64;
+        _converted_type = ConvertedType::DECIMAL;
+        _stats_dtype    = statistics_dtype::dtype_decimal64;
+        _decimal_scale  = -_leaf_col.type().scale();  // parquet and cudf disagree about scale signs
+        CUDF_EXPECTS(decimal_precision.size() > decimal_precision_idx,
+                     "Not enough decimal precision values passed for data!");
+        CUDF_EXPECTS(decimal_precision[decimal_precision_idx] > _decimal_scale,
+                     "Precision must be greater than scale!");
+        _decimal_precision = decimal_precision[decimal_precision_idx++];
+        break;
       default:
         _physical_type = UNDEFINED_TYPE;
         _stats_dtype   = dtype_none;
@@ -381,6 +405,8 @@ class parquet_column_view {
   uint32_t const *nulls() const noexcept { return _nulls; }
   size_type offset() const noexcept { return _offset; }
   bool level_nullable(size_t level) const { return _nullability[level]; }
+  int32_t decimal_scale() const noexcept { return _decimal_scale; }
+  uint8_t decimal_precision() const noexcept { return _decimal_precision; }
 
   // List related data
   column_view cudf_col() const noexcept { return _col; }
@@ -466,6 +492,10 @@ class parquet_column_view {
 
   // String-related members
   rmm::device_buffer _indexes;
+
+  // Decimal-related members
+  int32_t _decimal_scale     = 0;
+  uint8_t _decimal_precision = 0;
 };
 
 void writer::impl::init_page_fragments(hostdevice_vector<gpu::PageFragment> &frag,
@@ -656,10 +686,11 @@ std::unique_ptr<std::vector<uint8_t>> writer::impl::write(
   const table_metadata *metadata,
   bool return_filemetadata,
   const std::string &column_chunks_file_path,
-  bool int96_timestamps,
+  std::vector<uint8_t> const &decimal_precisions,
   rmm::cuda_stream_view stream)
 {
-  pq_chunked_state state{metadata, SingleWriteMode::YES, int96_timestamps, stream};
+  pq_chunked_state state{
+    metadata, SingleWriteMode::YES, int96_timestamps, decimal_precisions, stream};
 
   write_chunked_begin(state);
   write_chunk(table, state);
@@ -697,6 +728,8 @@ void writer::impl::write_chunk(table_view const &table, pq_chunked_state &state)
       ? std::vector<std::vector<bool>>{}
       : get_per_column_nullability(table, state.user_metadata_with_nullability.column_nullable);
 
+  uint decimal_precision_idx = 0;
+
   for (auto it = table.begin(); it < table.end(); ++it) {
     const auto col        = *it;
     const auto current_id = parquet_columns.size();
@@ -714,9 +747,14 @@ void writer::impl::write_chunk(table_view const &table, pq_chunked_state &state)
                                  this_column_nullability,
                                  state.user_metadata,
                                  state.int96_timestamps,
+                                 state._decimal_precision,
+                                 decimal_precision_idx,
                                  state.stream);
   }
 
+  CUDF_EXPECTS(decimal_precision_idx == state._decimal_precision.size(),
+               "Too many decimal precision values!");
+
   // first call. setup metadata. num_rows will get incremented as write_chunk is
   // called multiple times.
   // Calculate the sum of depths of all list columns
@@ -767,7 +805,9 @@ void writer::impl::write_chunk(table_view const &table, pq_chunked_state &state)
         list_schema[nesting_depth * 2].type = physical_type;
         list_schema[nesting_depth * 2].converted_type =
           physical_type == parquet::Type::INT96 ? ConvertedType::UNKNOWN : col.converted_type();
-        list_schema[nesting_depth * 2].num_children = 0;
+        list_schema[nesting_depth * 2].num_children      = 0;
+        list_schema[nesting_depth * 2].decimal_precision = col.decimal_precision();
+        list_schema[nesting_depth * 2].decimal_scale     = col.decimal_scale();
 
         std::vector<std::string> path_in_schema;
         std::transform(
@@ -790,8 +830,10 @@ void writer::impl::write_chunk(table_view const &table, pq_chunked_state &state)
             ? OPTIONAL
             : REQUIRED;
 
-        col_schema.name         = col.name();
-        col_schema.num_children = 0;  // Leaf node
+        col_schema.name              = col.name();
+        col_schema.num_children      = 0;  // Leaf node
+        col_schema.decimal_precision = col.decimal_precision();
+        col_schema.decimal_scale     = col.decimal_scale();
 
         this_table_schema.push_back(std::move(col_schema));
       }
@@ -1225,11 +1267,11 @@ std::unique_ptr<std::vector<uint8_t>> writer::write(table_view const &table,
                                                     const table_metadata *metadata,
                                                     bool return_filemetadata,
                                                     const std::string column_chunks_file_path,
-                                                    bool int96_timestamps,
+                                                    std::vector<uint8_t> const &decimal_precisions,
                                                     rmm::cuda_stream_view stream)
 {
   return _impl->write(
-    table, metadata, return_filemetadata, column_chunks_file_path, int96_timestamps, stream);
+    table, metadata, return_filemetadata, column_chunks_file_path, decimal_precisions, stream);
 }
 
 // Forward to implementation
diff --git a/cpp/src/io/parquet/writer_impl.hpp b/cpp/src/io/parquet/writer_impl.hpp
index 7d9e4b92a36..b664926f970 100644
--- a/cpp/src/io/parquet/writer_impl.hpp
+++ b/cpp/src/io/parquet/writer_impl.hpp
@@ -88,7 +88,7 @@ class writer::impl {
                                               const table_metadata* metadata,
                                               bool return_filemetadata,
                                               const std::string& column_chunks_file_path,
-                                              bool int96_timestamps,
+                                              std::vector<uint8_t> const& decimal_precisions,
                                               rmm::cuda_stream_view stream);
 
   /**
diff --git a/cpp/tests/io/parquet_test.cpp b/cpp/tests/io/parquet_test.cpp
index ada2eadaa31..89cbb82c726 100644
--- a/cpp/tests/io/parquet_test.cpp
+++ b/cpp/tests/io/parquet_test.cpp
@@ -16,6 +16,7 @@
 
 #include <cudf/concatenate.hpp>
 #include <cudf/copying.hpp>
+#include <cudf/fixed_point/fixed_point.hpp>
 #include <cudf/io/data_sink.hpp>
 #include <cudf/io/parquet.hpp>
 #include <cudf/strings/string_view.cuh>
@@ -286,6 +287,14 @@ TEST_F(ParquetWriterTest, MultiColumn)
   auto col3_data = random_values<int32_t>(num_rows);
   auto col4_data = random_values<float>(num_rows);
   auto col5_data = random_values<double>(num_rows);
+  auto col6_vals = random_values<int32_t>(num_rows);
+  auto col7_vals = random_values<int64_t>(num_rows);
+  auto col6_data = cudf::test::make_counting_transform_iterator(0, [col6_vals](auto i) {
+    return numeric::decimal32{col6_vals[i], numeric::scale_type{5}};
+  });
+  auto col7_data = cudf::test::make_counting_transform_iterator(0, [col7_vals](auto i) {
+    return numeric::decimal64{col7_vals[i], numeric::scale_type{-5}};
+  });
   auto validity  = cudf::test::make_counting_transform_iterator(0, [](auto i) { return true; });
 
   // column_wrapper<bool> col0{
@@ -295,6 +304,8 @@ TEST_F(ParquetWriterTest, MultiColumn)
   column_wrapper<int32_t> col3{col3_data.begin(), col3_data.end(), validity};
   column_wrapper<float> col4{col4_data.begin(), col4_data.end(), validity};
   column_wrapper<double> col5{col5_data.begin(), col5_data.end(), validity};
+  column_wrapper<numeric::decimal32> col6{col6_data, col6_data + num_rows, validity};
+  column_wrapper<numeric::decimal64> col7{col7_data, col7_data + num_rows, validity};
 
   cudf_io::table_metadata expected_metadata;
   // expected_metadata.column_names.emplace_back("bools");
@@ -303,6 +314,8 @@ TEST_F(ParquetWriterTest, MultiColumn)
   expected_metadata.column_names.emplace_back("int32s");
   expected_metadata.column_names.emplace_back("floats");
   expected_metadata.column_names.emplace_back("doubles");
+  expected_metadata.column_names.emplace_back("decimal32s");
+  expected_metadata.column_names.emplace_back("decimal64s");
 
   std::vector<std::unique_ptr<column>> cols;
   // cols.push_back(col0.release());
@@ -311,13 +324,17 @@ TEST_F(ParquetWriterTest, MultiColumn)
   cols.push_back(col3.release());
   cols.push_back(col4.release());
   cols.push_back(col5.release());
+  cols.push_back(col6.release());
+  cols.push_back(col7.release());
   auto expected = std::make_unique<table>(std::move(cols));
-  EXPECT_EQ(5, expected->num_columns());
+  EXPECT_EQ(7, expected->num_columns());
 
   auto filepath = temp_env->get_temp_filepath("MultiColumn.parquet");
   cudf_io::parquet_writer_options out_opts =
     cudf_io::parquet_writer_options::builder(cudf_io::sink_info{filepath}, expected->view())
       .metadata(&expected_metadata);
+  std::vector<uint8_t> precisions = {10, 20};
+  out_opts.set_decimal_precision(precisions);
   cudf_io::write_parquet(out_opts);
 
   cudf_io::parquet_reader_options in_opts =
@@ -338,6 +355,14 @@ TEST_F(ParquetWriterTest, MultiColumnWithNulls)
   auto col3_data = random_values<int32_t>(num_rows);
   auto col4_data = random_values<float>(num_rows);
   auto col5_data = random_values<double>(num_rows);
+  auto col6_vals = random_values<int32_t>(num_rows);
+  auto col7_vals = random_values<int64_t>(num_rows);
+  auto col6_data = cudf::test::make_counting_transform_iterator(0, [col6_vals](auto i) {
+    return numeric::decimal32{col6_vals[i], numeric::scale_type{-2}};
+  });
+  auto col7_data = cudf::test::make_counting_transform_iterator(0, [col7_vals](auto i) {
+    return numeric::decimal64{col7_vals[i], numeric::scale_type{-8}};
+  });
   // auto col0_mask = cudf::test::make_counting_transform_iterator(
   //    0, [](auto i) { return (i % 2); });
   auto col1_mask = cudf::test::make_counting_transform_iterator(0, [](auto i) { return (i < 10); });
@@ -347,6 +372,9 @@ TEST_F(ParquetWriterTest, MultiColumnWithNulls)
   auto col4_mask =
     cudf::test::make_counting_transform_iterator(0, [](auto i) { return (i >= 40 || i <= 60); });
   auto col5_mask = cudf::test::make_counting_transform_iterator(0, [](auto i) { return (i > 80); });
+  auto col6_mask = cudf::test::make_counting_transform_iterator(0, [](auto i) { return (i % 5); });
+  auto col7_mask =
+    cudf::test::make_counting_transform_iterator(0, [](auto i) { return (i != 55); });
 
   // column_wrapper<bool> col0{
   //    col0_data.begin(), col0_data.end(), col0_mask};
@@ -355,6 +383,8 @@ TEST_F(ParquetWriterTest, MultiColumnWithNulls)
   column_wrapper<int32_t> col3{col3_data.begin(), col3_data.end(), col3_mask};
   column_wrapper<float> col4{col4_data.begin(), col4_data.end(), col4_mask};
   column_wrapper<double> col5{col5_data.begin(), col5_data.end(), col5_mask};
+  column_wrapper<numeric::decimal32> col6{col6_data, col6_data + num_rows, col6_mask};
+  column_wrapper<numeric::decimal64> col7{col7_data, col7_data + num_rows, col7_mask};
 
   cudf_io::table_metadata expected_metadata;
   // expected_metadata.column_names.emplace_back("bools");
@@ -363,6 +393,8 @@ TEST_F(ParquetWriterTest, MultiColumnWithNulls)
   expected_metadata.column_names.emplace_back("int32s");
   expected_metadata.column_names.emplace_back("floats");
   expected_metadata.column_names.emplace_back("doubles");
+  expected_metadata.column_names.emplace_back("decimal32s");
+  expected_metadata.column_names.emplace_back("decimal64s");
 
   std::vector<std::unique_ptr<column>> cols;
   // cols.push_back(col0.release());
@@ -371,13 +403,18 @@ TEST_F(ParquetWriterTest, MultiColumnWithNulls)
   cols.push_back(col3.release());
   cols.push_back(col4.release());
   cols.push_back(col5.release());
+  cols.push_back(col6.release());
+  cols.push_back(col7.release());
   auto expected = std::make_unique<table>(std::move(cols));
-  EXPECT_EQ(5, expected->num_columns());
+  EXPECT_EQ(7, expected->num_columns());
 
   auto filepath = temp_env->get_temp_filepath("MultiColumnWithNulls.parquet");
   cudf_io::parquet_writer_options out_opts =
     cudf_io::parquet_writer_options::builder(cudf_io::sink_info{filepath}, expected->view())
       .metadata(&expected_metadata);
+  std::vector<uint8_t> precisions = {9, 20};
+  out_opts.set_decimal_precision(precisions);
+
   cudf_io::write_parquet(out_opts);
 
   cudf_io::parquet_reader_options in_opts =
@@ -1282,6 +1319,61 @@ TEST_F(ParquetChunkedWriterTest, ReadRowGroupsError)
   EXPECT_THROW(cudf_io::read_parquet(read_opts), cudf::logic_error);
 }
 
+TEST_F(ParquetChunkedWriterTest, DecimalWrite)
+{
+  constexpr cudf::size_type num_rows = 500;
+  auto seq_col0                      = random_values<int32_t>(num_rows);
+  auto seq_col1                      = random_values<int64_t>(num_rows);
+
+  auto valids = cudf::test::make_counting_transform_iterator(0, [](auto i) { return i % 2 == 0; });
+
+  auto col0 = cudf::test::fixed_point_column_wrapper<int32_t>{
+    seq_col0.begin(), seq_col0.end(), valids, numeric::scale_type{5}};
+  auto col1 = cudf::test::fixed_point_column_wrapper<int64_t>{
+    seq_col1.begin(), seq_col1.end(), valids, numeric::scale_type{-9}};
+
+  auto table = table_view({col0, col1});
+
+  auto filepath = temp_env->get_temp_filepath("DecimalWrite.parquet");
+  cudf_io::chunked_parquet_writer_options args =
+    cudf_io::chunked_parquet_writer_options::builder(cudf_io::sink_info{filepath});
+
+  // verify failure if no decimal precision given
+  auto state = cudf_io::write_parquet_chunked_begin(args);
+  EXPECT_THROW(cudf_io::write_parquet_chunked(table, state), cudf::logic_error);
+
+  // verify failure if too small a precision is given
+  std::vector<uint8_t> precisions{7, 1};
+  args.set_decimal_precision_data(precisions);
+  state = cudf_io::write_parquet_chunked_begin(args);
+  EXPECT_THROW(cudf_io::write_parquet_chunked(table, state), cudf::logic_error);
+
+  // verify failure if too few precisions given
+  precisions.pop_back();
+  args.set_decimal_precision_data(precisions);
+  state = cudf_io::write_parquet_chunked_begin(args);
+  EXPECT_THROW(cudf_io::write_parquet_chunked(table, state), cudf::logic_error);
+
+  // verify failure if too many precisions given
+  precisions = {7, 14, 11};
+  args.set_decimal_precision_data(precisions);
+  state = cudf_io::write_parquet_chunked_begin(args);
+  EXPECT_THROW(cudf_io::write_parquet_chunked(table, state), cudf::logic_error);
+
+  // write correctly
+  precisions.pop_back();
+  args.set_decimal_precision_data(precisions);
+  state = cudf_io::write_parquet_chunked_begin(args);
+  cudf_io::write_parquet_chunked(table, state);
+  cudf_io::write_parquet_chunked_end(state);
+
+  cudf_io::parquet_reader_options read_opts =
+    cudf_io::parquet_reader_options::builder(cudf_io::source_info{filepath});
+  auto result = cudf_io::read_parquet(read_opts);
+
+  CUDF_TEST_EXPECT_TABLES_EQUAL(*result.tbl, table);
+}
+
 TYPED_TEST(ParquetChunkedWriterNumericTypeTest, UnalignedSize)
 {
   // write out two 31 row tables and make sure they get

From 6ebd264c5655de19150f4d93c97800424105dffe Mon Sep 17 00:00:00 2001
From: David <45795991+davidwendt@users.noreply.github.com>
Date: Tue, 5 Jan 2021 11:52:26 -0500
Subject: [PATCH 04/20] Add days check to cudf::is_timestamp using
 cuda::std::chrono classes(#7028)

Closes #6774

This PR adds a check for a valid day value for a year/month (if these are specified in the format) in the `cudf::is_timestamp()` API. Also, a chunk of messy year/month/day logic in a related functor was replaced with libcu++ implementation of the `year_month_day()` function instead.
A gtest is also updated to include to test for an invalid day.

Authors:
  - davidwendt <dwendt@nvidia.com>

Approvers:
  - Vukasin Milovanovic
  - Devavret Makkar
  - Karthikeyan
  - Jake Hemstad

URL: https://github.com/rapidsai/cudf/pull/7028
---
 .../cudf/strings/convert/convert_datetime.hpp |  4 +-
 cpp/src/strings/convert/convert_datetime.cu   | 91 +++++++++++++++----
 cpp/tests/strings/datetime_tests.cpp          | 24 ++++-
 3 files changed, 96 insertions(+), 23 deletions(-)

diff --git a/cpp/include/cudf/strings/convert/convert_datetime.hpp b/cpp/include/cudf/strings/convert/convert_datetime.hpp
index d65ab960982..39bd6c639aa 100644
--- a/cpp/include/cudf/strings/convert/convert_datetime.hpp
+++ b/cpp/include/cudf/strings/convert/convert_datetime.hpp
@@ -36,7 +36,7 @@ namespace strings {
  * | :-------: | ----------- |
  * | \%d | Day of the month: 01-31 |
  * | \%m | Month of the year: 01-12 |
- * | \%y | Year without century: 00-99 |
+ * | \%y | Year without century: 00-99. [0,68] maps to [2000,2068] and [69,99] maps to [1969,1999] |
  * | \%Y | Year with century: 0001-9999 |
  * | \%H | 24-hour of the day: 00-23 |
  * | \%I | 12-hour of the day: 01-12 |
@@ -84,7 +84,7 @@ std::unique_ptr<column> to_timestamps(
  * | :-------: | ----------- |
  * | \%d | Day of the month: 01-31 |
  * | \%m | Month of the year: 01-12 |
- * | \%y | Year without century: 00-99 |
+ * | \%y | Year without century: 00-99. [0,68] maps to [2000,2068] and [69,99] maps to [1969,1999] |
  * | \%Y | Year with century: 0001-9999 |
  * | \%H | 24-hour of the day: 00-23 |
  * | \%I | 12-hour of the day: 01-12 |
diff --git a/cpp/src/strings/convert/convert_datetime.cu b/cpp/src/strings/convert/convert_datetime.cu
index bc49b4768d0..91ec1e2de2d 100644
--- a/cpp/src/strings/convert/convert_datetime.cu
+++ b/cpp/src/strings/convert/convert_datetime.cu
@@ -229,7 +229,11 @@ struct parse_datetime {
       // special logic for each specifier
       switch (item.value) {
         case 'Y': timeparts[TP_YEAR] = str2int(ptr, item.length); break;
-        case 'y': timeparts[TP_YEAR] = str2int(ptr, item.length) + 1900; break;
+        case 'y': {
+          auto const year    = str2int(ptr, item.length);
+          timeparts[TP_YEAR] = year + (year < 69 ? 2000 : 1900);
+          break;
+        }
         case 'm': timeparts[TP_MONTH] = str2int(ptr, item.length); break;
         case 'd': timeparts[TP_DAY] = str2int(ptr, item.length); break;
         case 'j': timeparts[TP_DAY_OF_YEAR] = str2int(ptr, item.length); break;
@@ -280,18 +284,11 @@ struct parse_datetime {
     if (units == timestamp_units::months)
       return ((year - 1970) * 12) + (month - 1);  // months are 1-12, need to 0-base it here
     auto day = timeparts[TP_DAY];
-    // The months are shifted so that March is the starting month and February
-    // (possible leap day in it) is the last month for the linear calculation
-    year -= (month <= 2) ? 1 : 0;
-    // date cycle repeats every 400 years (era)
-    constexpr int32_t erasInDays  = 146097;
-    constexpr int32_t erasInYears = (erasInDays / 365);
-    auto era                      = (year >= 0 ? year : year - 399) / erasInYears;
-    auto yoe                      = year - era * erasInYears;
-    auto doy = month == 0 ? day : ((153 * (month + (month > 2 ? -3 : 9)) + 2) / 5 + day - 1);
-    auto doe = (yoe * 365) + (yoe / 4) - (yoe / 100) + doy;
-    int32_t days =
-      (era * erasInDays) + doe - 719468;  // 719468 = days from 0000-00-00 to 1970-03-01
+    auto ymd =  // convenient chrono class handles the leap year calculations for us
+      cuda::std::chrono::year_month_day(cuda::std::chrono::year{year},
+                                        cuda::std::chrono::month{static_cast<uint32_t>(month)},
+                                        cuda::std::chrono::day{static_cast<uint32_t>(day)});
+    int32_t days = cuda::std::chrono::sys_days(ymd).time_since_epoch().count();
     if (units == timestamp_units::days) return days;
 
     auto tzadjust = timeparts[TP_TZ_MINUTES];  // in minutes
@@ -456,6 +453,29 @@ struct check_datetime_format {
     });
   }
 
+  /**
+   * @brief Specialized function to return the value and check for non-decimal characters.
+   *
+   * If non-decimal characters are found within `str` and `str + bytes` then
+   * the returned result is `thrust::nullopt` (_does not contain a value_).
+   * Otherwise, the parsed integer result is returned.
+   *
+   * @param str Beginning of characters to read/check.
+   * @param bytes Number of bytes in str to read/check.
+   * @return Integer value if characters are valid.
+   */
+  __device__ thrust::optional<int32_t> str2int(const char* str, size_type bytes)
+  {
+    const char* ptr = str;
+    int32_t value   = 0;
+    for (size_type idx = 0; idx < bytes; ++idx) {
+      char chr = *ptr++;
+      if (chr < '0' || chr > '9') return thrust::nullopt;
+      value = (value * 10) + static_cast<int32_t>(chr - '0');
+    }
+    return value;
+  }
+
   /**
    * @brief Check the specified characters are between ['0','9']
    * and the resulting integer is within [`min_value`, `max_value`].
@@ -486,7 +506,7 @@ struct check_datetime_format {
    * The checking here is a little more strict than the actual
    * parser used for conversion.
    */
-  __device__ bool check_string(string_view const& d_string)
+  __device__ bool check_string(string_view const& d_string, int32_t* dateparts)
   {
     auto ptr    = d_string.data();
     auto length = d_string.size_bytes();
@@ -508,10 +528,35 @@ struct check_datetime_format {
       // reference: https://man7.org/linux/man-pages/man3/strptime.3.html
       bool result = false;
       switch (item.value) {
-        case 'Y': result = check_digits(ptr, item.length); break;
-        case 'y': result = check_digits(ptr, item.length); break;
-        case 'm': result = check_value(ptr, item.length, 1, 12); break;
-        case 'd': result = check_value(ptr, item.length, 1, 31); break;
+        case 'Y': {
+          if (auto value = str2int(ptr, item.length)) {
+            result             = true;
+            dateparts[TP_YEAR] = value.value();
+          }
+          break;
+        }
+        case 'y': {
+          if (auto value = str2int(ptr, item.length)) {
+            result             = true;
+            auto const year    = value.value();
+            dateparts[TP_YEAR] = year + (year < 69 ? 2000 : 1900);
+          }
+          break;
+        }
+        case 'm': {
+          if (auto value = str2int(ptr, item.length)) {
+            result              = true;
+            dateparts[TP_MONTH] = value.value();
+          }
+          break;
+        }
+        case 'd': {
+          if (auto value = str2int(ptr, item.length)) {
+            result            = true;
+            dateparts[TP_DAY] = value.value();
+          }
+          break;
+        }
         case 'j': result = check_value(ptr, item.length, 1, 366); break;
         case 'H': result = check_value(ptr, item.length, 0, 23); break;
         case 'I': result = check_value(ptr, item.length, 1, 12); break;
@@ -552,7 +597,15 @@ struct check_datetime_format {
     if (d_strings.is_null(idx)) return false;
     string_view d_str = d_strings.element<string_view>(idx);
     if (d_str.empty()) return false;
-    return check_string(d_str);
+    int32_t dateparts[] = {1970, 1, 1};  // year, month, day
+    if (!check_string(d_str, dateparts)) return false;
+    auto year  = dateparts[TP_YEAR];
+    auto month = static_cast<uint32_t>(dateparts[TP_MONTH]);
+    auto day   = static_cast<uint32_t>(dateparts[TP_DAY]);
+    return cuda::std::chrono::year_month_day(cuda::std::chrono::year{year},
+                                             cuda::std::chrono::month{month},
+                                             cuda::std::chrono::day{day})
+      .ok();
   }
 };
 
diff --git a/cpp/tests/strings/datetime_tests.cpp b/cpp/tests/strings/datetime_tests.cpp
index 2aa40461e56..a0f1eed9935 100644
--- a/cpp/tests/strings/datetime_tests.cpp
+++ b/cpp/tests/strings/datetime_tests.cpp
@@ -384,9 +384,29 @@ TEST_F(StringsDatetimeTest, IsTimestamp)
                                              "2020-10-32 01:32:03",
                                              "2020-10-07 25:02:03",
                                              "2020-10-07 01:62:03",
-                                             "2020-10-07 01:02:63"};
+                                             "2020-10-07 01:02:63",
+                                             "2020-02-29 01:32:03",
+                                             "2020-02-30 01:32:03",
+                                             "2020-00-31 01:32:03",
+                                             "2020-02-00 01:32:03"};
   auto strings_view = cudf::strings_column_view(strings);
   auto results      = cudf::strings::is_timestamp(strings_view, "%Y-%m-%d %H:%M:%S");
   CUDF_TEST_EXPECT_COLUMNS_EQUAL(
-    *results, cudf::test::fixed_width_column_wrapper<bool>{1, 0, 0, 0, 0, 0, 0, 0});
+    *results, cudf::test::fixed_width_column_wrapper<bool>{1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0});
+}
+
+TEST_F(StringsDatetimeTest, ToTimestampYear)
+{
+  cudf::test::strings_column_wrapper strings{
+    "28/02/74", "17/07/68", "20/03/19", "29/02/20", "07/02/69"};
+  auto strings_view = cudf::strings_column_view(strings);
+  auto results      = cudf::strings::to_timestamps(
+    strings_view, cudf::data_type{cudf::type_id::TIMESTAMP_DAYS}, "%d/%m/%y");
+  cudf::test::fixed_width_column_wrapper<cudf::timestamp_D, cudf::timestamp_D::rep> expected{
+    1519, 35992, 17975, 18321, -328};
+  CUDF_TEST_EXPECT_COLUMNS_EQUAL(*results, expected);
+
+  results = cudf::strings::is_timestamp(strings_view, "%d/%m/%y");
+  cudf::test::fixed_width_column_wrapper<bool> is_expected({1, 1, 1, 1, 1});
+  CUDF_TEST_EXPECT_COLUMNS_EQUAL(*results, is_expected);
 }

From 873ab4af165529c1811b7fc042341b0cc22f64a5 Mon Sep 17 00:00:00 2001
From: Ray Douglass <3107146+raydouglass@users.noreply.github.com>
Date: Tue, 5 Jan 2021 12:22:25 -0500
Subject: [PATCH 05/20] Only upload packages that were built(#7077)

Should only upload the packages that were actually built. Project Flash sets `BUILD_CUDF` and `BUILD_LIBCUDF` as needed to control this.

Skipping CI as this change only affects uploads which isn't tested by CI.

Authors:
  - Raymond Douglass <ray@raydouglass.com>

Approvers:
  - Dillon Cullinan

URL: https://github.com/rapidsai/cudf/pull/7077
---
 ci/cpu/upload.sh | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/ci/cpu/upload.sh b/ci/cpu/upload.sh
index d6fa2c399aa..0465197e861 100755
--- a/ci/cpu/upload.sh
+++ b/ci/cpu/upload.sh
@@ -40,14 +40,14 @@ export CUSTREAMZ_FILE=`conda build conda/recipes/custreamz --python=$PYTHON --ou
 ################################################################################
 
 gpuci_logger "Starting conda uploads"
-if [ "$UPLOAD_LIBCUDF" == "1" ]; then
+if [[ "$BUILD_LIBCUDF" == "1" && "$UPLOAD_LIBCUDF" == "1" ]]; then
   test -e ${LIBCUDF_FILE}
   echo "Upload libcudf"
   echo ${LIBCUDF_FILE}
   gpuci_retry anaconda -t ${MY_UPLOAD_KEY} upload -u ${CONDA_USERNAME:-rapidsai} ${LABEL_OPTION} --skip-existing ${LIBCUDF_FILE}
 fi
 
-if [ "$UPLOAD_CUDF" == "1" ]; then
+if [[ "$BUILD_CUDF" == "1" && "$UPLOAD_CUDF" == "1" ]]; then
   test -e ${CUDF_FILE}
   echo "Upload cudf"
   echo ${CUDF_FILE}
@@ -64,14 +64,14 @@ if [ "$UPLOAD_CUDF" == "1" ]; then
   gpuci_retry anaconda -t ${MY_UPLOAD_KEY} upload -u ${CONDA_USERNAME:-rapidsai} ${LABEL_OPTION} --skip-existing ${CUSTREAMZ_FILE}
 fi
 
-if [ "$UPLOAD_LIBCUDF_KAFKA" == "1" ]; then
+if [[ "$BUILD_LIBCUDF" == "1" && "$UPLOAD_LIBCUDF_KAFKA" == "1" ]]; then
   test -e ${LIBCUDF_KAFKA_FILE}
   echo "Upload libcudf_kafka"
   echo ${LIBCUDF_KAFKA_FILE}
   gpuci_retry anaconda -t ${MY_UPLOAD_KEY} upload -u ${CONDA_USERNAME:-rapidsai} ${LABEL_OPTION} --skip-existing ${LIBCUDF_KAFKA_FILE}
 fi
 
-if [ "$UPLOAD_CUDF_KAFKA" == "1" ]; then
+if [[ "$BUILD_CUDF" == "1" && "$UPLOAD_CUDF_KAFKA" == "1" ]]; then
   test -e ${CUDF_KAFKA_FILE}
   echo "Upload cudf_kafka"
   echo ${CUDF_KAFKA_FILE}

From 91322ba7d772446151bce3e729e5e1dc61854127 Mon Sep 17 00:00:00 2001
From: Conor Hoekstra <36027403+codereport@users.noreply.github.com>
Date: Tue, 5 Jan 2021 13:22:25 -0500
Subject: [PATCH 06/20] `cudf::rolling` `ROW_NUMBER` support for `decimal32`
 and `decimal64`(#7061)

This PR adds support for `cudf::rolling` for the `ROW_NUMBER` option for `decimal32` and `decimal64`. It also clarifies the documentation.

Authors:
  - Conor Hoekstra <codereport@outlook.com>

Approvers:
  - David
  - Devavret Makkar

URL: https://github.com/rapidsai/cudf/pull/7061
---
 cpp/include/cudf/aggregation.hpp   |  2 +-
 cpp/src/rolling/rolling_detail.cuh | 11 +++++------
 cpp/src/rolling/rolling_detail.hpp |  6 +-----
 cpp/tests/rolling/rolling_test.cpp | 19 +++++++++++++------
 4 files changed, 20 insertions(+), 18 deletions(-)

diff --git a/cpp/include/cudf/aggregation.hpp b/cpp/include/cudf/aggregation.hpp
index fe912a86e37..4a7bc129aaf 100644
--- a/cpp/include/cudf/aggregation.hpp
+++ b/cpp/include/cudf/aggregation.hpp
@@ -73,7 +73,7 @@ class aggregation {
     ARGMIN,          ///< Index of min element
     NUNIQUE,         ///< count number of unique elements
     NTH_ELEMENT,     ///< get the nth element
-    ROW_NUMBER,      ///< get row-number of element
+    ROW_NUMBER,      ///< get row-number of current index (relative to rolling window)
     COLLECT,         ///< collect values into a list
     LEAD,            ///< window function, accesses row at specified offset following current row
     LAG,             ///< window function, accesses row at specified offset preceding current row
diff --git a/cpp/src/rolling/rolling_detail.cuh b/cpp/src/rolling/rolling_detail.cuh
index 20d8abf6701..e0ae16bed7f 100644
--- a/cpp/src/rolling/rolling_detail.cuh
+++ b/cpp/src/rolling/rolling_detail.cuh
@@ -124,16 +124,15 @@ bool __device__ process_rolling_window(column_device_view input,
 {
   cudf::size_type count = end_index - start_index;
 
-  bool output_is_valid                      = (count >= min_periods);
+  bool output_is_valid                      = count >= min_periods;
   output.element<OutputType>(current_index) = count;
 
   return output_is_valid;
 }
 
 /**
- * @brief Calculates row-number within [start_index, end_index).
- *        Count is updated depending on `min_periods`
- *        Returns true if it was valid, else false.
+ * @brief Calculates row-number of current index within [start_index, end_index). Count is updated
+ *        depending on `min_periods`. Returns `true` if it was valid, else `false`.
  */
 template <typename InputType,
           typename OutputType,
@@ -149,8 +148,8 @@ bool __device__ process_rolling_window(column_device_view input,
                                        size_type current_index,
                                        size_type min_periods)
 {
-  bool output_is_valid                      = ((end_index - start_index) >= min_periods);
-  output.element<OutputType>(current_index) = ((current_index - start_index) + 1);
+  bool output_is_valid                      = end_index - start_index >= min_periods;
+  output.element<OutputType>(current_index) = current_index - start_index + 1;
 
   return output_is_valid;
 }
diff --git a/cpp/src/rolling/rolling_detail.hpp b/cpp/src/rolling/rolling_detail.hpp
index a69d39b4957..ed136405a79 100644
--- a/cpp/src/rolling/rolling_detail.hpp
+++ b/cpp/src/rolling/rolling_detail.hpp
@@ -50,14 +50,10 @@ static constexpr bool is_rolling_supported()
 
     return is_valid_numeric_agg;
 
-  } else if (cudf::is_timestamp<ColumnType>()) {
+  } else if (cudf::is_timestamp<ColumnType>() || cudf::is_fixed_point<ColumnType>()) {
     return (op == aggregation::MIN) or (op == aggregation::MAX) or
            (op == aggregation::COUNT_VALID) or (op == aggregation::COUNT_ALL) or
            (op == aggregation::ROW_NUMBER) or (op == aggregation::LEAD) or (op == aggregation::LAG);
-  } else if (cudf::is_fixed_point<ColumnType>()) {
-    return (op == aggregation::MIN) or (op == aggregation::MAX) or
-           (op == aggregation::COUNT_VALID) or (op == aggregation::COUNT_ALL) or
-           (op == aggregation::LEAD) or (op == aggregation::LAG);
   } else if (std::is_same<ColumnType, cudf::string_view>()) {
     return (op == aggregation::MIN) or (op == aggregation::MAX) or
            (op == aggregation::COUNT_VALID) or (op == aggregation::COUNT_ALL) or
diff --git a/cpp/tests/rolling/rolling_test.cpp b/cpp/tests/rolling/rolling_test.cpp
index a83f8361536..7b1ab3cf273 100644
--- a/cpp/tests/rolling/rolling_test.cpp
+++ b/cpp/tests/rolling/rolling_test.cpp
@@ -975,7 +975,8 @@ TYPED_TEST(FixedPointTests, MinMaxCountLagLead)
   auto const expected_lead = fp_wrapper{{1729, 55, 3, 1, 2, 0}, {1, 1, 1, 1, 1, 0}, scale};
   auto const expected_count_val = fw_wrapper{{2, 3, 3, 3, 3, 2}, {1, 1, 1, 1, 1, 1}};
   auto const expected_count_all = fw_wrapper{{2, 3, 3, 3, 3, 2}, {1, 1, 1, 1, 1, 1}};
-  // auto const expected_rowno     = fw_wrapper{{1, 2, 2, 2, 2, 2}, {1, 1, 1, 1, 1, 1}};
+  auto const expected_rowno     = fw_wrapper{{1, 2, 2, 2, 2, 2}, {1, 1, 1, 1, 1, 1}};
+  auto const expected_rowno1    = fw_wrapper{{1, 1, 1, 1, 1, 1}, {1, 1, 1, 1, 1, 1}};
 
   auto const min   = rolling_window(input, 2, 1, 1, make_min_aggregation());
   auto const max   = rolling_window(input, 2, 1, 1, make_max_aggregation());
@@ -983,7 +984,7 @@ TYPED_TEST(FixedPointTests, MinMaxCountLagLead)
   auto const lead  = rolling_window(input, 2, 1, 1, make_lead_aggregation(1));
   auto const valid = rolling_window(input, 2, 1, 1, make_count_aggregation());
   auto const all   = rolling_window(input, 2, 1, 1, make_count_aggregation(null_policy::INCLUDE));
-  EXPECT_THROW(rolling_window(input, 2, 1, 1, make_row_number_aggregation()), cudf::logic_error);
+  auto const rowno = rolling_window(input, 2, 1, 1, make_row_number_aggregation());
 
   CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected_min, min->view());
   CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected_max, max->view());
@@ -991,7 +992,13 @@ TYPED_TEST(FixedPointTests, MinMaxCountLagLead)
   CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected_lead, lead->view());
   CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected_count_val, valid->view());
   CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected_count_all, all->view());
-  // CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected_rowno, rowno->view());
+  CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected_rowno, rowno->view());
+
+  // ROW_NUMBER will always return row 1 if the preceding window is set to a constant 1
+  for (int following = 1; following < 5; ++following) {
+    auto const rowno1 = rolling_window(input, 1, following, 1, make_row_number_aggregation());
+    CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected_rowno1, rowno1->view());
+  }
 }
 
 TYPED_TEST(FixedPointTests, MinMaxCountLagLeadNulls)
@@ -1011,7 +1018,7 @@ TYPED_TEST(FixedPointTests, MinMaxCountLagLeadNulls)
   auto const expected_lead      = fp_wrapper{{1729, 55, 343, 1, 2, 0}, {0, 1, 0, 1, 1, 0}, scale};
   auto const expected_count_val = fw_wrapper{{1, 2, 1, 2, 2, 2}, {1, 1, 1, 1, 1, 1}};
   auto const expected_count_all = fw_wrapper{{2, 3, 3, 3, 3, 2}, {1, 1, 1, 1, 1, 1}};
-  // auto const expected_rowno     = fw_wrapper{{1, 2, 2, 2, 2, 2}, {1, 1, 1, 1, 1, 1}};
+  auto const expected_rowno     = fw_wrapper{{1, 2, 2, 2, 2, 2}, {1, 1, 1, 1, 1, 1}};
 
   auto const min   = rolling_window(input, 2, 1, 1, make_min_aggregation());
   auto const max   = rolling_window(input, 2, 1, 1, make_max_aggregation());
@@ -1019,7 +1026,7 @@ TYPED_TEST(FixedPointTests, MinMaxCountLagLeadNulls)
   auto const lead  = rolling_window(input, 2, 1, 1, make_lead_aggregation(1));
   auto const valid = rolling_window(input, 2, 1, 1, make_count_aggregation());
   auto const all   = rolling_window(input, 2, 1, 1, make_count_aggregation(null_policy::INCLUDE));
-  EXPECT_THROW(rolling_window(input, 2, 1, 1, make_row_number_aggregation()), cudf::logic_error);
+  auto const rowno = rolling_window(input, 2, 1, 1, make_row_number_aggregation());
 
   CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected_min, min->view());
   CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected_max, max->view());
@@ -1027,7 +1034,7 @@ TYPED_TEST(FixedPointTests, MinMaxCountLagLeadNulls)
   CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected_lead, lead->view());
   CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected_count_val, valid->view());
   CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected_count_all, all->view());
-  // CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected_rowno, rowno->view());
+  CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected_rowno, rowno->view());
 }
 
 CUDF_TEST_PROGRAM_MAIN()

From 7bf050536114e6f8ac6d600087fc24d132a7f700 Mon Sep 17 00:00:00 2001
From: Vukasin Milovanovic <vmilovanovic@nvidia.com>
Date: Tue, 5 Jan 2021 10:52:25 -0800
Subject: [PATCH 07/20] Refactor ORC `ProtobufReader` to make it more
 extendable(#7055)

Related to #5826

Refactor the `ProtobufReader` API to facilitate expansion to support robust reading of column statistics.
Changes include:

- Move `orc::metadata` from `readder_impl.cu` to `orc.h` so it can be reused for statistics related APIs.
- Removed duplicated code in `read_orc_statistics` - use `orc::metadata` instead.
- Rename `ColumnStatistics` to `ColStatsBlob`, since that's what it currently is.
- Avoid redundant copies in `read_orc_statistics`,
- Replace `get_u32`, `get_i32`, etc. with templated `get`.
- Replace per-type functors (e.g. `FieldUInt64`) with templated `field_reader`s to reduce code repetition.
- The two type-specific parts of `FieldXYZ` functors (field enum and read impl) are now separate to avoid redundant code.
- `field_reader` dispatches based on the value type, so also added `packed_field_reader` and `raw_field_reader` for packed fields and blob reads (respectively).
- Replace return value based error checking in `ProtobufReader` with `CUDF_EXPECTS`.
- Removed `InitSchema` from `ProtobufReader` - schema is only used to determine column names. The names are now lazily calculated in `metadata::get_column_name`

Authors:
  - vuule <vmilovanovic@nvidia.com>
  - Vukasin Milovanovic <vukasin.milovanovic.87@gmail.com>

Approvers:
  - Kumar Aatish
  - Conor Hoekstra

URL: https://github.com/rapidsai/cudf/pull/7055
---
 cpp/src/io/functions.cpp            |  60 +----
 cpp/src/io/orc/orc.cpp              | 343 +++++++++++++++++--------
 cpp/src/io/orc/orc.h                | 385 ++++++++++++++++++++++------
 cpp/src/io/orc/orc_field_reader.hpp | 295 ++-------------------
 cpp/src/io/orc/reader_impl.cu       | 173 +------------
 cpp/src/io/orc/reader_impl.hpp      |   2 +-
 6 files changed, 576 insertions(+), 682 deletions(-)

diff --git a/cpp/src/io/functions.cpp b/cpp/src/io/functions.cpp
index 69dd69e5505..69a7513cfef 100644
--- a/cpp/src/io/functions.cpp
+++ b/cpp/src/io/functions.cpp
@@ -211,65 +211,29 @@ std::vector<std::vector<std::string>> read_orc_statistics(source_info const& src
     CUDF_FAIL("Unsupported source type");
   }
 
-  // Get size of file and size of postscript
-  const auto len         = source->size();
-  const auto max_ps_size = std::min(len, static_cast<size_t>(256));
-
-  // Read uncompressed postscript section (max 255 bytes + 1 byte for length)
-  auto buffer            = source->host_read(len - max_ps_size, max_ps_size);
-  const size_t ps_length = buffer->data()[max_ps_size - 1];
-  const uint8_t* ps_data = &buffer->data()[max_ps_size - ps_length - 1];
-  orc::ProtobufReader pb;
-  orc::PostScript ps;
-  pb.init(ps_data, ps_length);
-  CUDF_EXPECTS(pb.read(ps, ps_length), "Cannot read postscript");
-  CUDF_EXPECTS(ps.footerLength + ps_length < len, "Invalid footer length");
-
-  // If compression is used, all the rest of the metadata is compressed
-  // If no compressed is used, the decompressor is simply a pass-through
-  std::unique_ptr<orc::OrcDecompressor> decompressor =
-    std::make_unique<orc::OrcDecompressor>(ps.compression, ps.compressionBlockSize);
-
-  // Read compressed filefooter section
-  buffer           = source->host_read(len - ps_length - 1 - ps.footerLength, ps.footerLength);
-  size_t ff_length = 0;
-  auto ff_data     = decompressor->Decompress(buffer->data(), ps.footerLength, &ff_length);
-  orc::FileFooter ff;
-  pb.init(ff_data, ff_length);
-  CUDF_EXPECTS(pb.read(ff, ff_length), "Cannot read filefooter");
-  CUDF_EXPECTS(ff.types.size() > 0, "No columns found");
-
-  // Read compressed metadata section
-  buffer =
-    source->host_read(len - ps_length - 1 - ps.footerLength - ps.metadataLength, ps.metadataLength);
-  size_t md_length = 0;
-  auto md_data     = decompressor->Decompress(buffer->data(), ps.metadataLength, &md_length);
-  orc::Metadata md;
-  pb.init(md_data, md_length);
-  CUDF_EXPECTS(pb.read(md, md_length), "Cannot read metadata");
+  orc::metadata metadata(source.get());
 
   // Initialize statistics to return
   std::vector<std::vector<std::string>> statistics_blobs;
 
   // Get column names
-  std::vector<std::string> column_names;
-  for (auto i = 0; i < ff.types.size(); i++) { column_names.push_back(ff.GetColumnName(i)); }
-  statistics_blobs.push_back(column_names);
+  statistics_blobs.emplace_back();
+  for (auto i = 0; i < metadata.get_num_columns(); i++) {
+    statistics_blobs.back().push_back(metadata.get_column_name(i));
+  }
 
   // Get file-level statistics, statistics of each column of file
-  std::vector<std::string> file_column_statistics_blobs;
-  for (orc::ColumnStatistics stats : ff.statistics) {
-    file_column_statistics_blobs.push_back(std::string(stats.begin(), stats.end()));
+  statistics_blobs.emplace_back();
+  for (auto const& stats : metadata.ff.statistics) {
+    statistics_blobs.back().push_back(std::string(stats.cbegin(), stats.cend()));
   }
-  statistics_blobs.push_back(file_column_statistics_blobs);
 
   // Get stripe-level statistics
-  for (orc::StripeStatistics stripe_stats : md.stripeStats) {
-    std::vector<std::string> stripe_column_statistics_blobs;
-    for (orc::ColumnStatistics stats : stripe_stats.colStats) {
-      stripe_column_statistics_blobs.push_back(std::string(stats.begin(), stats.end()));
+  for (auto const& stripe_stats : metadata.md.stripeStats) {
+    statistics_blobs.emplace_back();
+    for (auto const& stats : stripe_stats.colStats) {
+      statistics_blobs.back().push_back(std::string(stats.cbegin(), stats.cend()));
     }
-    statistics_blobs.push_back(stripe_column_statistics_blobs);
   }
 
   return statistics_blobs;
diff --git a/cpp/src/io/orc/orc.cpp b/cpp/src/io/orc/orc.cpp
index c816f2ccbd8..c75e4fc754e 100644
--- a/cpp/src/io/orc/orc.cpp
+++ b/cpp/src/io/orc/orc.cpp
@@ -22,12 +22,20 @@
 namespace cudf {
 namespace io {
 namespace orc {
+
+uint32_t ProtobufReader::read_field_size(const uint8_t *end)
+{
+  auto const size = get<uint32_t>();
+  CUDF_EXPECTS(size <= static_cast<uint32_t>(end - m_cur), "Protobuf parsing out of bounds");
+  return size;
+}
+
 void ProtobufReader::skip_struct_field(int t)
 {
   switch (t) {
-    case PB_TYPE_VARINT: get_u32(); break;
+    case PB_TYPE_VARINT: get<uint32_t>(); break;
     case PB_TYPE_FIXED64: skip_bytes(8); break;
-    case PB_TYPE_FIXEDLEN: skip_bytes(get_u32()); break;
+    case PB_TYPE_FIXEDLEN: skip_bytes(get<uint32_t>()); break;
     case PB_TYPE_FIXED32: skip_bytes(4); break;
     default:
       // printf("invalid type (%d)\n", t);
@@ -35,138 +43,88 @@ void ProtobufReader::skip_struct_field(int t)
   }
 }
 
-bool ProtobufReader::read(PostScript &s, size_t maxlen)
+void ProtobufReader::read(PostScript &s, size_t maxlen)
 {
-  auto op = std::make_tuple(FieldUInt64(1, s.footerLength),
-                            FieldEnum<CompressionKind>(2, s.compression),
-                            FieldUInt32(3, s.compressionBlockSize),
-                            FieldPackedUInt32(4, s.version),
-                            FieldUInt64(5, s.metadataLength),
-                            FieldString(8000, s.magic));
-  return function_builder(s, maxlen, op);
+  auto op = std::make_tuple(make_field_reader(1, s.footerLength),
+                            make_field_reader(2, s.compression),
+                            make_field_reader(3, s.compressionBlockSize),
+                            make_packed_field_reader(4, s.version),
+                            make_field_reader(5, s.metadataLength),
+                            make_field_reader(8000, s.magic));
+  function_builder(s, maxlen, op);
 }
 
-bool ProtobufReader::read(FileFooter &s, size_t maxlen)
+void ProtobufReader::read(FileFooter &s, size_t maxlen)
 {
-  auto op = std::make_tuple(FieldUInt64(1, s.headerLength),
-                            FieldUInt64(2, s.contentLength),
-                            FieldRepeatedStruct(3, s.stripes),
-                            FieldRepeatedStruct(4, s.types),
-                            FieldRepeatedStruct(5, s.metadata),
-                            FieldUInt64(6, s.numberOfRows),
-                            FieldRepeatedStructBlob(7, s.statistics),
-                            FieldUInt32(8, s.rowIndexStride));
-  return function_builder(s, maxlen, op);
+  auto op = std::make_tuple(make_field_reader(1, s.headerLength),
+                            make_field_reader(2, s.contentLength),
+                            make_field_reader(3, s.stripes),
+                            make_field_reader(4, s.types),
+                            make_field_reader(5, s.metadata),
+                            make_field_reader(6, s.numberOfRows),
+                            make_raw_field_reader(7, s.statistics),
+                            make_field_reader(8, s.rowIndexStride));
+  function_builder(s, maxlen, op);
 }
 
-bool ProtobufReader::read(StripeInformation &s, size_t maxlen)
+void ProtobufReader::read(StripeInformation &s, size_t maxlen)
 {
-  auto op = std::make_tuple(FieldUInt64(1, s.offset),
-                            FieldUInt64(2, s.indexLength),
-                            FieldUInt64(3, s.dataLength),
-                            FieldUInt32(4, s.footerLength),
-                            FieldUInt32(5, s.numberOfRows));
-  return function_builder(s, maxlen, op);
+  auto op = std::make_tuple(make_field_reader(1, s.offset),
+                            make_field_reader(2, s.indexLength),
+                            make_field_reader(3, s.dataLength),
+                            make_field_reader(4, s.footerLength),
+                            make_field_reader(5, s.numberOfRows));
+  function_builder(s, maxlen, op);
 }
 
-bool ProtobufReader::read(SchemaType &s, size_t maxlen)
+void ProtobufReader::read(SchemaType &s, size_t maxlen)
 {
-  auto op = std::make_tuple(FieldEnum<TypeKind>(1, s.kind),
-                            FieldPackedUInt32(2, s.subtypes),
-                            FieldRepeatedString(3, s.fieldNames),
-                            FieldUInt32(4, s.maximumLength),
-                            FieldUInt32(5, s.precision),
-                            FieldUInt32(6, s.scale));
-  return function_builder(s, maxlen, op);
+  auto op = std::make_tuple(make_field_reader(1, s.kind),
+                            make_packed_field_reader(2, s.subtypes),
+                            make_field_reader(3, s.fieldNames),
+                            make_field_reader(4, s.maximumLength),
+                            make_field_reader(5, s.precision),
+                            make_field_reader(6, s.scale));
+  function_builder(s, maxlen, op);
 }
 
-bool ProtobufReader::read(UserMetadataItem &s, size_t maxlen)
+void ProtobufReader::read(UserMetadataItem &s, size_t maxlen)
 {
-  auto op = std::make_tuple(FieldString(1, s.name), FieldString(2, s.value));
-  return function_builder(s, maxlen, op);
+  auto op = std::make_tuple(make_field_reader(1, s.name), make_field_reader(2, s.value));
+  function_builder(s, maxlen, op);
 }
 
-bool ProtobufReader::read(StripeFooter &s, size_t maxlen)
+void ProtobufReader::read(StripeFooter &s, size_t maxlen)
 {
-  auto op = std::make_tuple(FieldRepeatedStruct(1, s.streams),
-                            FieldRepeatedStruct(2, s.columns),
-                            FieldString(3, s.writerTimezone));
-  return function_builder(s, maxlen, op);
+  auto op = std::make_tuple(make_field_reader(1, s.streams),
+                            make_field_reader(2, s.columns),
+                            make_field_reader(3, s.writerTimezone));
+  function_builder(s, maxlen, op);
 }
 
-bool ProtobufReader::read(Stream &s, size_t maxlen)
+void ProtobufReader::read(Stream &s, size_t maxlen)
 {
   auto op = std::make_tuple(
-    FieldEnum<StreamKind>(1, s.kind), FieldUInt32(2, s.column), FieldUInt64(3, s.length));
-  return function_builder(s, maxlen, op);
-}
-
-bool ProtobufReader::read(ColumnEncoding &s, size_t maxlen)
-{
-  auto op =
-    std::make_tuple(FieldEnum<ColumnEncodingKind>(1, s.kind), FieldUInt32(2, s.dictionarySize));
-  return function_builder(s, maxlen, op);
-}
-
-bool ProtobufReader::read(StripeStatistics &s, size_t maxlen)
-{
-  auto op = std::make_tuple(FieldRepeatedStructBlob(1, s.colStats));
-  return function_builder(s, maxlen, op);
+    make_field_reader(1, s.kind), make_field_reader(2, s.column), make_field_reader(3, s.length));
+  function_builder(s, maxlen, op);
 }
 
-bool ProtobufReader::read(Metadata &s, size_t maxlen)
+void ProtobufReader::read(ColumnEncoding &s, size_t maxlen)
 {
-  auto op = std::make_tuple(FieldRepeatedStruct(1, s.stripeStats));
-  return function_builder(s, maxlen, op);
+  auto op = std::make_tuple(make_field_reader(1, s.kind), make_field_reader(2, s.dictionarySize));
+  function_builder(s, maxlen, op);
 }
 
-// return the column name
-std::string FileFooter::GetColumnName(uint32_t column_id)
+void ProtobufReader::read(StripeStatistics &s, size_t maxlen)
 {
-  std::string s       = "";
-  uint32_t parent_idx = column_id, idx, field_idx;
-  do {
-    idx        = parent_idx;
-    parent_idx = (idx < types.size()) ? (uint32_t)types[idx].parent_idx : ~0;
-    field_idx  = (parent_idx < types.size()) ? (uint32_t)types[idx].field_idx : ~0;
-    if (parent_idx >= types.size()) break;
-    if (field_idx < types[parent_idx].fieldNames.size()) {
-      if (s.length() > 0)
-        s = types[parent_idx].fieldNames[field_idx] + "." + s;
-      else
-        s = types[parent_idx].fieldNames[field_idx];
-    }
-  } while (parent_idx != idx);
-  // If we have no name (root column), generate a name
-  if (s.length() == 0) { s = "col" + std::to_string(column_id); }
-  return s;
+  auto op = std::make_tuple(make_raw_field_reader(1, s.colStats));
+  function_builder(s, maxlen, op);
 }
 
-// Initializes the parent_idx field in the schema
-bool ProtobufReader::InitSchema(FileFooter &ff)
+void ProtobufReader::read(Metadata &s, size_t maxlen)
 {
-  int32_t schema_size = (int32_t)ff.types.size();
-  for (int32_t i = 0; i < schema_size; i++) {
-    int32_t num_children = (int32_t)ff.types[i].subtypes.size();
-    if (ff.types[i].parent_idx == -1)  // Not initialized
-    {
-      ff.types[i].parent_idx = i;  // set root node as its own parent
-    }
-    for (int32_t j = 0; j < num_children; j++) {
-      uint32_t column_id = ff.types[i].subtypes[j];
-      if (column_id <= (uint32_t)i || column_id >= (uint32_t)schema_size) {
-        // Invalid column id (or at least not a schema index)
-        return false;
-      }
-      if (ff.types[column_id].parent_idx != -1) {
-        // Same node referenced twice
-        return false;
-      }
-      ff.types[column_id].parent_idx = i;
-      ff.types[column_id].field_idx  = j;
-    }
-  }
-  return true;
+  auto op = std::make_tuple(make_field_reader(1, s.stripeStats));
+  function_builder(s, maxlen, op);
 }
 
 /**
@@ -402,6 +360,181 @@ const uint8_t *OrcDecompressor::Decompress(const uint8_t *srcBytes, size_t srcLe
   return m_buf.data();
 }
 
+metadata::metadata(datasource *const src) : source(src)
+{
+  const auto len         = source->size();
+  const auto max_ps_size = std::min(len, static_cast<size_t>(256));
+
+  // Read uncompressed postscript section (max 255 bytes + 1 byte for length)
+  auto buffer            = source->host_read(len - max_ps_size, max_ps_size);
+  const size_t ps_length = buffer->data()[max_ps_size - 1];
+  const uint8_t *ps_data = &buffer->data()[max_ps_size - ps_length - 1];
+  ProtobufReader(ps_data, ps_length).read(ps);
+  CUDF_EXPECTS(ps.footerLength + ps_length < len, "Invalid footer length");
+
+  // If compression is used, the rest of the metadata is compressed
+  // If no compressed is used, the decompressor is simply a pass-through
+  decompressor = std::make_unique<OrcDecompressor>(ps.compression, ps.compressionBlockSize);
+
+  // Read compressed filefooter section
+  buffer           = source->host_read(len - ps_length - 1 - ps.footerLength, ps.footerLength);
+  size_t ff_length = 0;
+  auto ff_data     = decompressor->Decompress(buffer->data(), ps.footerLength, &ff_length);
+  ProtobufReader(ff_data, ff_length).read(ff);
+  CUDF_EXPECTS(get_num_columns() > 0, "No columns found");
+
+  // Read compressed metadata section
+  buffer =
+    source->host_read(len - ps_length - 1 - ps.footerLength - ps.metadataLength, ps.metadataLength);
+  size_t md_length = 0;
+  auto md_data     = decompressor->Decompress(buffer->data(), ps.metadataLength, &md_length);
+  orc::ProtobufReader(md_data, md_length).read(md);
+}
+
+std::vector<metadata::OrcStripeInfo> metadata::select_stripes(const std::vector<size_type> &stripes,
+                                                              size_type &row_start,
+                                                              size_type &row_count)
+{
+  std::vector<OrcStripeInfo> selection;
+
+  if (!stripes.empty()) {
+    size_t stripe_rows = 0;
+    for (const auto &stripe_idx : stripes) {
+      CUDF_EXPECTS(stripe_idx >= 0 && stripe_idx < get_num_stripes(), "Invalid stripe index");
+      selection.emplace_back(&ff.stripes[stripe_idx], nullptr);
+      stripe_rows += ff.stripes[stripe_idx].numberOfRows;
+    }
+    // row_start is 0 if stripes are set. If this is not true anymore, then
+    // row_start needs to be subtracted to get the correct row_count
+    CUDF_EXPECTS(row_start == 0, "Start row index should be 0");
+    row_count = static_cast<size_type>(stripe_rows);
+  } else {
+    row_start = std::max(row_start, 0);
+    if (row_count < 0) {
+      row_count = static_cast<size_type>(
+        std::min<size_t>(get_total_rows() - row_start, std::numeric_limits<size_type>::max()));
+    } else {
+      row_count = static_cast<size_type>(std::min<size_t>(get_total_rows() - row_start, row_count));
+    }
+    CUDF_EXPECTS(row_count >= 0 && row_start >= 0, "Negative row count or starting row");
+    CUDF_EXPECTS(
+      !(row_start > 0 && (row_count > (std::numeric_limits<size_type>::max() - row_start))),
+      "Summation of starting row index and number of rows would cause overflow");
+
+    size_type stripe_skip_rows = 0;
+    for (size_t i = 0, count = 0; i < ff.stripes.size(); ++i) {
+      count += ff.stripes[i].numberOfRows;
+      if (count > static_cast<size_t>(row_start)) {
+        if (selection.empty()) {
+          stripe_skip_rows =
+            static_cast<size_type>(row_start - (count - ff.stripes[i].numberOfRows));
+        }
+        selection.emplace_back(&ff.stripes[i], nullptr);
+      }
+      if (count >= static_cast<size_t>(row_start) + static_cast<size_t>(row_count)) { break; }
+    }
+    row_start = stripe_skip_rows;
+  }
+
+  // Read each stripe's stripefooter metadata
+  if (not selection.empty()) {
+    stripefooters.resize(selection.size());
+    for (size_t i = 0; i < selection.size(); ++i) {
+      const auto stripe         = selection[i].first;
+      const auto sf_comp_offset = stripe->offset + stripe->indexLength + stripe->dataLength;
+      const auto sf_comp_length = stripe->footerLength;
+      CUDF_EXPECTS(sf_comp_offset + sf_comp_length < source->size(), "Invalid stripe information");
+
+      const auto buffer = source->host_read(sf_comp_offset, sf_comp_length);
+      size_t sf_length  = 0;
+      auto sf_data      = decompressor->Decompress(buffer->data(), sf_comp_length, &sf_length);
+      ProtobufReader(sf_data, sf_length).read(stripefooters[i]);
+      selection[i].second = &stripefooters[i];
+    }
+  }
+
+  return selection;
+}
+
+std::vector<int> metadata::select_columns(std::vector<std::string> use_names,
+                                          bool &has_timestamp_column)
+{
+  std::vector<int> selection;
+
+  if (not use_names.empty()) {
+    int index = 0;
+    for (const auto &use_name : use_names) {
+      for (int i = 0; i < get_num_columns(); ++i, ++index) {
+        if (index >= get_num_columns()) { index = 0; }
+        if (get_column_name(index) == use_name) {
+          selection.emplace_back(index);
+          if (ff.types[index].kind == orc::TIMESTAMP) { has_timestamp_column = true; }
+          index++;
+          break;
+        }
+      }
+    }
+  } else {
+    // For now, only select all leaf nodes
+    for (int i = 0; i < get_num_columns(); ++i) {
+      if (ff.types[i].subtypes.empty()) {
+        selection.emplace_back(i);
+        if (ff.types[i].kind == orc::TIMESTAMP) { has_timestamp_column = true; }
+      }
+    }
+  }
+  CUDF_EXPECTS(selection.size() > 0, "Filtered out all columns");
+
+  return selection;
+}
+
+void metadata::init_column_names()
+{
+  auto const schema_idxs = get_schema_indexes();
+  auto const &types      = ff.types;
+  for (int32_t col_id = 0; col_id < get_num_columns(); ++col_id) {
+    std::string col_name;
+    uint32_t parent_idx = col_id;
+    uint32_t idx        = col_id;
+    do {
+      idx        = parent_idx;
+      parent_idx = (idx < types.size()) ? static_cast<uint32_t>(schema_idxs[idx].parent) : ~0;
+      if (parent_idx >= types.size()) break;
+
+      auto const field_idx =
+        (parent_idx < types.size()) ? static_cast<uint32_t>(schema_idxs[idx].field) : ~0;
+      if (field_idx < types[parent_idx].fieldNames.size()) {
+        col_name =
+          types[parent_idx].fieldNames[field_idx] + (col_name.empty() ? "" : ("." + col_name));
+      }
+    } while (parent_idx != idx);
+    // If we have no name (root column), generate a name
+    column_names.push_back(col_name.empty() ? "col" + std::to_string(col_id) : col_name);
+  }
+}
+
+std::vector<metadata::schema_indexes> metadata::get_schema_indexes() const
+{
+  std::vector<schema_indexes> result(ff.types.size());
+
+  auto const schema_size = static_cast<uint32_t>(result.size());
+  for (uint32_t i = 0; i < schema_size; i++) {
+    auto const &subtypes    = ff.types[i].subtypes;
+    auto const num_children = static_cast<uint32_t>(subtypes.size());
+    if (result[i].parent == -1) {  // Not initialized
+      result[i].parent = i;        // set root node as its own parent
+    }
+    for (uint32_t j = 0; j < num_children; j++) {
+      auto const column_id = subtypes[j];
+      CUDF_EXPECTS(column_id > i && column_id < schema_size, "Invalid column id");
+      CUDF_EXPECTS(result[column_id].parent == -1, "Same node referenced twice");
+      result[column_id].parent = i;
+      result[column_id].field  = j;
+    }
+  }
+  return result;
+}
+
 }  // namespace orc
 }  // namespace io
 }  // namespace cudf
diff --git a/cpp/src/io/orc/orc.h b/cpp/src/io/orc/orc.h
index e425c0b8eef..1539cbd1cb3 100644
--- a/cpp/src/io/orc/orc.h
+++ b/cpp/src/io/orc/orc.h
@@ -23,7 +23,10 @@
 #include <string>
 #include <vector>
 
-#include "io/comp/io_uncomp.h"
+#include <cudf/utilities/error.hpp>
+
+#include <io/comp/io_uncomp.h>
+#include <cudf/io/datasource.hpp>
 #include "orc_common.h"
 
 namespace cudf {
@@ -54,9 +57,6 @@ struct SchemaType {
     0;  // optional: the maximum length of the type for varchar or char in UTF-8 characters
   uint32_t precision = 0;  // optional: the precision and scale for decimal
   uint32_t scale     = 0;
-  // Inferred fields
-  int32_t parent_idx = -1;  // parent node (equal to current node for root nodes)
-  int32_t field_idx  = -1;  // field index in parent's subtype vector
 };
 
 struct UserMetadataItem {
@@ -64,19 +64,17 @@ struct UserMetadataItem {
   std::string value;  // the user defined binary value as string
 };
 
-typedef std::vector<uint8_t> ColumnStatistics;  // Column statistics blob
+typedef std::vector<uint8_t> ColStatsBlob;  // Column statistics blob
 
 struct FileFooter {
-  uint64_t headerLength  = 0;                // the length of the file header in bytes (always 3)
-  uint64_t contentLength = 0;                // the length of the file header and body in bytes
-  std::vector<StripeInformation> stripes;    // the information about the stripes
-  std::vector<SchemaType> types;             // the schema information
-  std::vector<UserMetadataItem> metadata;    // the user metadata that was added
-  uint64_t numberOfRows = 0;                 // the total number of rows in the file
-  std::vector<ColumnStatistics> statistics;  // Column statistics blobs
-  uint32_t rowIndexStride = 0;               // the maximum number of rows in each index entry
-  // Helper methods
-  std::string GetColumnName(uint32_t column_id);  // return the column name
+  uint64_t headerLength  = 0;              // the length of the file header in bytes (always 3)
+  uint64_t contentLength = 0;              // the length of the file header and body in bytes
+  std::vector<StripeInformation> stripes;  // the information about the stripes
+  std::vector<SchemaType> types;           // the schema information
+  std::vector<UserMetadataItem> metadata;  // the user metadata that was added
+  uint64_t numberOfRows = 0;               // the total number of rows in the file
+  std::vector<ColStatsBlob> statistics;    // Column statistics blobs
+  uint32_t rowIndexStride = 0;             // the maximum number of rows in each index entry
 };
 
 struct Stream {
@@ -97,7 +95,7 @@ struct StripeFooter {
 };
 
 struct StripeStatistics {
-  std::vector<ColumnStatistics> colStats;  // Column statistics blobs
+  std::vector<ColStatsBlob> colStats;  // Column statistics blobs
 };
 
 struct Metadata {
@@ -112,99 +110,260 @@ struct Metadata {
 
 class ProtobufReader {
  public:
-  ProtobufReader() { m_base = m_cur = m_end = nullptr; }
-  ProtobufReader(const uint8_t *base, size_t len) { init(base, len); }
-  void init(const uint8_t *base, size_t len)
+  ProtobufReader(const uint8_t *base, size_t len) : m_base(base), m_cur(base), m_end(base + len) {}
+
+  template <typename T>
+  void read(T &s)
   {
-    m_base = m_cur = base;
-    m_end          = base + len;
+    read(s, m_end - m_cur);
   }
-  ptrdiff_t bytecount() const { return m_cur - m_base; }
-  unsigned int getb() { return (m_cur < m_end) ? *m_cur++ : 0; }
+  void read(PostScript &, size_t maxlen);
+  void read(FileFooter &, size_t maxlen);
+  void read(StripeInformation &, size_t maxlen);
+  void read(SchemaType &, size_t maxlen);
+  void read(UserMetadataItem &, size_t maxlen);
+  void read(StripeFooter &, size_t maxlen);
+  void read(Stream &, size_t maxlen);
+  void read(ColumnEncoding &, size_t maxlen);
+  void read(StripeStatistics &, size_t maxlen);
+  void read(Metadata &, size_t maxlen);
+
+ private:
+  template <int index>
+  friend class FunctionSwitchImpl;
+
   void skip_bytes(size_t bytecnt)
   {
     bytecnt = std::min(bytecnt, (size_t)(m_end - m_cur));
     m_cur += bytecnt;
   }
-  uint32_t get_u32()
+
+  template <typename T>
+  T get();
+
+  void skip_struct_field(int t);
+
+  template <typename T, typename... Operator>
+  void function_builder(T &s, size_t maxlen, std::tuple<Operator...> &op);
+
+  template <
+    typename T,
+    typename std::enable_if_t<!std::is_integral<T>::value and !std::is_enum<T>::value> * = nullptr>
+  int static constexpr encode_field_number(int field_number) noexcept
   {
-    uint32_t v = 0;
-    for (uint32_t l = 0;; l += 7) {
-      uint32_t c = getb();
-      v |= (c & 0x7f) << l;
-      if (c < 0x80) return v;
-    }
+    return (field_number * 8) + PB_TYPE_FIXEDLEN;
   }
-  uint64_t get_u64()
+
+  template <
+    typename T,
+    typename std::enable_if_t<std::is_integral<T>::value or std::is_enum<T>::value> * = nullptr>
+  int static constexpr encode_field_number(int field_number) noexcept
   {
-    uint64_t v = 0;
-    for (uint64_t l = 0;; l += 7) {
-      uint64_t c = getb();
-      v |= (c & 0x7f) << l;
-      if (c < 0x80) return v;
-    }
+    return (field_number * 8) + PB_TYPE_VARINT;
   }
-  int32_t get_i32()
+
+  uint32_t read_field_size(const uint8_t *end);
+
+  template <typename T, typename std::enable_if_t<std::is_integral<T>::value> * = nullptr>
+  void read_field(T &value, const uint8_t *end)
   {
-    uint32_t u = get_u32();
-    return (int32_t)((u >> 1u) ^ -(int32_t)(u & 1));
+    value = get<T>();
   }
-  int64_t get_i64()
+
+  template <typename T, typename std::enable_if_t<std::is_enum<T>::value> * = nullptr>
+  void read_field(T &value, const uint8_t *end)
   {
-    uint64_t u = get_u64();
-    return (int64_t)((u >> 1u) ^ -(int64_t)(u & 1));
+    value = static_cast<T>(get<uint32_t>());
   }
-  void skip_struct_field(int t);
 
- public:
-  bool read(PostScript &, size_t maxlen);
-  bool read(FileFooter &, size_t maxlen);
-  bool read(StripeInformation &, size_t maxlen);
-  bool read(SchemaType &, size_t maxlen);
-  bool read(UserMetadataItem &, size_t maxlen);
-  bool read(StripeFooter &, size_t maxlen);
-  bool read(Stream &, size_t maxlen);
-  bool read(ColumnEncoding &, size_t maxlen);
-  bool read(StripeStatistics &, size_t maxlen);
-  bool read(Metadata &, size_t maxlen);
+  template <typename T, typename std::enable_if_t<std::is_same<T, std::string>::value> * = nullptr>
+  void read_field(T &value, const uint8_t *end)
+  {
+    auto const size = read_field_size(end);
+    value.assign(reinterpret_cast<const char *>(m_cur), size);
+    m_cur += size;
+  }
 
- protected:
-  bool InitSchema(FileFooter &);
+  template <typename T,
+            typename std::enable_if_t<std::is_same<T, std::vector<std::string>>::value> * = nullptr>
+  void read_field(T &value, const uint8_t *end)
+  {
+    auto const size = read_field_size(end);
+    value.emplace_back(reinterpret_cast<const char *>(m_cur), size);
+    m_cur += size;
+  }
+
+  template <typename T,
+            typename std::enable_if_t<
+              std::is_same<T, std::vector<typename T::value_type>>::value and
+              !std::is_same<std::string, typename T::value_type>::value> * = nullptr>
+  void read_field(T &value, const uint8_t *end)
+  {
+    auto const size = read_field_size(end);
+    value.emplace_back();
+    read(value.back(), size);
+  }
 
-  template <typename T, typename... Operator>
-  bool function_builder(T &s, size_t maxlen, std::tuple<Operator...> &op);
   template <typename T>
-  bool function_builder_return(T &s, const uint8_t *end);
-  struct FieldInt32;
-  struct FieldUInt32;
-  struct FieldInt64;
-  struct FieldUInt64;
-  template <typename Enum>
-  struct FieldEnum;
-  struct FieldPackedUInt32;
-  struct FieldString;
-  struct FieldRepeatedString;
-  template <typename Enum>
-  struct FieldRepeatedStructFunctor;
-  template <typename Enum>
-  struct FieldRepeatedStructBlobFunctor;
-  template <typename Enum>
-  FieldRepeatedStructFunctor<Enum> FieldRepeatedStruct(int f, std::vector<Enum> &v)
+  void read_packed_field(T &value, const uint8_t *end)
   {
-    return FieldRepeatedStructFunctor<Enum>(f, v);
+    auto const len       = get<uint32_t>();
+    auto const field_end = std::min(m_cur + len, end);
+    while (m_cur < field_end) value.push_back(get<typename T::value_type>());
   }
-  template <typename Enum>
-  FieldRepeatedStructBlobFunctor<Enum> FieldRepeatedStructBlob(int f, std::vector<Enum> &v)
+
+  template <typename T>
+  void read_raw_field(T &value, const uint8_t *end)
   {
-    return FieldRepeatedStructBlobFunctor<Enum>(f, v);
+    auto const size = read_field_size(end);
+    value.emplace_back(m_cur, m_cur + size);
+    m_cur += size;
   }
 
- protected:
-  const uint8_t *m_base;
+  template <typename T>
+  struct field_reader {
+    int const encoded_field_number;
+    T &output_value;
+
+    field_reader(int field_number, T &field_value)
+      : encoded_field_number(encode_field_number<T>(field_number)), output_value(field_value)
+    {
+    }
+
+    inline void operator()(ProtobufReader *pbr, const uint8_t *end)
+    {
+      pbr->read_field(output_value, end);
+    }
+  };
+
+  template <typename T>
+  struct packed_field_reader {
+    int const encoded_field_number;
+    T &output_value;
+
+    packed_field_reader(int field_number, T &field_value)
+      : encoded_field_number(encode_field_number<T>(field_number)), output_value(field_value)
+    {
+    }
+
+    inline void operator()(ProtobufReader *pbr, const uint8_t *end)
+    {
+      pbr->read_packed_field(output_value, end);
+    }
+  };
+
+  template <typename T>
+  struct raw_field_reader {
+    int const encoded_field_number;
+    T &output_value;
+
+    raw_field_reader(int field_number, T &field_value)
+      : encoded_field_number(encode_field_number<T>(field_number)), output_value(field_value)
+    {
+    }
+
+    inline void operator()(ProtobufReader *pbr, const uint8_t *end)
+    {
+      pbr->read_raw_field(output_value, end);
+    }
+  };
+
+  const uint8_t *const m_base;
   const uint8_t *m_cur;
-  const uint8_t *m_end;
+  const uint8_t *const m_end;
+
+ public:
+  /**
+   * @brief Returns a field reader object of correct type, based on the `field_value` type.
+   *
+   * @tparam Type of the field (inferred from `field_value` type)
+   * @param field_number The field number of the field to be read
+   * @param field_value Reference to the object the field reader will write to
+   * @return the field reader object of the right type
+   */
+  template <typename T>
+  static auto make_field_reader(int field_number, T &field_value)
+  {
+    return field_reader<T>(field_number, field_value);
+  }
+
+  /**
+   * @brief Returns a reader object for packed fields, based on the `field_value` type.
+   *
+   * @tparam Type of the field (inferred from `field_value` type)
+   * @param field_number The field number of the field to be read
+   * @param field_value Reference to the object the field reader will write to
+   * @return the packed field reader object of the right type
+   */
+  template <typename T>
+  static auto make_packed_field_reader(int field_number, T &field_value)
+  {
+    return packed_field_reader<T>(field_number, field_value);
+  }
+
+  /**
+   * @brief Returns a field reader that does not decode data, with type based on the `field_value`
+   * type.
+   *
+   * @tparam Type of the field (inferred from `field_value` type)
+   * @param field_number The field number of the field to be read
+   * @param field_value Reference to the object the field reader will write to
+   * @return the raw field reader object of the right type
+   */
+  template <typename T>
+  static auto make_raw_field_reader(int field_number, T &field_value)
+  {
+    return raw_field_reader<T>(field_number, field_value);
+  }
 };
 
+template <>
+inline uint8_t ProtobufReader::get<uint8_t>()
+{
+  return (m_cur < m_end) ? *m_cur++ : 0;
+};
+
+template <>
+inline uint32_t ProtobufReader::get<uint32_t>()
+{
+  uint32_t v = 0;
+  for (uint32_t l = 0;; l += 7) {
+    uint32_t c = get<uint8_t>();
+    v |= (c & 0x7f) << l;
+    if (c < 0x80) return v;
+  }
+}
+
+template <>
+inline uint64_t ProtobufReader::get<uint64_t>()
+{
+  uint64_t v = 0;
+  for (uint64_t l = 0;; l += 7) {
+    uint64_t c = get<uint8_t>();
+    v |= (c & 0x7f) << l;
+    if (c < 0x80) return v;
+  }
+}
+
+template <typename T>
+auto decode_zigzag(T u)
+{
+  using signed_t = std::make_signed_t<T>;
+  return static_cast<signed_t>((u >> 1u) ^ -static_cast<signed_t>(u & 1));
+}
+
+template <>
+inline int32_t ProtobufReader::get<int32_t>()
+{
+  return decode_zigzag(get<uint32_t>());
+}
+
+template <>
+inline int64_t ProtobufReader::get<int64_t>()
+{
+  return decode_zigzag(get<uint64_t>());
+}
+
 /**
  * @brief Class for encoding Orc's metadata with Protocol Buffers
  */
@@ -279,6 +438,68 @@ class OrcDecompressor {
   std::vector<uint8_t> m_buf;
 };
 
+/**
+ * @brief A helper class for ORC file metadata. Provides some additional
+ * convenience methods for initializing and accessing metadata.
+ */
+class metadata {
+  using OrcStripeInfo = std::pair<const StripeInformation *, const StripeFooter *>;
+
+ public:
+  explicit metadata(datasource *const src);
+
+  /**
+   * @brief Filters and reads the info of only a selection of stripes
+   *
+   * @param[in] stripes Indices of individual stripes
+   * @param[in] row_start Starting row of the selection
+   * @param[in,out] row_count Total number of rows selected
+   *
+   * @return List of stripe info and total number of selected rows
+   */
+  std::vector<OrcStripeInfo> select_stripes(const std::vector<size_type> &stripes,
+                                            size_type &row_start,
+                                            size_type &row_count);
+
+  /**
+   * @brief Filters and reduces down to a selection of columns
+   *
+   * @param[in] use_names List of column names to select
+   * @param[out] has_timestamp_column Whether there is a orc::TIMESTAMP column
+   *
+   * @return List of ORC column indexes
+   */
+  std::vector<int> select_columns(std::vector<std::string> use_names, bool &has_timestamp_column);
+
+  size_t get_total_rows() const { return ff.numberOfRows; }
+  int get_num_stripes() const { return ff.stripes.size(); }
+  int get_num_columns() const { return ff.types.size(); }
+  std::string const &get_column_name(int32_t column_id)
+  {
+    if (column_names.empty() && get_num_columns() != 0) { init_column_names(); }
+    return column_names[column_id];
+  }
+  int get_row_index_stride() const { return ff.rowIndexStride; }
+
+ public:
+  PostScript ps;
+  FileFooter ff;
+  Metadata md;
+  std::vector<StripeFooter> stripefooters;
+  std::unique_ptr<OrcDecompressor> decompressor;
+
+ private:
+  struct schema_indexes {
+    int32_t parent = -1;
+    int32_t field  = -1;
+  };
+  std::vector<schema_indexes> get_schema_indexes() const;
+  void init_column_names();
+
+  std::vector<std::string> column_names;
+  datasource *const source;
+};
+
 }  // namespace orc
 }  // namespace io
 }  // namespace cudf
diff --git a/cpp/src/io/orc/orc_field_reader.hpp b/cpp/src/io/orc/orc_field_reader.hpp
index f15e554b500..9bb1ff4310b 100644
--- a/cpp/src/io/orc/orc_field_reader.hpp
+++ b/cpp/src/io/orc/orc_field_reader.hpp
@@ -37,22 +37,19 @@ namespace orc {
  *
  * If the field does not match any of the functors then skip_struct_field is
  * called by the ProtobufReader
- *
- * Returns the output of the selected operator or false if no operator
- * matched the field value
  */
 template <int index>
 struct FunctionSwitchImpl {
   template <typename... Operator>
-  static inline bool run(ProtobufReader *pbr,
+  static inline void run(ProtobufReader *pbr,
                          const uint8_t *end,
-                         const int &field,
+                         const int &encoded_field_number,
                          std::tuple<Operator...> &ops)
   {
-    if (field == std::get<index>(ops).field) {
-      return std::get<index>(ops)(pbr, end);
+    if (encoded_field_number == std::get<index>(ops).encoded_field_number) {
+      std::get<index>(ops)(pbr, end);
     } else {
-      return FunctionSwitchImpl<index - 1>::run(pbr, end, field, ops);
+      FunctionSwitchImpl<index - 1>::run(pbr, end, encoded_field_number, ops);
     }
   }
 };
@@ -60,292 +57,38 @@ struct FunctionSwitchImpl {
 template <>
 struct FunctionSwitchImpl<0> {
   template <typename... Operator>
-  static inline bool run(ProtobufReader *pbr,
+  static inline void run(ProtobufReader *pbr,
                          const uint8_t *end,
-                         const int &field,
+                         const int &encoded_field_number,
                          std::tuple<Operator...> &ops)
   {
-    if (field == std::get<0>(ops).field) {
-      return std::get<0>(ops)(pbr, end);
+    if (encoded_field_number == std::get<0>(ops).encoded_field_number) {
+      std::get<0>(ops)(pbr, end);
     } else {
-      pbr->skip_struct_field(field & 7);
-      return false;
+      pbr->skip_struct_field(encoded_field_number & 7);
     }
   }
 };
 
 /**
- * @brief Function to ascertain the return value of ProtobufReader::read
- * function
- *
- * @return Returns 'false' if current pointer to metadata stream is out of
- * bounds
- */
-template <typename T>
-inline bool ProtobufReader::function_builder_return(T &s, const uint8_t *end)
-{
-  return m_cur <= end;
-}
-
-/**
- * @brief Function to ascertain the return value of
- * `ProtobufReader::read(FileFooter*, ...)` function
- *
- * @return Returns 'false' if current pointer to metadata stream is out of
- * bounds or if the initialization of the parent_idx field of FileFooter
- * is not done correctly
- */
-template <>
-inline bool ProtobufReader::function_builder_return<FileFooter>(FileFooter &s, const uint8_t *end)
-{
-  return (m_cur <= end) && InitSchema(s);
-}
-
-/**
- * @brief Function to implement ProtobufReader::read based on the tuple of
- * functors provided
- *
- * Bytes are read from the internal metadata stream and field types are
- * matched up against user supplied reading functors. If they match then the
- * corresponding values are written to references pointed to by the functors.
+ * @brief Function to implement ProtobufReader::read based on the tuple of functors provided.
  *
- * @return Returns 'false' if an unexpected field is encountered while reading.
- * Otherwise 'true' is returned.
+ * Bytes are read from the internal metadata stream and field types are matched up against user
+ * supplied reading functors. If they match then the corresponding values are written to references
+ * pointed to by the functors.
  */
 template <typename T, typename... Operator>
-inline bool ProtobufReader::function_builder(T &s, size_t maxlen, std::tuple<Operator...> &op)
+inline void ProtobufReader::function_builder(T &s, size_t maxlen, std::tuple<Operator...> &op)
 {
   constexpr int index = std::tuple_size<std::tuple<Operator...>>::value - 1;
-  const uint8_t *end  = std::min(m_cur + maxlen, m_end);
+  auto *const end     = std::min(m_cur + maxlen, m_end);
   while (m_cur < end) {
-    int field          = get_u32();
-    bool exit_function = FunctionSwitchImpl<index>::run(this, end, field, op);
-    if (exit_function) { return false; }
+    auto const field = get<uint32_t>();
+    FunctionSwitchImpl<index>::run(this, end, field, op);
   }
-  return function_builder_return(s, end);
+  CUDF_EXPECTS(m_cur <= end, "Current pointer to metadata stream is out of bounds");
 }
 
-/**
- * @brief Functor to set value to 32 bit integer read from metadata stream
- *
- * Returns 'false'
- */
-struct ProtobufReader::FieldInt32 {
-  int field;
-  int32_t &value;
-
-  FieldInt32(int f, int32_t &v) : field((f * 8) + PB_TYPE_VARINT), value(v) {}
-
-  inline bool operator()(ProtobufReader *pbr, const uint8_t *end)
-  {
-    value = pbr->get_i32();
-    return false;
-  }
-};
-
-/**
- * @brief Functor to set value to 32 bit unsigned integer read from metadata
- * stream
- *
- * Returns 'false'
- */
-struct ProtobufReader::FieldUInt32 {
-  int field;
-  uint32_t &value;
-
-  FieldUInt32(int f, uint32_t &v) : field((f * 8) + PB_TYPE_VARINT), value(v) {}
-
-  inline bool operator()(ProtobufReader *pbr, const uint8_t *end)
-  {
-    value = pbr->get_u32();
-    return false;
-  }
-};
-
-/**
- * @brief Functor to set value to 64 bit integer read from metadata stream
- *
- * Returns 'false'
- */
-struct ProtobufReader::FieldInt64 {
-  int field;
-  int64_t &value;
-
-  FieldInt64(int f, int64_t &v) : field((f * 8) + PB_TYPE_VARINT), value(v) {}
-
-  inline bool operator()(ProtobufReader *pbr, const uint8_t *end)
-  {
-    value = pbr->get_i64();
-    return false;
-  }
-};
-
-/**
- * @brief Functor to set value to 64 bit unsigned integer read from metadata
- * stream
- *
- * Returns 'false'
- */
-struct ProtobufReader::FieldUInt64 {
-  int field;
-  uint64_t &value;
-
-  FieldUInt64(int f, uint64_t &v) : field((f * 8) + PB_TYPE_VARINT), value(v) {}
-
-  inline bool operator()(ProtobufReader *pbr, const uint8_t *end)
-  {
-    value = pbr->get_u64();
-    return false;
-  }
-};
-
-/**
- * @brief Functor to set value to enum read from metadata stream
- *
- * Returns 'false'
- */
-template <typename Enum>
-struct ProtobufReader::FieldEnum {
-  int field;
-  Enum &value;
-
-  FieldEnum(int f, Enum &v) : field((f * 8) + PB_TYPE_VARINT), value(v) {}
-
-  inline bool operator()(ProtobufReader *pbr, const uint8_t *end)
-  {
-    value = static_cast<Enum>(pbr->get_u32());
-    return false;
-  }
-};
-
-/**
- * @brief Functor to append a 32 bit integer to a vector of integers
- * read from metadata stream
- *
- * Returns 'false'
- */
-struct ProtobufReader::FieldPackedUInt32 {
-  int field;
-  std::vector<uint32_t> &value;
-
-  FieldPackedUInt32(int f, std::vector<uint32_t> &v) : field((f * 8) + PB_TYPE_FIXEDLEN), value(v)
-  {
-  }
-
-  inline bool operator()(ProtobufReader *pbr, const uint8_t *end)
-  {
-    uint32_t len             = pbr->get_u32();
-    const uint8_t *field_end = std::min(pbr->m_cur + len, end);
-    while (pbr->m_cur < field_end) value.push_back(pbr->get_u32());
-    return false;
-  }
-};
-
-/**
- * @brief Functor to set value to string read from metadata stream
- *
- * Returns 'true' if the length of the string exceeds bounds of the
- * metadata stream
- */
-struct ProtobufReader::FieldString {
-  int field;
-  std::string &value;
-
-  FieldString(int f, std::string &v) : field((f * 8) + PB_TYPE_FIXEDLEN), value(v) {}
-
-  inline bool operator()(ProtobufReader *pbr, const uint8_t *end)
-  {
-    uint32_t n = pbr->get_u32();
-    if (n > (size_t)(end - pbr->m_cur)) return true;
-    value.assign((const char *)(pbr->m_cur), n);
-    pbr->m_cur += n;
-    return false;
-  }
-};
-
-/**
- * @brief Functor to append a string read from metadata stream
- * to a vector of strings
- *
- * Returns 'true' if the length of the string exceeds bounds of the
- * metadata stream
- */
-struct ProtobufReader::FieldRepeatedString {
-  int field;
-  std::vector<std::string> &value;
-
-  FieldRepeatedString(int f, std::vector<std::string> &v)
-    : field((f * 8) + PB_TYPE_FIXEDLEN), value(v)
-  {
-  }
-
-  inline bool operator()(ProtobufReader *pbr, const uint8_t *end)
-  {
-    uint32_t n = pbr->get_u32();
-    if (n > (size_t)(end - pbr->m_cur)) return true;
-    value.resize(value.size() + 1);
-    value.back().assign((const char *)(pbr->m_cur), n);
-    pbr->m_cur += n;
-    return false;
-  }
-};
-
-/**
- * @brief Functor to append an enum read from metadata stream
- * to a vector of enums
- *
- * Returns 'true' if the maximum length read by the stream could
- * cause out of bounds read of the buffer or if the process of
- * reading the struct fails
- */
-template <typename Enum>
-struct ProtobufReader::FieldRepeatedStructFunctor {
-  int field;
-  std::vector<Enum> &value;
-
-  FieldRepeatedStructFunctor(int f, std::vector<Enum> &v)
-    : field((f * 8) + PB_TYPE_FIXEDLEN), value(v)
-  {
-  }
-
-  inline bool operator()(ProtobufReader *pbr, const uint8_t *end)
-  {
-    uint32_t n = pbr->get_u32();
-    if (n > (size_t)(end - pbr->m_cur)) return true;
-    value.resize(value.size() + 1);
-    if (!pbr->read(value.back(), n)) return true;
-    return false;
-  }
-};
-
-/**
- * @brief Functor to append an enum read from metadata stream
- * to a vector of enums
- *
- * Returns 'true' if the maximum length read by the stream could
- * cause out of bounds read of the buffer
- */
-template <typename Enum>
-struct ProtobufReader::FieldRepeatedStructBlobFunctor {
-  int field;
-  std::vector<Enum> &value;
-
-  FieldRepeatedStructBlobFunctor(int f, std::vector<Enum> &v)
-    : field((f * 8) + PB_TYPE_FIXEDLEN), value(v)
-  {
-  }
-
-  inline bool operator()(ProtobufReader *pbr, const uint8_t *end)
-  {
-    uint32_t n = pbr->get_u32();
-    if (n > (size_t)(end - pbr->m_cur)) return true;
-    value.resize(value.size() + 1);
-    value.back().assign(pbr->m_cur, pbr->m_cur + n);
-    pbr->m_cur += n;
-    return false;
-  }
-};
-
 }  // namespace orc
 }  // namespace io
 }  // namespace cudf
diff --git a/cpp/src/io/orc/reader_impl.cu b/cpp/src/io/orc/reader_impl.cu
index e2676a1dce0..26815f5dac0 100644
--- a/cpp/src/io/orc/reader_impl.cu
+++ b/cpp/src/io/orc/reader_impl.cu
@@ -23,6 +23,7 @@
 #include "timezone.cuh"
 
 #include <io/comp/gpuinflate.h>
+#include <io/orc/orc.h>
 
 #include <cudf/table/table.hpp>
 #include <cudf/utilities/error.hpp>
@@ -121,174 +122,6 @@ constexpr std::pair<gpu::StreamIndexType, uint32_t> get_index_type_and_pos(
 
 }  // namespace
 
-/**
- * @brief A helper class for ORC file metadata. Provides some additional
- * convenience methods for initializing and accessing metadata.
- */
-class metadata {
-  using OrcStripeInfo = std::pair<const StripeInformation *, const StripeFooter *>;
-
- public:
-  explicit metadata(datasource *const src) : source(src)
-  {
-    const auto len         = source->size();
-    const auto max_ps_size = std::min(len, static_cast<size_t>(256));
-
-    // Read uncompressed postscript section (max 255 bytes + 1 byte for length)
-    auto buffer            = source->host_read(len - max_ps_size, max_ps_size);
-    const size_t ps_length = buffer->data()[max_ps_size - 1];
-    const uint8_t *ps_data = &buffer->data()[max_ps_size - ps_length - 1];
-    ProtobufReader pb;
-    pb.init(ps_data, ps_length);
-    CUDF_EXPECTS(pb.read(ps, ps_length), "Cannot read postscript");
-    CUDF_EXPECTS(ps.footerLength + ps_length < len, "Invalid footer length");
-
-    // If compression is used, all the rest of the metadata is compressed
-    // If no compressed is used, the decompressor is simply a pass-through
-    decompressor = std::make_unique<OrcDecompressor>(ps.compression, ps.compressionBlockSize);
-
-    // Read compressed filefooter section
-    buffer           = source->host_read(len - ps_length - 1 - ps.footerLength, ps.footerLength);
-    size_t ff_length = 0;
-    auto ff_data     = decompressor->Decompress(buffer->data(), ps.footerLength, &ff_length);
-    pb.init(ff_data, ff_length);
-    CUDF_EXPECTS(pb.read(ff, ff_length), "Cannot read filefooter");
-    CUDF_EXPECTS(get_num_columns() > 0, "No columns found");
-  }
-
-  /**
-   * @brief Filters and reads the info of only a selection of stripes
-   *
-   * @param[in] stripes Indices of individual stripes
-   * @param[in] row_start Starting row of the selection
-   * @param[in,out] row_count Total number of rows selected
-   *
-   * @return List of stripe info and total number of selected rows
-   */
-  auto select_stripes(const std::vector<size_type> &stripes,
-                      size_type &row_start,
-                      size_type &row_count)
-  {
-    std::vector<OrcStripeInfo> selection;
-
-    if (!stripes.empty()) {
-      size_t stripe_rows = 0;
-      for (const auto &stripe_idx : stripes) {
-        CUDF_EXPECTS(stripe_idx >= 0 && stripe_idx < get_num_stripes(), "Invalid stripe index");
-        selection.emplace_back(&ff.stripes[stripe_idx], nullptr);
-        stripe_rows += ff.stripes[stripe_idx].numberOfRows;
-      }
-      // row_start is 0 if stripes are set. If this is not true anymore, then
-      // row_start needs to be subtracted to get the correct row_count
-      CUDF_EXPECTS(row_start == 0, "Start row index should be 0");
-      row_count = static_cast<size_type>(stripe_rows);
-    } else {
-      row_start = std::max(row_start, 0);
-      if (row_count < 0) {
-        row_count = static_cast<size_type>(
-          std::min<size_t>(get_total_rows() - row_start, std::numeric_limits<size_type>::max()));
-      } else {
-        row_count =
-          static_cast<size_type>(std::min<size_t>(get_total_rows() - row_start, row_count));
-      }
-      CUDF_EXPECTS(row_count >= 0 && row_start >= 0, "Negative row count or starting row");
-      CUDF_EXPECTS(
-        !(row_start > 0 && (row_count > (std::numeric_limits<size_type>::max() - row_start))),
-        "Summation of starting row index and number of rows would cause overflow");
-
-      size_type stripe_skip_rows = 0;
-      for (size_t i = 0, count = 0; i < ff.stripes.size(); ++i) {
-        count += ff.stripes[i].numberOfRows;
-        if (count > static_cast<size_t>(row_start)) {
-          if (selection.empty()) {
-            stripe_skip_rows =
-              static_cast<size_type>(row_start - (count - ff.stripes[i].numberOfRows));
-          }
-          selection.emplace_back(&ff.stripes[i], nullptr);
-        }
-        if (count >= static_cast<size_t>(row_start) + static_cast<size_t>(row_count)) { break; }
-      }
-      row_start = stripe_skip_rows;
-    }
-
-    // Read each stripe's stripefooter metadata
-    if (not selection.empty()) {
-      orc::ProtobufReader pb;
-
-      stripefooters.resize(selection.size());
-      for (size_t i = 0; i < selection.size(); ++i) {
-        const auto stripe         = selection[i].first;
-        const auto sf_comp_offset = stripe->offset + stripe->indexLength + stripe->dataLength;
-        const auto sf_comp_length = stripe->footerLength;
-        CUDF_EXPECTS(sf_comp_offset + sf_comp_length < source->size(),
-                     "Invalid stripe information");
-
-        const auto buffer = source->host_read(sf_comp_offset, sf_comp_length);
-        size_t sf_length  = 0;
-        auto sf_data      = decompressor->Decompress(buffer->data(), sf_comp_length, &sf_length);
-        pb.init(sf_data, sf_length);
-        CUDF_EXPECTS(pb.read(stripefooters[i], sf_length), "Cannot read stripefooter");
-        selection[i].second = &stripefooters[i];
-      }
-    }
-
-    return selection;
-  }
-
-  /**
-   * @brief Filters and reduces down to a selection of columns
-   *
-   * @param[in] use_names List of column names to select
-   * @param[out] has_timestamp_column Whether there is a orc::TIMESTAMP column
-   *
-   * @return List of ORC column indexes
-   */
-  auto select_columns(std::vector<std::string> use_names, bool &has_timestamp_column)
-  {
-    std::vector<int> selection;
-
-    if (not use_names.empty()) {
-      int index = 0;
-      for (const auto &use_name : use_names) {
-        for (int i = 0; i < get_num_columns(); ++i, ++index) {
-          if (index >= get_num_columns()) { index = 0; }
-          if (ff.GetColumnName(index) == use_name) {
-            selection.emplace_back(index);
-            if (ff.types[index].kind == orc::TIMESTAMP) { has_timestamp_column = true; }
-            index++;
-            break;
-          }
-        }
-      }
-    } else {
-      // For now, only select all leaf nodes
-      for (int i = 0; i < get_num_columns(); ++i) {
-        if (ff.types[i].subtypes.empty()) {
-          selection.emplace_back(i);
-          if (ff.types[i].kind == orc::TIMESTAMP) { has_timestamp_column = true; }
-        }
-      }
-    }
-    CUDF_EXPECTS(selection.size() > 0, "Filtered out all columns");
-
-    return selection;
-  }
-
-  inline size_t get_total_rows() const { return ff.numberOfRows; }
-  inline int get_num_stripes() const { return ff.stripes.size(); }
-  inline int get_num_columns() const { return ff.types.size(); }
-  inline int get_row_index_stride() const { return ff.rowIndexStride; }
-
- public:
-  PostScript ps;
-  FileFooter ff;
-  std::vector<StripeFooter> stripefooters;
-  std::unique_ptr<OrcDecompressor> decompressor;
-
- private:
-  datasource *const source;
-};
-
 namespace {
 /**
  * @brief Struct that maps ORC streams to columns
@@ -594,7 +427,7 @@ reader::impl::impl(std::unique_ptr<datasource> source,
   : _mr(mr), _source(std::move(source))
 {
   // Open and parse the source dataset metadata
-  _metadata = std::make_unique<metadata>(_source.get());
+  _metadata = std::make_unique<cudf::io::orc::metadata>(_source.get());
 
   // Select only columns required by the options
   _selected_columns = _metadata->select_columns(options.get_columns(), _has_timestamp_column);
@@ -815,7 +648,7 @@ table_with_metadata reader::impl::read(size_type skip_rows,
   // Return column names (must match order of returned columns)
   out_metadata.column_names.resize(_selected_columns.size());
   for (size_t i = 0; i < _selected_columns.size(); i++) {
-    out_metadata.column_names[i] = _metadata->ff.GetColumnName(_selected_columns[i]);
+    out_metadata.column_names[i] = _metadata->get_column_name(_selected_columns[i]);
   }
   // Return user metadata
   for (const auto &kv : _metadata->ff.metadata) {
diff --git a/cpp/src/io/orc/reader_impl.hpp b/cpp/src/io/orc/reader_impl.hpp
index 4684dbdcf96..818e70b15e7 100644
--- a/cpp/src/io/orc/reader_impl.hpp
+++ b/cpp/src/io/orc/reader_impl.hpp
@@ -127,7 +127,7 @@ class reader::impl {
  private:
   rmm::mr::device_memory_resource *_mr = nullptr;
   std::unique_ptr<datasource> _source;
-  std::unique_ptr<metadata> _metadata;
+  std::unique_ptr<cudf::io::orc::metadata> _metadata;
 
   std::vector<int> _selected_columns;
   bool _use_index                  = true;

From 6828e2ce559569c55b8f04e660a93ab4c21f25b4 Mon Sep 17 00:00:00 2001
From: David <45795991+davidwendt@users.noreply.github.com>
Date: Tue, 5 Jan 2021 16:07:28 -0500
Subject: [PATCH 08/20] Add dictionary support to libcudf groupby
 functions(#6585)

Reference #5963 Add dictionary support to groupby.

- [x] argmax
- [x] argmin
- [x] collect
- [x] count
- [x] max
- [x] mean*
- [x] median
- [x] min
- [x] nth element
- [x] nunique
- [x] quantile
- [x] std*
- [x] sum*
- [x] var*

* _not supported due to 10.2 compile segfault_

Authors:
  - davidwendt <dwendt@nvidia.com>

Approvers:
  - Jake Hemstad
  - Karthikeyan

URL: https://github.com/rapidsai/cudf/pull/6585
---
 CHANGELOG.md                                  |   1 +
 .../cudf/detail/aggregation/aggregation.cuh   |  59 ++++++++
 cpp/include/cudf_test/column_wrapper.hpp      |  10 ++
 cpp/src/groupby/groupby.cu                    |  45 ++++--
 cpp/src/groupby/hash/groupby.cu               |  24 +++-
 cpp/src/groupby/hash/multi_pass_kernels.cuh   |  17 ++-
 cpp/src/groupby/sort/group_quantiles.cu       | 131 +++++++++++-------
 .../sort/group_single_pass_reduction_util.cuh |  30 ++--
 cpp/src/groupby/sort/group_std.cu             | 103 +++++++++-----
 cpp/src/groupby/sort/group_sum.cu             |  13 +-
 cpp/tests/groupby/group_argmax_test.cpp       |  26 ++++
 cpp/tests/groupby/group_argmin_test.cpp       |  25 ++++
 cpp/tests/groupby/group_collect_test.cpp      |  21 +++
 cpp/tests/groupby/group_count_test.cpp        |  23 ++-
 cpp/tests/groupby/group_max_test.cpp          |  29 +++-
 cpp/tests/groupby/group_mean_test.cpp         |  35 ++++-
 cpp/tests/groupby/group_median_test.cpp       |  21 ++-
 cpp/tests/groupby/group_min_test.cpp          |  29 +++-
 cpp/tests/groupby/group_nth_element_test.cpp  |  22 ++-
 cpp/tests/groupby/group_nunique_test.cpp      |  27 ++++
 cpp/tests/groupby/group_quantile_test.cpp     |  25 +++-
 cpp/tests/groupby/group_std_test.cpp          |  23 ++-
 cpp/tests/groupby/group_sum_test.cpp          |  23 ++-
 cpp/tests/groupby/group_var_test.cpp          |  23 ++-
 cpp/tests/groupby/groupby_keys_test.cpp       |  23 ++-
 25 files changed, 666 insertions(+), 142 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index a7332b5c2f1..e895a657de8 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -17,6 +17,7 @@
 - PR #6275 Update to official libcu++ on Github
 - PR #6838 Fix `columns` & `index` handling in dataframe constructor
 - PR #6750 Remove **kwargs from string/categorical methods
+- PR #6585 Add dictionary support to libcudf groupby functions
 - PR #6909 Support reading byte array backed decimal columns from parquet files
 - PR #6939 Use simplified `rmm::exec_policy`
 - PR #6512 Refactor rolling.cu to reduce compile time
diff --git a/cpp/include/cudf/detail/aggregation/aggregation.cuh b/cpp/include/cudf/detail/aggregation/aggregation.cuh
index 04326f3a61a..aece4d53faf 100644
--- a/cpp/include/cudf/detail/aggregation/aggregation.cuh
+++ b/cpp/include/cudf/detail/aggregation/aggregation.cuh
@@ -21,6 +21,7 @@
 #include <cudf/detail/aggregation/aggregation.hpp>
 #include <cudf/detail/utilities/device_atomics.cuh>
 #include <cudf/detail/utilities/release_assert.cuh>
+#include <cudf/dictionary/dictionary_column_view.hpp>
 #include <cudf/table/table_device_view.cuh>
 
 #include <rmm/cuda_stream_view.hpp>
@@ -172,6 +173,64 @@ struct update_target_element<
   }
 };
 
+/**
+ * @brief Function object to update a single element in a target column using
+ * the dictionary key addressed by the specific index.
+ *
+ * `target[target_index] = d_dictionary.keys[d_dictionary.indices[source_index]]`
+ */
+struct update_target_from_dictionary {
+  template <typename KeyType,
+            std::enable_if_t<is_fixed_width<KeyType>() && !is_fixed_point<KeyType>()>* = nullptr>
+  __device__ void operator()(mutable_column_device_view& target,
+                             size_type target_index,
+                             column_device_view& d_dictionary,
+                             size_type source_index) const noexcept
+  {
+// This code will segfault in nvcc/ptxas 10.2 only
+// https://nvbugswb.nvidia.com/NvBugs5/SWBug.aspx?bugid=3186317
+#if (__CUDACC_VER_MAJOR__ != 10) or (__CUDACC_VER_MINOR__ != 2)
+    auto const keys  = d_dictionary.child(cudf::dictionary_column_view::keys_column_index);
+    auto const value = keys.element<KeyType>(
+      static_cast<cudf::size_type>(d_dictionary.element<dictionary32>(source_index)));
+    using Target = target_type_t<KeyType, aggregation::SUM>;
+    atomicAdd(&target.element<Target>(target_index), static_cast<Target>(value));
+#endif
+  }
+  template <typename KeyType,
+            std::enable_if_t<!is_fixed_width<KeyType>() || is_fixed_point<KeyType>()>* = nullptr>
+  __device__ void operator()(mutable_column_device_view& target,
+                             size_type target_index,
+                             column_device_view& d_dictionary,
+                             size_type source_index) const noexcept {};
+};
+
+/**
+ * @brief Specialization function for dictionary type and aggregation SUM.
+ *
+ * @tparam target_has_nulls Indicates presence of null elements in `target`
+ * @tparam source_has_nulls Indicates presence of null elements in `source`.
+ */
+template <bool target_has_nulls, bool source_has_nulls>
+struct update_target_element<dictionary32, aggregation::SUM, target_has_nulls, source_has_nulls> {
+  __device__ void operator()(mutable_column_device_view target,
+                             size_type target_index,
+                             column_device_view source,
+                             size_type source_index) const noexcept
+  {
+    if (source_has_nulls and source.is_null(source_index)) { return; }
+
+    type_dispatcher(source.child(cudf::dictionary_column_view::keys_column_index).type(),
+                    update_target_from_dictionary{},
+                    target,
+                    target_index,
+                    source,
+                    source_index);
+
+    if (target_has_nulls and target.is_null(target_index)) { target.set_valid(target_index); }
+  }
+};
+
 template <typename Source, bool target_has_nulls, bool source_has_nulls>
 struct update_target_element<
   Source,
diff --git a/cpp/include/cudf_test/column_wrapper.hpp b/cpp/include/cudf_test/column_wrapper.hpp
index 54a51f08798..0b10c76a256 100644
--- a/cpp/include/cudf_test/column_wrapper.hpp
+++ b/cpp/include/cudf_test/column_wrapper.hpp
@@ -1053,6 +1053,16 @@ class dictionary_column_wrapper<std::string> : public detail::column_wrapper {
    */
   operator dictionary_column_view() const { return cudf::dictionary_column_view{wrapped->view()}; }
 
+  /**
+   * @brief Access keys column view
+   */
+  column_view keys() const { return cudf::dictionary_column_view{wrapped->view()}.keys(); }
+
+  /**
+   * @brief Access indices column view
+   */
+  column_view indices() const { return cudf::dictionary_column_view{wrapped->view()}.indices(); }
+
   /**
    * @brief Default constructor initializes an empty dictionary column of strings
    */
diff --git a/cpp/src/groupby/groupby.cu b/cpp/src/groupby/groupby.cu
index 862cbcf5e82..3166b2be4d4 100644
--- a/cpp/src/groupby/groupby.cu
+++ b/cpp/src/groupby/groupby.cu
@@ -23,6 +23,7 @@
 #include <cudf/detail/groupby.hpp>
 #include <cudf/detail/groupby/sort_helper.hpp>
 #include <cudf/detail/nvtx/ranges.hpp>
+#include <cudf/dictionary/dictionary_column_view.hpp>
 #include <cudf/groupby.hpp>
 #include <cudf/table/table.hpp>
 #include <cudf/table/table_view.hpp>
@@ -103,17 +104,39 @@ auto empty_results(std::vector<aggregation_request> const& requests)
 /// Verifies the agg requested on the request's values is valid
 void verify_valid_requests(std::vector<aggregation_request> const& requests)
 {
-  CUDF_EXPECTS(std::all_of(requests.begin(),
-                           requests.end(),
-                           [](auto const& request) {
-                             return std::all_of(request.aggregations.begin(),
-                                                request.aggregations.end(),
-                                                [&request](auto const& agg) {
-                                                  return cudf::detail::is_valid_aggregation(
-                                                    request.values.type(), agg->kind);
-                                                });
-                           }),
-               "Invalid type/aggregation combination.");
+  CUDF_EXPECTS(
+    std::all_of(
+      requests.begin(),
+      requests.end(),
+      [](auto const& request) {
+        return std::all_of(
+          request.aggregations.begin(), request.aggregations.end(), [&request](auto const& agg) {
+            auto values_type = cudf::is_dictionary(request.values.type())
+                                 ? cudf::dictionary_column_view(request.values).keys().type()
+                                 : request.values.type();
+            return cudf::detail::is_valid_aggregation(values_type, agg->kind);
+          });
+      }),
+    "Invalid type/aggregation combination.");
+
+// The aggregations listed in the lambda below will not work with a values column of type
+// dictionary if this is compiled with nvcc/ptxas 10.2.
+// https://nvbugswb.nvidia.com/NvBugs5/SWBug.aspx?bugid=3186317&cp=
+#if (__CUDACC_VER_MAJOR__ == 10) and (__CUDACC_VER_MINOR__ == 2)
+  CUDF_EXPECTS(
+    std::all_of(
+      requests.begin(),
+      requests.end(),
+      [](auto const& request) {
+        return std::all_of(
+          request.aggregations.begin(), request.aggregations.end(), [&request](auto const& agg) {
+            return (!cudf::is_dictionary(request.values.type()) ||
+                    !(agg->kind == aggregation::SUM or agg->kind == aggregation::MEAN or
+                      agg->kind == aggregation::STD or agg->kind == aggregation::VARIANCE));
+          });
+      }),
+    "dictionary type not supported for this aggregation");
+#endif
 }
 
 }  // namespace
diff --git a/cpp/src/groupby/hash/groupby.cu b/cpp/src/groupby/hash/groupby.cu
index 7520472aa3b..e22d355f8a8 100644
--- a/cpp/src/groupby/hash/groupby.cu
+++ b/cpp/src/groupby/hash/groupby.cu
@@ -33,6 +33,7 @@
 #include <cudf/detail/unary.hpp>
 #include <cudf/detail/utilities/cuda.cuh>
 #include <cudf/detail/utilities/hash_functions.cuh>
+#include <cudf/dictionary/dictionary_column_view.hpp>
 #include <cudf/groupby.hpp>
 #include <cudf/scalar/scalar.hpp>
 #include <cudf/table/row_operators.cuh>
@@ -104,6 +105,7 @@ template <typename Map>
 class hash_compound_agg_finalizer final : public cudf::detail::aggregation_finalizer {
   size_t col_idx;
   column_view col;
+  data_type result_type;
   cudf::detail::result_cache* sparse_results;
   cudf::detail::result_cache* dense_results;
   rmm::device_vector<size_type> const& gather_map;
@@ -135,6 +137,8 @@ class hash_compound_agg_finalizer final : public cudf::detail::aggregation_final
       stream(stream),
       mr(mr)
   {
+    result_type = cudf::is_dictionary(col.type()) ? cudf::dictionary_column_view(col).keys().type()
+                                                  : col.type();
   }
 
   auto to_dense_agg_result(cudf::aggregation const& agg)
@@ -184,7 +188,7 @@ class hash_compound_agg_finalizer final : public cudf::detail::aggregation_final
   void visit(cudf::detail::min_aggregation const& agg) override
   {
     if (dense_results->has_result(col_idx, agg)) return;
-    if (col.type().id() == type_id::STRING)
+    if (result_type.id() == type_id::STRING)
       dense_results->add_result(col_idx, agg, gather_argminmax(aggregation::ARGMIN));
     else
       dense_results->add_result(col_idx, agg, to_dense_agg_result(agg));
@@ -194,7 +198,7 @@ class hash_compound_agg_finalizer final : public cudf::detail::aggregation_final
   {
     if (dense_results->has_result(col_idx, agg)) return;
 
-    if (col.type().id() == type_id::STRING)
+    if (result_type.id() == type_id::STRING)
       dense_results->add_result(col_idx, agg, gather_argminmax(aggregation::ARGMAX));
     else
       dense_results->add_result(col_idx, agg, to_dense_agg_result(agg));
@@ -215,7 +219,7 @@ class hash_compound_agg_finalizer final : public cudf::detail::aggregation_final
       cudf::detail::binary_operation(sum_result,
                                      count_result,
                                      binary_operator::DIV,
-                                     cudf::detail::target_type(col.type(), aggregation::MEAN),
+                                     cudf::detail::target_type(result_type, aggregation::MEAN),
                                      stream,
                                      mr);
     dense_results->add_result(col_idx, agg, std::move(result));
@@ -237,7 +241,7 @@ class hash_compound_agg_finalizer final : public cudf::detail::aggregation_final
     auto count_view  = column_device_view::create(count_result);
 
     auto var_result = make_fixed_width_column(
-      cudf::detail::target_type(col.type(), agg.kind), col.size(), mask_state::ALL_NULL, stream);
+      cudf::detail::target_type(result_type, agg.kind), col.size(), mask_state::ALL_NULL, stream);
     auto var_result_view = mutable_column_device_view::create(var_result->mutable_view());
     mutable_table_view var_table_view{{var_result->mutable_view()}};
     cudf::detail::initialize_with_identity(var_table_view, {agg.kind}, stream);
@@ -285,11 +289,15 @@ flatten_single_pass_aggs(std::vector<aggregation_request> const& requests)
       }
     };
 
+    auto values_type = cudf::is_dictionary(request.values.type())
+                         ? cudf::dictionary_column_view(request.values).keys().type()
+                         : request.values.type();
     for (auto&& agg : agg_v) {
-      for (auto const& agg_s : agg->get_simple_aggregations(request.values.type()))
+      for (auto const& agg_s : agg->get_simple_aggregations(values_type))
         insert_agg(i, request.values, agg_s);
     }
   }
+
   return std::make_tuple(table_view(columns), std::move(agg_kinds), std::move(col_ids));
 }
 
@@ -389,8 +397,12 @@ auto create_sparse_results_table(table_view const& flattened_values,
           : (col.has_nulls() or agg == aggregation::VARIANCE or agg == aggregation::STD);
       auto mask_flag = (nullable) ? mask_state::ALL_NULL : mask_state::UNALLOCATED;
 
+      auto col_type = cudf::is_dictionary(col.type())
+                        ? cudf::dictionary_column_view(col).keys().type()
+                        : col.type();
+
       return make_fixed_width_column(
-        cudf::detail::target_type(col.type(), agg), col.size(), mask_flag, stream);
+        cudf::detail::target_type(col_type, agg), col.size(), mask_flag, stream);
     });
 
   table sparse_table(std::move(sparse_columns));
diff --git a/cpp/src/groupby/hash/multi_pass_kernels.cuh b/cpp/src/groupby/hash/multi_pass_kernels.cuh
index 8c7d4675cf0..a491b50478a 100644
--- a/cpp/src/groupby/hash/multi_pass_kernels.cuh
+++ b/cpp/src/groupby/hash/multi_pass_kernels.cuh
@@ -61,14 +61,16 @@ struct var_hash_functor {
   }
 
   template <typename Source>
-  __device__ std::enable_if_t<!is_supported<Source>()> operator()(size_type source_index,
+  __device__ std::enable_if_t<!is_supported<Source>()> operator()(column_device_view const& source,
+                                                                  size_type source_index,
                                                                   size_type target_index) noexcept
   {
     release_assert(false and "Invalid source type for std, var aggregation combination.");
   }
 
   template <typename Source>
-  __device__ std::enable_if_t<is_supported<Source>()> operator()(size_type source_index,
+  __device__ std::enable_if_t<is_supported<Source>()> operator()(column_device_view const& source,
+                                                                 size_type source_index,
                                                                  size_type target_index) noexcept
   {
     using Target    = target_type_t<Source, aggregation::VARIANCE>;
@@ -92,7 +94,16 @@ struct var_hash_functor {
     if (row_bitmask == nullptr or cudf::bit_is_set(row_bitmask, source_index)) {
       auto result       = map.find(source_index);
       auto target_index = result->second;
-      type_dispatcher(source.type(), *this, source_index, target_index);
+
+      auto col         = source;
+      auto source_type = source.type();
+      if (source_type.id() == type_id::DICTIONARY32) {
+        col          = source.child(cudf::dictionary_column_view::keys_column_index);
+        source_type  = col.type();
+        source_index = static_cast<size_type>(source.element<dictionary32>(source_index));
+      }
+
+      type_dispatcher(source_type, *this, col, source_index, target_index);
     }
   }
 };
diff --git a/cpp/src/groupby/sort/group_quantiles.cu b/cpp/src/groupby/sort/group_quantiles.cu
index 5d736fec131..d5af2de0f7a 100644
--- a/cpp/src/groupby/sort/group_quantiles.cu
+++ b/cpp/src/groupby/sort/group_quantiles.cu
@@ -14,26 +14,64 @@
  * limitations under the License.
  */
 
-#include "group_reductions.hpp"
-
-#include <quantiles/quantiles_util.hpp>
-
 #include <cudf/column/column_device_view.cuh>
 #include <cudf/column/column_factories.hpp>
 #include <cudf/column/column_view.hpp>
 #include <cudf/detail/aggregation/aggregation.hpp>
-#include <cudf/types.hpp>
+#include <cudf/dictionary/detail/iterator.cuh>
+#include <cudf/dictionary/dictionary_column_view.hpp>
 
+#include <groupby/sort/group_reductions.hpp>
+#include <quantiles/quantiles_util.hpp>
 #include <rmm/cuda_stream_view.hpp>
 #include <rmm/device_vector.hpp>
 #include <rmm/exec_policy.hpp>
 
+#include <rmm/thrust_rmm_allocator.h>
 #include <thrust/for_each.h>
 
 namespace cudf {
 namespace groupby {
 namespace detail {
 namespace {
+
+template <typename ResultType, typename Iterator>
+struct calculate_quantile_fn {
+  Iterator values_iter;
+  column_device_view d_group_size;
+  mutable_column_device_view d_result;
+  size_type const* d_group_offset;
+  double const* d_quantiles;
+  size_type num_quantiles;
+  interpolation interpolation;
+
+  __device__ void operator()(size_type i)
+  {
+    size_type segment_size = d_group_size.element<size_type>(i);
+
+    auto d_itr = values_iter + d_group_offset[i];
+    thrust::transform(thrust::seq,
+                      d_quantiles,
+                      d_quantiles + num_quantiles,
+                      d_result.data<ResultType>() + i * num_quantiles,
+                      [d_itr, segment_size, interpolation = interpolation](auto q) {
+                        return cudf::detail::select_quantile_data<ResultType>(
+                          d_itr, segment_size, q, interpolation);
+                      });
+
+    size_type offset = i * num_quantiles;
+    thrust::for_each_n(thrust::seq,
+                       thrust::make_counting_iterator(0),
+                       num_quantiles,
+                       [d_result = d_result, segment_size, offset](size_type j) {
+                         if (segment_size == 0)
+                           d_result.set_null(offset + j);
+                         else
+                           d_result.set_valid(offset + j);
+                       });
+  }
+};
+
 struct quantiles_functor {
   template <typename T>
   std::enable_if_t<std::is_arithmetic<T>::value, std::unique_ptr<column>> operator()(
@@ -58,55 +96,38 @@ struct quantiles_functor {
     //            so that sorting isn't required. Then add support for pre-sorted
 
     // prepare args to be used by lambda below
-    auto values_view     = column_device_view::create(values);
-    auto group_size_view = column_device_view::create(group_sizes);
-    auto result_view     = mutable_column_device_view::create(result->mutable_view());
+    auto values_view     = column_device_view::create(values, stream);
+    auto group_size_view = column_device_view::create(group_sizes, stream);
+    auto result_view     = mutable_column_device_view::create(result->mutable_view(), stream);
 
     // For each group, calculate quantile
-    thrust::for_each_n(rmm::exec_policy(stream),
-                       thrust::make_counting_iterator(0),
-                       num_groups,
-                       [d_values       = *values_view,
-                        d_group_size   = *group_size_view,
-                        d_result       = *result_view,
-                        d_group_offset = group_offsets.data().get(),
-                        d_quantiles    = quantile.data().get(),
-                        num_quantiles  = quantile.size(),
-                        interpolation] __device__(size_type i) {
-                         size_type segment_size = d_group_size.element<size_type>(i);
-
-                         auto selector = [i, &d_values, d_group_offset](size_type j) {
-                           return d_values.element<T>(d_group_offset[i] + j);
-                         };
-
-                         thrust::transform(thrust::seq,
-                                           d_quantiles,
-                                           d_quantiles + num_quantiles,
-                                           d_result.data<ResultType>() + i * num_quantiles,
-                                           [selector, segment_size, interpolation](auto q) {
-                                             return cudf::detail::select_quantile<ResultType>(
-                                               selector, segment_size, q, interpolation);
-                                           });
-
-                         for (size_t j = 0; j < num_quantiles; j++) {
-                           size_type group_size = d_group_size.element<size_type>(i);
-                           if (group_size == 0)
-                             d_result.set_null(i * num_quantiles + j);
-                           else
-                             d_result.set_valid(i * num_quantiles + j);
-                         }
-
-                         // DEAR REVIEWER, would you prefer I do this instead of the loop above?
-                         // thrust::for_each_n(thrust::seq,
-                         //   thrust::make_counting_iterator(0), num_quantiles,
-                         //   [&d_result, &d_group_size, num_quantiles, i] (size_type j) {
-                         //     size_type group_size = d_group_size.element<size_type>(i);
-                         //     if (group_size == 0)
-                         //       d_result.set_null(i * num_quantiles + j);
-                         //     else
-                         //       d_result.set_valid(i * num_quantiles + j);
-                         //   });
-                       });
+    if (!cudf::is_dictionary(values.type())) {
+      auto values_iter = values_view->begin<T>();
+      thrust::for_each_n(rmm::exec_policy(stream),
+                         thrust::make_counting_iterator(0),
+                         num_groups,
+                         calculate_quantile_fn<ResultType, decltype(values_iter)>{
+                           values_iter,
+                           *group_size_view,
+                           *result_view,
+                           group_offsets.data().get(),
+                           quantile.data().get(),
+                           static_cast<size_type>(quantile.size()),
+                           interpolation});
+    } else {
+      auto values_iter = cudf::dictionary::detail::make_dictionary_iterator<T>(*values_view);
+      thrust::for_each_n(rmm::exec_policy(stream),
+                         thrust::make_counting_iterator(0),
+                         num_groups,
+                         calculate_quantile_fn<ResultType, decltype(values_iter)>{
+                           values_iter,
+                           *group_size_view,
+                           *result_view,
+                           group_offsets.data().get(),
+                           quantile.data().get(),
+                           static_cast<size_type>(quantile.size()),
+                           interpolation});
+    }
 
     return result;
   }
@@ -133,7 +154,11 @@ std::unique_ptr<column> group_quantiles(column_view const& values,
 {
   rmm::device_vector<double> dv_quantiles(quantiles);
 
-  return type_dispatcher(values.type(),
+  auto values_type = cudf::is_dictionary(values.type())
+                       ? dictionary_column_view(values).keys().type()
+                       : values.type();
+
+  return type_dispatcher(values_type,
                          quantiles_functor{},
                          values,
                          group_sizes,
diff --git a/cpp/src/groupby/sort/group_single_pass_reduction_util.cuh b/cpp/src/groupby/sort/group_single_pass_reduction_util.cuh
index 8d26e472cfc..7fc80c14171 100644
--- a/cpp/src/groupby/sort/group_single_pass_reduction_util.cuh
+++ b/cpp/src/groupby/sort/group_single_pass_reduction_util.cuh
@@ -76,15 +76,27 @@ struct reduce_functor {
     auto resultview = mutable_column_device_view::create(result->mutable_view(), stream);
     auto valuesview = column_device_view::create(values, stream);
 
-    thrust::for_each_n(rmm::exec_policy(stream),
-                       thrust::make_counting_iterator(0),
-                       values.size(),
-                       [d_values     = *valuesview,
-                        d_result     = *resultview,
-                        dest_indices = group_labels.data().get()] __device__(auto i) {
-                         cudf::detail::update_target_element<T, K, true, true>{}(
-                           d_result, dest_indices[i], d_values, i);
-                       });
+    if (!cudf::is_dictionary(values.type())) {
+      thrust::for_each_n(rmm::exec_policy(stream),
+                         thrust::make_counting_iterator(0),
+                         values.size(),
+                         [d_values     = *valuesview,
+                          d_result     = *resultview,
+                          dest_indices = group_labels.data().get()] __device__(auto i) {
+                           cudf::detail::update_target_element<T, K, true, true>{}(
+                             d_result, dest_indices[i], d_values, i);
+                         });
+    } else {
+      thrust::for_each_n(rmm::exec_policy(stream),
+                         thrust::make_counting_iterator(0),
+                         values.size(),
+                         [d_values     = *valuesview,
+                          d_result     = *resultview,
+                          dest_indices = group_labels.data().get()] __device__(auto i) {
+                           cudf::detail::update_target_element<dictionary32, K, true, true>{}(
+                             d_result, dest_indices[i], d_values, i);
+                         });
+    }
 
     return result;
   }
diff --git a/cpp/src/groupby/sort/group_std.cu b/cpp/src/groupby/sort/group_std.cu
index 6524268e8c0..eb85932d8eb 100644
--- a/cpp/src/groupby/sort/group_std.cu
+++ b/cpp/src/groupby/sort/group_std.cu
@@ -20,6 +20,8 @@
 #include <cudf/column/column_factories.hpp>
 #include <cudf/column/column_view.hpp>
 #include <cudf/detail/aggregation/aggregation.hpp>
+#include <cudf/dictionary/detail/iterator.cuh>
+#include <cudf/dictionary/dictionary_column_view.hpp>
 #include <cudf/utilities/type_dispatcher.hpp>
 
 #include <rmm/cuda_stream_view.hpp>
@@ -33,11 +35,13 @@ namespace cudf {
 namespace groupby {
 namespace detail {
 namespace {
-template <typename ResultType, typename T>
+
+template <typename ResultType, typename Iterator>
 struct var_transform {
-  column_device_view d_values;
-  column_device_view d_means;
-  column_device_view d_group_sizes;
+  column_device_view const d_values;
+  Iterator values_iter;
+  ResultType const* d_means;
+  size_type const* d_group_sizes;
   size_type const* d_group_labels;
   size_type ddof;
 
@@ -45,18 +49,42 @@ struct var_transform {
   {
     if (d_values.is_null(i)) return 0.0;
 
-    ResultType x         = d_values.element<T>(i);
+    ResultType x = static_cast<ResultType>(values_iter[i]);
+
     size_type group_idx  = d_group_labels[i];
-    size_type group_size = d_group_sizes.element<size_type>(group_idx);
+    size_type group_size = d_group_sizes[group_idx];
 
     // prevent divide by zero error
     if (group_size == 0 or group_size - ddof <= 0) return 0.0;
 
-    ResultType mean = d_means.element<ResultType>(group_idx);
+    ResultType mean = d_means[group_idx];
     return (x - mean) * (x - mean) / (group_size - ddof);
   }
 };
 
+template <typename ResultType, typename Iterator>
+void reduce_by_key_fn(column_device_view const& values,
+                      Iterator values_iter,
+                      rmm::device_vector<size_type> const& group_labels,
+                      ResultType const* d_means,
+                      size_type const* d_group_sizes,
+                      size_type ddof,
+                      ResultType* d_result,
+                      rmm::cuda_stream_view stream)
+{
+  auto var_iter = thrust::make_transform_iterator(
+    thrust::make_counting_iterator(0),
+    var_transform<ResultType, decltype(values_iter)>{
+      values, values_iter, d_means, d_group_sizes, group_labels.data().get(), ddof});
+
+  thrust::reduce_by_key(rmm::exec_policy(stream),
+                        group_labels.begin(),
+                        group_labels.end(),
+                        var_iter,
+                        thrust::make_discard_iterator(),
+                        d_result);
+}
+
 struct var_functor {
   template <typename T>
   std::enable_if_t<std::is_arithmetic<T>::value, std::unique_ptr<column>> operator()(
@@ -71,11 +99,7 @@ struct var_functor {
 // Running this in debug build causes a runtime error:
 // `reduce_by_key failed on 2nd step: invalid device function`
 #if !defined(__CUDACC_DEBUG__)
-    using ResultType                = cudf::detail::target_type_t<T, aggregation::Kind::VARIANCE>;
-    size_type const* d_group_labels = group_labels.data().get();
-    auto values_view                = column_device_view::create(values, stream);
-    auto means_view                 = column_device_view::create(group_means, stream);
-    auto group_size_view            = column_device_view::create(group_sizes, stream);
+    using ResultType = cudf::detail::target_type_t<T, aggregation::Kind::VARIANCE>;
 
     std::unique_ptr<column> result = make_numeric_column(data_type(type_to_id<ResultType>()),
                                                          group_sizes.size(),
@@ -83,35 +107,36 @@ struct var_functor {
                                                          stream,
                                                          mr);
 
-    auto d_values      = *values_view;
-    auto d_means       = *means_view;
-    auto d_group_sizes = *group_size_view;
+    auto values_view = column_device_view::create(values, stream);
+    auto d_values    = *values_view;
 
-    auto values_it = thrust::make_transform_iterator(
-      thrust::make_counting_iterator(0),
-      var_transform<ResultType, T>{d_values, d_means, d_group_sizes, d_group_labels, ddof});
+    auto d_group_labels = group_labels.data().get();
+    auto d_means        = group_means.data<ResultType>();
+    auto d_group_sizes  = group_sizes.data<size_type>();
+    auto d_result       = result->mutable_view().data<ResultType>();
 
-    thrust::reduce_by_key(rmm::exec_policy(stream),
-                          group_labels.begin(),
-                          group_labels.end(),
-                          values_it,
-                          thrust::make_discard_iterator(),
-                          result->mutable_view().data<ResultType>());
+    if (!cudf::is_dictionary(values.type())) {
+      auto values_iter = d_values.begin<T>();
+      reduce_by_key_fn(
+        d_values, values_iter, group_labels, d_means, d_group_sizes, ddof, d_result, stream);
+    } else {
+      auto values_iter = cudf::dictionary::detail::make_dictionary_iterator<T>(*values_view);
+      reduce_by_key_fn(
+        d_values, values_iter, group_labels, d_means, d_group_sizes, ddof, d_result, stream);
+    }
 
     // set nulls
     auto result_view = mutable_column_device_view::create(*result, stream);
-
-    thrust::for_each_n(
-      rmm::exec_policy(stream),
-      thrust::make_counting_iterator(0),
-      group_sizes.size(),
-      [d_result = *result_view, d_group_sizes = *group_size_view, ddof] __device__(size_type i) {
-        size_type group_size = d_group_sizes.element<size_type>(i);
-        if (group_size == 0 or group_size - ddof <= 0)
-          d_result.set_null(i);
-        else
-          d_result.set_valid(i);
-      });
+    thrust::for_each_n(rmm::exec_policy(stream),
+                       thrust::make_counting_iterator(0),
+                       group_sizes.size(),
+                       [d_result = *result_view, d_group_sizes, ddof] __device__(size_type i) {
+                         size_type group_size = d_group_sizes[i];
+                         if (group_size == 0 or group_size - ddof <= 0)
+                           d_result.set_null(i);
+                         else
+                           d_result.set_valid(i);
+                       });
 
     return result;
 #else
@@ -137,8 +162,12 @@ std::unique_ptr<column> group_var(column_view const& values,
                                   rmm::cuda_stream_view stream,
                                   rmm::mr::device_memory_resource* mr)
 {
+  auto values_type = cudf::is_dictionary(values.type())
+                       ? dictionary_column_view(values).keys().type()
+                       : values.type();
+
   return type_dispatcher(
-    values.type(), var_functor{}, values, group_means, group_sizes, group_labels, ddof, stream, mr);
+    values_type, var_functor{}, values, group_means, group_sizes, group_labels, ddof, stream, mr);
 }
 
 }  // namespace detail
diff --git a/cpp/src/groupby/sort/group_sum.cu b/cpp/src/groupby/sort/group_sum.cu
index bf3aff91c99..f0b50e910c4 100644
--- a/cpp/src/groupby/sort/group_sum.cu
+++ b/cpp/src/groupby/sort/group_sum.cu
@@ -14,6 +14,7 @@
  * limitations under the License.
  */
 
+#include <cudf/dictionary/dictionary_column_view.hpp>
 #include <groupby/sort/group_single_pass_reduction_util.cuh>
 
 #include <rmm/cuda_stream_view.hpp>
@@ -27,13 +28,11 @@ std::unique_ptr<column> group_sum(column_view const& values,
                                   rmm::cuda_stream_view stream,
                                   rmm::mr::device_memory_resource* mr)
 {
-  return type_dispatcher(values.type(),
-                         reduce_functor<aggregation::SUM>{},
-                         values,
-                         num_groups,
-                         group_labels,
-                         stream,
-                         mr);
+  auto values_type = cudf::is_dictionary(values.type())
+                       ? dictionary_column_view(values).keys().type()
+                       : values.type();
+  return type_dispatcher(
+    values_type, reduce_functor<aggregation::SUM>{}, values, num_groups, group_labels, stream, mr);
 }
 
 }  // namespace detail
diff --git a/cpp/tests/groupby/group_argmax_test.cpp b/cpp/tests/groupby/group_argmax_test.cpp
index bb4716c9c73..dafa061e7ff 100644
--- a/cpp/tests/groupby/group_argmax_test.cpp
+++ b/cpp/tests/groupby/group_argmax_test.cpp
@@ -160,8 +160,34 @@ TEST_F(groupby_argmax_string_test, zero_valid_values)
     auto agg2 = cudf::make_argmax_aggregation();
     test_single_agg(keys, vals, expect_keys, expect_vals, std::move(agg2), force_use_sort_impl::YES);
 }
+
 // clang-format on
 
+struct groupby_dictionary_argmax_test : public cudf::test::BaseFixture {
+};
+
+TEST_F(groupby_dictionary_argmax_test, basic)
+{
+  using K = int32_t;
+  using V = std::string;
+  using R = cudf::detail::target_type_t<V, aggregation::ARGMAX>;
+
+  // clang-format off
+  fixed_width_column_wrapper<K> keys{     1,     2,    3,     1,     2,     2,     1,    3,    3,    2 };
+  dictionary_column_wrapper<V>  vals{ "año", "bit", "₹1", "aaa", "zit", "bat", "aab", "$1", "€1", "wut"};
+  fixed_width_column_wrapper<K> expect_keys({ 1, 2, 3 });
+  fixed_width_column_wrapper<R> expect_vals({ 0, 4, 2 });
+  // clang-format on
+
+  test_single_agg(keys, vals, expect_keys, expect_vals, cudf::make_argmax_aggregation());
+  test_single_agg(keys,
+                  vals,
+                  expect_keys,
+                  expect_vals,
+                  cudf::make_argmax_aggregation(),
+                  force_use_sort_impl::YES);
+}
+
 }  // namespace test
 }  // namespace cudf
 
diff --git a/cpp/tests/groupby/group_argmin_test.cpp b/cpp/tests/groupby/group_argmin_test.cpp
index 439493901e1..28f13a3b813 100644
--- a/cpp/tests/groupby/group_argmin_test.cpp
+++ b/cpp/tests/groupby/group_argmin_test.cpp
@@ -163,5 +163,30 @@ TEST_F(groupby_argmin_string_test, zero_valid_values)
 }
 // clang-format on
 
+struct groupby_dictionary_argmin_test : public cudf::test::BaseFixture {
+};
+
+TEST_F(groupby_dictionary_argmin_test, basic)
+{
+  using K = int32_t;
+  using V = std::string;
+  using R = cudf::detail::target_type_t<V, aggregation::ARGMIN>;
+
+  // clang-format off
+  fixed_width_column_wrapper<K> keys{     1,     2,    3,     1,     2,     2,     1,    3,    3,    2 };
+  dictionary_column_wrapper<V>  vals{ "año", "bit", "₹1", "aaa", "zit", "bat", "aab", "$1", "€1", "wut"};
+  fixed_width_column_wrapper<K> expect_keys({ 1, 2, 3 });
+  fixed_width_column_wrapper<R> expect_vals({ 3, 5, 7 });
+  // clang-format on
+
+  test_single_agg(keys, vals, expect_keys, expect_vals, cudf::make_argmin_aggregation());
+  test_single_agg(keys,
+                  vals,
+                  expect_keys,
+                  expect_vals,
+                  cudf::make_argmin_aggregation(),
+                  force_use_sort_impl::YES);
+}
+
 }  // namespace test
 }  // namespace cudf
diff --git a/cpp/tests/groupby/group_collect_test.cpp b/cpp/tests/groupby/group_collect_test.cpp
index 5fe18584a63..1e387fa3455 100644
--- a/cpp/tests/groupby/group_collect_test.cpp
+++ b/cpp/tests/groupby/group_collect_test.cpp
@@ -87,5 +87,26 @@ TYPED_TEST(groupby_collect_test, CollectLists)
   test_single_agg(keys, values, expect_keys, expect_vals, std::move(agg));
 }
 
+TYPED_TEST(groupby_collect_test, dictionary)
+{
+  using K = int32_t;
+  using V = TypeParam;
+
+  fixed_width_column_wrapper<K, int32_t> keys{1, 1, 1, 2, 2, 2};
+  dictionary_column_wrapper<V, int32_t> vals{1, 2, 3, 4, 5, 6};
+
+  fixed_width_column_wrapper<K, int32_t> expect_keys{1, 2};
+  lists_column_wrapper<V, int32_t> expect_vals_w{{1, 2, 3}, {4, 5, 6}};
+
+  fixed_width_column_wrapper<int32_t> offsets({0, 3, 6});
+  auto expect_vals = cudf::make_lists_column(cudf::column_view(offsets).size() - 1,
+                                             std::make_unique<cudf::column>(offsets),
+                                             std::make_unique<cudf::column>(vals),
+                                             0,
+                                             rmm::device_buffer{0});
+
+  test_single_agg(keys, vals, expect_keys, expect_vals->view(), cudf::make_collect_aggregation());
+}
+
 }  // namespace test
 }  // namespace cudf
diff --git a/cpp/tests/groupby/group_count_test.cpp b/cpp/tests/groupby/group_count_test.cpp
index f4b74ded89d..77d1e275d24 100644
--- a/cpp/tests/groupby/group_count_test.cpp
+++ b/cpp/tests/groupby/group_count_test.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019, NVIDIA CORPORATION.
+ * Copyright (c) 2019-2020, NVIDIA CORPORATION.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -168,5 +168,26 @@ TEST_F(groupby_count_string_test, basic)
 }
 // clang-format on
 
+struct groupby_dictionary_count_test : public cudf::test::BaseFixture {
+};
+
+TEST_F(groupby_dictionary_count_test, basic)
+{
+  using K = int32_t;
+  using V = std::string;
+  using R = cudf::detail::target_type_t<V, aggregation::COUNT_VALID>;
+
+  // clang-format off
+  strings_column_wrapper       keys{"1", "3", "3", "5", "5", "0"};
+  dictionary_column_wrapper<K> vals{ 1,   1,   1,   1,   1,   1};
+  strings_column_wrapper             expect_keys{"0", "1", "3", "5"};
+  fixed_width_column_wrapper<R, int> expect_vals{ 1,   1,   2,   2};
+  // clang-format on
+
+  test_single_agg(keys, vals, expect_keys, expect_vals, cudf::make_count_aggregation());
+  test_single_agg(
+    keys, vals, expect_keys, expect_vals, cudf::make_count_aggregation(), force_use_sort_impl::YES);
+}
+
 }  // namespace test
 }  // namespace cudf
diff --git a/cpp/tests/groupby/group_max_test.cpp b/cpp/tests/groupby/group_max_test.cpp
index 95aa10b95ac..cc55e70910a 100644
--- a/cpp/tests/groupby/group_max_test.cpp
+++ b/cpp/tests/groupby/group_max_test.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019, NVIDIA CORPORATION.
+ * Copyright (c) 2019-2020, NVIDIA CORPORATION.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -21,6 +21,7 @@
 #include <cudf_test/type_lists.hpp>
 
 #include <cudf/detail/aggregation/aggregation.hpp>
+#include <cudf/dictionary/update_keys.hpp>
 
 namespace cudf {
 namespace test {
@@ -169,5 +170,31 @@ TEST_F(groupby_max_string_test, zero_valid_values)
 }
 // clang-format on
 
+struct groupby_dictionary_max_test : public cudf::test::BaseFixture {
+};
+
+TEST_F(groupby_dictionary_max_test, basic)
+{
+  using K = int32_t;
+  using V = std::string;
+
+  // clang-format off
+  fixed_width_column_wrapper<K> keys{     1,     2,    3,     1,     2,     2,     1,    3,    3,    2 };
+  dictionary_column_wrapper<V>  vals{ "año", "bit", "₹1", "aaa", "zit", "bat", "aaa", "$1", "₹1", "wut"};
+  fixed_width_column_wrapper<K> expect_keys   {     1,     2,    3 };
+  dictionary_column_wrapper<V>  expect_vals_w({ "año", "zit", "₹1" });
+  // clang-format on
+
+  auto expect_vals = cudf::dictionary::set_keys(expect_vals_w, vals.keys());
+
+  test_single_agg(keys, vals, expect_keys, expect_vals->view(), cudf::make_max_aggregation());
+  test_single_agg(keys,
+                  vals,
+                  expect_keys,
+                  expect_vals->view(),
+                  cudf::make_max_aggregation(),
+                  force_use_sort_impl::YES);
+}
+
 }  // namespace test
 }  // namespace cudf
diff --git a/cpp/tests/groupby/group_mean_test.cpp b/cpp/tests/groupby/group_mean_test.cpp
index fe1fe06e849..6d12516a8cf 100644
--- a/cpp/tests/groupby/group_mean_test.cpp
+++ b/cpp/tests/groupby/group_mean_test.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019, NVIDIA CORPORATION.
+ * Copyright (c) 2019-2020, NVIDIA CORPORATION.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -14,19 +14,20 @@
  * limitations under the License.
  */
 
-#include <initializer_list>
-#include <iterator>
-#include <tests/groupby/groupby_test_util.hpp>
+#include <cudf/detail/aggregation/aggregation.hpp>
+#include <cudf/utilities/traits.hpp>
 
 #include <cudf_test/base_fixture.hpp>
 #include <cudf_test/column_wrapper.hpp>
 #include <cudf_test/type_list_utilities.hpp>
 #include <cudf_test/type_lists.hpp>
 
-#include <cudf/detail/aggregation/aggregation.hpp>
+#include <tests/groupby/groupby_test_util.hpp>
+
+#include <initializer_list>
+#include <iterator>
 #include <type_traits>
 #include <vector>
-#include "cudf/utilities/traits.hpp"
 
 namespace cudf {
 namespace test {
@@ -137,5 +138,27 @@ TYPED_TEST(groupby_mean_test, null_keys_and_values)
 }
 // clang-format on
 
+struct groupby_dictionary_mean_test : public cudf::test::BaseFixture {
+};
+
+// This tests will not work until the following ptxas bug is fixed in 10.2
+// https://nvbugswb.nvidia.com/NvBugs5/SWBug.aspx?bugid=3186317&cp=
+TEST_F(groupby_dictionary_mean_test, DISABLED_basic)
+{
+  using K = int32_t;
+  using V = int16_t;
+  using R = cudf::detail::target_type_t<V, aggregation::MEAN>;
+
+  // clang-format off
+  fixed_width_column_wrapper<K>     keys{ 1, 2, 3, 1, 2, 2, 1, 3, 3, 2};
+  dictionary_column_wrapper<V, int> vals{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
+
+  fixed_width_column_wrapper<K>         expect_keys({ 1,    2,     3    });
+  fixed_width_column_wrapper<R, double> expect_vals({ 9./3, 19./4, 17./3});
+  // clang-format on
+
+  test_single_agg(keys, vals, expect_keys, expect_vals, cudf::make_mean_aggregation());
+}
+
 }  // namespace test
 }  // namespace cudf
diff --git a/cpp/tests/groupby/group_median_test.cpp b/cpp/tests/groupby/group_median_test.cpp
index d5400f16b94..28acadf9ecc 100644
--- a/cpp/tests/groupby/group_median_test.cpp
+++ b/cpp/tests/groupby/group_median_test.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019, NVIDIA CORPORATION.
+ * Copyright (c) 2019-2020, NVIDIA CORPORATION.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -121,5 +121,24 @@ TYPED_TEST(groupby_median_test, null_keys_and_values)
 }
 // clang-format on
 
+TYPED_TEST(groupby_median_test, dictionary)
+{
+  using K = int32_t;
+  using V = TypeParam;
+  using R = cudf::detail::target_type_t<V, aggregation::MEDIAN>;
+
+  // clang-format off
+  fixed_width_column_wrapper<K> keys{ 1, 2, 3, 1, 2, 2, 1, 3, 3, 2};
+  dictionary_column_wrapper<V>  vals{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
+
+                                        //  { 1, 1, 1, 2, 2, 2, 2, 3, 3, 3}
+  fixed_width_column_wrapper<K> expect_keys({ 1,       2,          3      });
+                                        //  { 0, 3, 6, 1, 4, 5, 9, 2, 7, 8}
+  fixed_width_column_wrapper<R> expect_vals({ 3.,       4.5,       7.   }, all_valid());
+  // clang-format on
+
+  test_single_agg(keys, vals, expect_keys, expect_vals, cudf::make_median_aggregation());
+}
+
 }  // namespace test
 }  // namespace cudf
diff --git a/cpp/tests/groupby/group_min_test.cpp b/cpp/tests/groupby/group_min_test.cpp
index 14b4af346a3..ca095f08547 100644
--- a/cpp/tests/groupby/group_min_test.cpp
+++ b/cpp/tests/groupby/group_min_test.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019, NVIDIA CORPORATION.
+ * Copyright (c) 2019-2020, NVIDIA CORPORATION.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -21,6 +21,7 @@
 #include <cudf_test/type_lists.hpp>
 
 #include <cudf/detail/aggregation/aggregation.hpp>
+#include <cudf/dictionary/update_keys.hpp>
 
 namespace cudf {
 namespace test {
@@ -169,5 +170,31 @@ TEST_F(groupby_min_string_test, zero_valid_values)
 }
 // clang-format on
 
+struct groupby_dictionary_min_test : public cudf::test::BaseFixture {
+};
+
+TEST_F(groupby_dictionary_min_test, basic)
+{
+  using K = int32_t;
+  using V = std::string;
+
+  // clang-format off
+  fixed_width_column_wrapper<K> keys{     1,     2,    3,     1,     2,     2,     1,    3,    3,    2 };
+  dictionary_column_wrapper<V>  vals{ "año", "bit", "₹1", "aaa", "zit", "bat", "aaa", "$1", "₹1", "wut"};
+  fixed_width_column_wrapper<K> expect_keys   {     1,     2,    3 };
+  dictionary_column_wrapper<V>  expect_vals_w({ "aaa", "bat", "$1" });
+  // clang-format on
+
+  auto expect_vals = cudf::dictionary::set_keys(expect_vals_w, vals.keys());
+
+  test_single_agg(keys, vals, expect_keys, expect_vals->view(), cudf::make_min_aggregation());
+  test_single_agg(keys,
+                  vals,
+                  expect_keys,
+                  expect_vals->view(),
+                  cudf::make_min_aggregation(),
+                  force_use_sort_impl::YES);
+}
+
 }  // namespace test
 }  // namespace cudf
diff --git a/cpp/tests/groupby/group_nth_element_test.cpp b/cpp/tests/groupby/group_nth_element_test.cpp
index 632b43442a7..3be88e8b117 100644
--- a/cpp/tests/groupby/group_nth_element_test.cpp
+++ b/cpp/tests/groupby/group_nth_element_test.cpp
@@ -21,6 +21,7 @@
 #include <cudf_test/type_lists.hpp>
 
 #include <cudf/detail/aggregation/aggregation.hpp>
+#include <cudf/dictionary/update_keys.hpp>
 
 namespace cudf {
 namespace test {
@@ -293,7 +294,10 @@ TYPED_TEST(groupby_nth_element_test, exclude_nulls_negative_index)
   test_single_agg(keys, vals, expect_keys, expect_vals2, std::move(agg));
 }
 
-TYPED_TEST(groupby_nth_element_test, basic_string)
+struct groupby_nth_element_string_test : public cudf::test::BaseFixture {
+};
+
+TEST_F(groupby_nth_element_string_test, basic_string)
 {
   using K = int32_t;
   using V = std::string;
@@ -344,5 +348,21 @@ TYPED_TEST(groupby_nth_element_test, basic_string)
 }
 // clang-format on
 
+TEST_F(groupby_nth_element_string_test, dictionary)
+{
+  using K = int32_t;
+  using V = std::string;
+
+  fixed_width_column_wrapper<K> keys{1, 2, 3, 1, 2, 2, 1, 3, 3, 2};
+  dictionary_column_wrapper<V> vals{"AB", "1", "2", "3", "4", "5", "6", "7", "8", "9"};
+  fixed_width_column_wrapper<K> expect_keys{1, 2, 3};
+  dictionary_column_wrapper<V> expect_vals_w{"6", "5", "8"};
+
+  auto expect_vals = cudf::dictionary::set_keys(expect_vals_w, vals.keys());
+
+  test_single_agg(
+    keys, vals, expect_keys, expect_vals->view(), cudf::make_nth_element_aggregation(2));
+}
+
 }  // namespace test
 }  // namespace cudf
diff --git a/cpp/tests/groupby/group_nunique_test.cpp b/cpp/tests/groupby/group_nunique_test.cpp
index ec043c467cc..192e2971fe2 100644
--- a/cpp/tests/groupby/group_nunique_test.cpp
+++ b/cpp/tests/groupby/group_nunique_test.cpp
@@ -199,5 +199,32 @@ TYPED_TEST(groupby_nunique_test, include_nulls)
 }
 // clang-format on
 
+TYPED_TEST(groupby_nunique_test, dictionary)
+{
+  using K = int32_t;
+  using V = TypeParam;
+  using R = cudf::detail::target_type_t<V, aggregation::NUNIQUE>;
+
+  // clang-format off
+  fixed_width_column_wrapper<K>         keys({1, 2, 3, 3, 1, 2, 2, 1, 0, 3, 2, 4, 4, 2},
+                                             {1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1});
+  dictionary_column_wrapper<V, int32_t> vals({0, 1, 2, 2, 3, 4, 0, 6, 7, 8, 9, 0, 0, 0},
+                                             {0, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 0, 0, 0});
+
+                                        // { 1, 1,   2, 2, 2,   3, 3,   4}
+  fixed_width_column_wrapper<K> expect_keys({1,      2,         3,      4}, all_valid());
+                                        // { 3, 6,-  1, 4, 9,-  2*, 8,  -*}
+                                        //  unique,  with null, dup,    dup null
+  fixed_width_column_wrapper<R> expect_fixed_vals({3,  4,         2,      1});
+  fixed_width_column_wrapper<R> expect_bool_vals { 2,  2,         1,      1};
+  // clang-format on
+
+  cudf::column_view expect_vals = (std::is_same<V, bool>()) ? cudf::column_view{expect_bool_vals}
+                                                            : cudf::column_view{expect_fixed_vals};
+
+  test_single_agg(
+    keys, vals, expect_keys, expect_vals, cudf::make_nunique_aggregation(null_policy::INCLUDE));
+}
+
 }  // namespace test
 }  // namespace cudf
diff --git a/cpp/tests/groupby/group_quantile_test.cpp b/cpp/tests/groupby/group_quantile_test.cpp
index fb71ef3376b..9908423d088 100644
--- a/cpp/tests/groupby/group_quantile_test.cpp
+++ b/cpp/tests/groupby/group_quantile_test.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019, NVIDIA CORPORATION.
+ * Copyright (c) 2019-2020, NVIDIA CORPORATION.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -191,5 +191,28 @@ TYPED_TEST(groupby_quantile_test, interpolation_types)
 }
 // clang-format on
 
+TYPED_TEST(groupby_quantile_test, dictionary)
+{
+  using K = int32_t;
+  using V = TypeParam;
+  using R = cudf::detail::target_type_t<V, aggregation::QUANTILE>;
+
+  // clang-format off
+  fixed_width_column_wrapper<K> keys{ 1, 2, 3, 1, 2, 2, 1, 3, 3, 2};
+  dictionary_column_wrapper<V>  vals{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
+
+                                        //  { 1, 1, 1, 2, 2, 2, 2, 3, 3, 3}
+  fixed_width_column_wrapper<K> expect_keys({ 1,       2,          3      });
+                                        //  { 0, 3, 6, 1, 4, 5, 9, 2, 7, 8}
+  fixed_width_column_wrapper<R> expect_vals({ 3.,      4.5,        7.   }, all_valid());
+  // clang-format on
+
+  test_single_agg(keys,
+                  vals,
+                  expect_keys,
+                  expect_vals,
+                  cudf::make_quantile_aggregation({0.5}, interpolation::LINEAR));
+}
+
 }  // namespace test
 }  // namespace cudf
diff --git a/cpp/tests/groupby/group_std_test.cpp b/cpp/tests/groupby/group_std_test.cpp
index e60aba08385..776e24f1fc1 100644
--- a/cpp/tests/groupby/group_std_test.cpp
+++ b/cpp/tests/groupby/group_std_test.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019, NVIDIA CORPORATION.
+ * Copyright (c) 2019-2020, NVIDIA CORPORATION.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -144,6 +144,27 @@ TYPED_TEST(groupby_std_test, ddof_non_default)
 }
 // clang-format on
 
+// This test will not work until the following ptxas bug is fixed in 10.2
+// https://nvbugswb.nvidia.com/NvBugs5/SWBug.aspx?bugid=3186317&cp=
+TYPED_TEST(groupby_std_test, DISABLED_dictionary)
+{
+  using K = int32_t;
+  using V = TypeParam;
+  using R = cudf::detail::target_type_t<V, aggregation::STD>;
+
+  // clang-format off
+  fixed_width_column_wrapper<K> keys{ 1, 2, 3, 1, 2, 2, 1, 3, 3, 2};
+  dictionary_column_wrapper<V>  vals{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
+
+                                        //   { 1, 1, 1,  2, 2, 2, 2,  3, 3, 3}
+  fixed_width_column_wrapper<K> expect_keys({ 1,        2,           3      });
+                                        //   { 0, 3, 6,  1, 4, 5, 9,  2, 7, 8}
+  fixed_width_column_wrapper<R> expect_vals({  3.,   sqrt(131./12),sqrt(31./3)}, all_valid());
+  // clang-format on
+
+  test_single_agg(keys, vals, expect_keys, expect_vals, cudf::make_std_aggregation());
+}
+
 }  // namespace test
 }  // namespace cudf
 
diff --git a/cpp/tests/groupby/group_sum_test.cpp b/cpp/tests/groupby/group_sum_test.cpp
index 371bdb634db..fb1d4184f62 100644
--- a/cpp/tests/groupby/group_sum_test.cpp
+++ b/cpp/tests/groupby/group_sum_test.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019, NVIDIA CORPORATION.
+ * Copyright (c) 2019-2020, NVIDIA CORPORATION.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -136,5 +136,26 @@ TYPED_TEST(groupby_sum_test, null_keys_and_values)
 }
 // clang-format on
 
+// These tests will not work until the following ptxas bug is fixed in 10.2
+// https://nvbugswb.nvidia.com/NvBugs5/SWBug.aspx?bugid=3186317&cp=
+TYPED_TEST(groupby_sum_test, DISABLED_dictionary)
+{
+  using K = int32_t;
+  using V = TypeParam;
+  using R = cudf::detail::target_type_t<V, aggregation::SUM>;
+
+  // clang-format off
+  fixed_width_column_wrapper<K>     keys{ 1, 2, 3, 1, 2, 2, 1, 3, 3, 2};
+  dictionary_column_wrapper<V, int> vals{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
+
+  fixed_width_column_wrapper<K>      expect_keys{ 1, 2,  3 };
+  fixed_width_column_wrapper<R, int> expect_vals{ 9, 19, 17};
+  // clang-format on
+
+  test_single_agg(keys, vals, expect_keys, expect_vals, cudf::make_sum_aggregation());
+  test_single_agg(
+    keys, vals, expect_keys, expect_vals, cudf::make_sum_aggregation(), force_use_sort_impl::YES);
+}
+
 }  // namespace test
 }  // namespace cudf
diff --git a/cpp/tests/groupby/group_var_test.cpp b/cpp/tests/groupby/group_var_test.cpp
index 6c1ea616212..5fa52bad22a 100644
--- a/cpp/tests/groupby/group_var_test.cpp
+++ b/cpp/tests/groupby/group_var_test.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019, NVIDIA CORPORATION.
+ * Copyright (c) 2019-2020, NVIDIA CORPORATION.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -144,6 +144,27 @@ TYPED_TEST(groupby_var_test, ddof_non_default)
 }
 // clang-format on
 
+// This test will not work until the following ptxas bug is fixed in 10.2
+// https://nvbugswb.nvidia.com/NvBugs5/SWBug.aspx?bugid=3186317&cp=
+TYPED_TEST(groupby_var_test, DISABLED_dictionary)
+{
+  using K = int32_t;
+  using V = TypeParam;
+  using R = cudf::detail::target_type_t<V, aggregation::VARIANCE>;
+
+  // clang-format off
+  fixed_width_column_wrapper<K> keys{ 1, 2, 3, 1, 2, 2, 1, 3, 3, 2};
+  dictionary_column_wrapper<V>  vals{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
+
+                                        //   { 1, 1, 1,  2, 2, 2, 2,  3, 3, 3}
+  fixed_width_column_wrapper<K> expect_keys({ 1,        2,           3      });
+                                        //   { 0, 3, 6,  1, 4, 5, 9,  2, 7, 8}
+  fixed_width_column_wrapper<R> expect_vals( { 9.,      131./12,     31./3 }, all_valid());
+  // clang-format on
+
+  test_single_agg(keys, vals, expect_keys, expect_vals, cudf::make_variance_aggregation());
+}
+
 }  // namespace test
 }  // namespace cudf
 
diff --git a/cpp/tests/groupby/groupby_keys_test.cpp b/cpp/tests/groupby/groupby_keys_test.cpp
index f9fd77b0de7..553c16e8349 100644
--- a/cpp/tests/groupby/groupby_keys_test.cpp
+++ b/cpp/tests/groupby/groupby_keys_test.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019, NVIDIA CORPORATION.
+ * Copyright (c) 2019-2020, NVIDIA CORPORATION.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -195,5 +195,26 @@ TEST_F(groupby_string_keys_test, basic)
 }
 // clang-format on
 
+struct groupby_dictionary_keys_test : public cudf::test::BaseFixture {
+};
+
+TEST_F(groupby_dictionary_keys_test, basic)
+{
+  using K = std::string;
+  using V = int32_t;
+  using R = cudf::detail::target_type_t<V, aggregation::SUM>;
+
+  // clang-format off
+  dictionary_column_wrapper<K> keys { "aaa", "año", "₹1", "aaa", "año", "año", "aaa", "₹1", "₹1", "año"};
+  fixed_width_column_wrapper<V> vals{     0,     1,    2,     3,     4,     5,     6,    7,    8,     9};
+  dictionary_column_wrapper<K>expect_keys  ({ "aaa", "año", "₹1" });
+  fixed_width_column_wrapper<R> expect_vals({     9,    19,   17 });
+  // clang-format on
+
+  test_single_agg(keys, vals, expect_keys, expect_vals, cudf::make_sum_aggregation());
+  test_single_agg(
+    keys, vals, expect_keys, expect_vals, cudf::make_sum_aggregation(), force_use_sort_impl::YES);
+}
+
 }  // namespace test
 }  // namespace cudf

From c0920e639f64b8d737a2b87b4ee848dd0c70d341 Mon Sep 17 00:00:00 2001
From: Michael Wang <isVoid@users.noreply.github.com>
Date: Tue, 5 Jan 2021 18:22:25 -0800
Subject: [PATCH 09/20] Improve `digitize` API(#7071)

Closes #6360

Updates `digitize()` API to accept any non-nullable column-like objects, as opposed to previously only `np.array`. Adds better error handling and further simplifies the implementation.

Authors:
  - Michael Wang <isVoid@users.noreply.github.com>

Approvers:
  - Ashwin Srinath

URL: https://github.com/rapidsai/cudf/pull/7071
---
 python/cudf/cudf/core/column/numerical.py | 17 ++++++++++++-----
 python/cudf/cudf/tests/test_dataframe.py  | 19 +++++++++++++++++--
 2 files changed, 29 insertions(+), 7 deletions(-)

diff --git a/python/cudf/cudf/core/column/numerical.py b/python/cudf/cudf/core/column/numerical.py
index ddac48b60ab..b8a25d83ea8 100644
--- a/python/cudf/cudf/core/column/numerical.py
+++ b/python/cudf/cudf/core/column/numerical.py
@@ -699,8 +699,9 @@ def digitize(column, bins, right=False):
     ----------
     column : Column
         Input column.
-    bins : np.array
-        1-D monotonically increasing array of bins with same type as `column`.
+    bins : Column-like
+        1-D column-like object of bins with same type as `column`, should be
+        monotonically increasing.
     right : bool
         Indicates whether interval contains the right or left bin edge.
 
@@ -708,9 +709,15 @@ def digitize(column, bins, right=False):
     -------
     A device array containing the indices
     """
-    assert column.dtype == bins.dtype
-    bins_buf = Buffer(bins.view("|u1"))
-    bin_col = NumericalColumn(data=bins_buf, dtype=bins.dtype)
+    if not column.dtype == bins.dtype:
+        raise ValueError(
+            "Digitize() expects bins and input column have the same dtype."
+        )
+
+    bin_col = as_column(bins, dtype=bins.dtype)
+    if bin_col.nullable:
+        raise ValueError("`bins` cannot contain null entries.")
+
     return as_column(
         libcudf.sort.digitize(column.as_frame(), bin_col.as_frame(), right)
     )
diff --git a/python/cudf/cudf/tests/test_dataframe.py b/python/cudf/cudf/tests/test_dataframe.py
index 88254f4fb48..32e76a1a79c 100644
--- a/python/cudf/cudf/tests/test_dataframe.py
+++ b/python/cudf/cudf/tests/test_dataframe.py
@@ -2226,16 +2226,31 @@ def test_arrow_handle_no_index_name(pdf, gdf):
 @pytest.mark.parametrize("num_bins", [1, 2, 4, 20])
 @pytest.mark.parametrize("right", [True, False])
 @pytest.mark.parametrize("dtype", NUMERIC_TYPES + ["bool"])
-def test_series_digitize(num_rows, num_bins, right, dtype):
+@pytest.mark.parametrize("series_bins", [True, False])
+def test_series_digitize(num_rows, num_bins, right, dtype, series_bins):
     data = np.random.randint(0, 100, num_rows).astype(dtype)
     bins = np.unique(np.sort(np.random.randint(2, 95, num_bins).astype(dtype)))
     s = gd.Series(data)
-    indices = s.digitize(bins, right)
+    if series_bins:
+        s_bins = gd.Series(bins)
+        indices = s.digitize(s_bins, right)
+    else:
+        indices = s.digitize(bins, right)
     np.testing.assert_array_equal(
         np.digitize(data, bins, right), indices.to_array()
     )
 
 
+def test_series_digitize_invalid_bins():
+    s = gd.Series(np.random.randint(0, 30, 80), dtype="int32")
+    bins = gd.Series([2, None, None, 50, 90], dtype="int32")
+
+    with pytest.raises(
+        ValueError, match="`bins` cannot contain null entries."
+    ):
+        _ = s.digitize(bins)
+
+
 def test_pandas_non_contiguious():
     arr1 = np.random.sample([5000, 10])
     assert arr1.flags["C_CONTIGUOUS"] is True

From 193043259d8d0cb28f7eb6640d05b11e3738f826 Mon Sep 17 00:00:00 2001
From: Michael Wang <isVoid@users.noreply.github.com>
Date: Tue, 5 Jan 2021 18:37:26 -0800
Subject: [PATCH 10/20] Add `unstack()` support for non-multiindexed
 dataframes(#7054)

Closes #6694

When `unstack()` receives a dataframe with "single" index, returns a series to match pandas behavior.

Authors:
  - Michael Wang <isVoid@users.noreply.github.com>

Approvers:
  - null

URL: https://github.com/rapidsai/cudf/pull/7054
---
 python/cudf/cudf/core/reshape.py       | 40 +++++++++++++++++---
 python/cudf/cudf/tests/test_reshape.py | 51 +++++++++++++++++++++++++-
 2 files changed, 85 insertions(+), 6 deletions(-)

diff --git a/python/cudf/cudf/core/reshape.py b/python/cudf/cudf/core/reshape.py
index b5707a3a07c..cd1ce36c6c5 100644
--- a/python/cudf/cudf/core/reshape.py
+++ b/python/cudf/cudf/core/reshape.py
@@ -902,6 +902,11 @@ def unstack(df, level, fill_value=None):
     Pivots the specified levels of the index labels of df to the innermost
     levels of the columns labels of the result.
 
+    * If the index of ``df`` has multiple levels, returns a ``Dataframe`` with
+      specified level of the index pivoted to the column levels.
+    * If the index of ``df`` has single level, returns a ``Series`` with all
+      column levels pivoted to the index levels.
+
     Parameters
     ----------
     df : DataFrame
@@ -913,7 +918,7 @@ def unstack(df, level, fill_value=None):
 
     Returns
     -------
-    DataFrame with specified index levels pivoted to column levels
+    Series or DataFrame
 
     Examples
     --------
@@ -964,7 +969,25 @@ def unstack(df, level, fill_value=None):
     a
     1     5  <NA>     6  <NA>     7
     2  <NA>     8  <NA>     9  <NA>
+
+    Unstacking single level index dataframe:
+
+    >>> df = cudf.DataFrame({('c', 1): [1, 2, 3], ('c', 2):[9, 8, 7]})
+    >>> df.unstack()
+    c  1  0    1
+          1    2
+          2    3
+       2  0    9
+          1    8
+          2    7
+    dtype: int64
     """
+    if not isinstance(df, cudf.DataFrame):
+        raise ValueError("`df` should be a cudf Dataframe object.")
+
+    if df.empty:
+        raise ValueError("Cannot unstack an empty dataframe.")
+
     if fill_value is not None:
         raise NotImplementedError("fill_value is not supported.")
     if pd.api.types.is_list_like(level):
@@ -972,10 +995,17 @@ def unstack(df, level, fill_value=None):
             return df
     df = df.copy(deep=False)
     if not isinstance(df.index, cudf.MultiIndex):
-        raise NotImplementedError(
-            "Calling unstack() on a DataFrame without a MultiIndex "
-            "is not supported"
-        )
+        dtype = df._columns[0].dtype
+        for col in df._columns:
+            if not col.dtype == dtype:
+                raise ValueError(
+                    "Calling unstack() on single index dataframe"
+                    " with different column datatype is not supported."
+                )
+        res = df.T.stack(dropna=False)
+        # Result's index is a multiindex
+        res.index.names = tuple(df.columns.names) + df.index.names
+        return res
     else:
         columns = df.index._poplevels(level)
         index = df.index
diff --git a/python/cudf/cudf/tests/test_reshape.py b/python/cudf/cudf/tests/test_reshape.py
index 890c69b660a..315762c931f 100644
--- a/python/cudf/cudf/tests/test_reshape.py
+++ b/python/cudf/cudf/tests/test_reshape.py
@@ -1,3 +1,5 @@
+import re
+
 import numpy as np
 import pandas as pd
 import pytest
@@ -402,7 +404,7 @@ def test_pivot_multi_values():
         ),
     ],
 )
-def test_unstack(level):
+def test_unstack_multiindex(level):
     pdf = pd.DataFrame(
         {
             "foo": ["one", "one", "one", "two", "two", "two"],
@@ -417,6 +419,53 @@ def test_unstack(level):
     )
 
 
+@pytest.mark.parametrize(
+    "data",
+    [{"A": [1.0, 2.0, 3.0, 4.0, 5.0], "B": [11.0, 12.0, 13.0, 14.0, 15.0]}],
+)
+@pytest.mark.parametrize(
+    "index",
+    [
+        pd.Index(range(0, 5), name=None),
+        pd.Index(range(0, 5), name="row_index"),
+    ],
+)
+@pytest.mark.parametrize(
+    "col_idx",
+    [
+        pd.Index(["a", "b"], name=None),
+        pd.Index(["a", "b"], name="col_index"),
+        pd.MultiIndex.from_tuples([("c", 1), ("c", 2)], names=[None, None]),
+        pd.MultiIndex.from_tuples(
+            [("c", 1), ("c", 2)], names=["col_index1", "col_index2"]
+        ),
+    ],
+)
+def test_unstack_index(data, index, col_idx):
+    pdf = pd.DataFrame(data)
+    gdf = cudf.from_pandas(pdf)
+
+    pdf.index = index
+    pdf.columns = col_idx
+
+    gdf.index = cudf.from_pandas(index)
+    gdf.columns = cudf.from_pandas(col_idx)
+
+    assert_eq(pdf.unstack(), gdf.unstack())
+
+
+def test_unstack_index_invalid():
+    gdf = cudf.DataFrame({"a": [1, 2, 3], "b": ["a", "b", "c"]})
+    with pytest.raises(
+        ValueError,
+        match=re.escape(
+            "Calling unstack() on single index dataframe with "
+            "different column datatype is not supported."
+        ),
+    ):
+        gdf.unstack()
+
+
 def test_pivot_duplicate_error():
     gdf = cudf.DataFrame(
         {"a": [0, 1, 2, 2], "b": [1, 2, 3, 3], "d": [1, 2, 3, 4]}

From 8787a647a4557167170a0761d63f945bfe909dec Mon Sep 17 00:00:00 2001
From: GALI PREM SAGAR <sagarprem75@gmail.com>
Date: Wed, 6 Jan 2021 09:07:25 -0600
Subject: [PATCH 11/20] Implement `__hash__` method for ListDtype(#7081)

Fixes: #7070

This PR fixes a failure in `to_pandas` when `nullable` is set to `True`. The changes in this PR implement `__hash__` in `listDtype`.

Authors:
  - galipremsagar <sagarprem75@gmail.com>

Approvers:
  - Ashwin Srinath

URL: https://github.com/rapidsai/cudf/pull/7081
---
 python/cudf/cudf/core/column/lists.py |  3 +--
 python/cudf/cudf/core/dtypes.py       |  5 ++++-
 python/cudf/cudf/tests/test_list.py   | 21 ++++++++++++++++++++-
 3 files changed, 25 insertions(+), 4 deletions(-)

diff --git a/python/cudf/cudf/core/column/lists.py b/python/cudf/cudf/core/column/lists.py
index 665c40b3faf..c2aa41a5de1 100644
--- a/python/cudf/cudf/core/column/lists.py
+++ b/python/cudf/cudf/core/column/lists.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2020, NVIDIA CORPORATION.
+# Copyright (c) 2020-2021, NVIDIA CORPORATION.
 
 import pickle
 
@@ -113,7 +113,6 @@ def set_base_data(self, value):
 
     def serialize(self):
         header = {}
-        frames = []
         header["type-serialized"] = pickle.dumps(type(self))
         header["dtype"] = pickle.dumps(self.dtype)
         header["null_count"] = self.null_count
diff --git a/python/cudf/cudf/core/dtypes.py b/python/cudf/cudf/core/dtypes.py
index 044ff16c405..288766913a4 100644
--- a/python/cudf/cudf/core/dtypes.py
+++ b/python/cudf/cudf/core/dtypes.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2020, NVIDIA CORPORATION.
+# Copyright (c) 2020-2021, NVIDIA CORPORATION.
 
 import decimal
 import pickle
@@ -165,6 +165,9 @@ def __repr__(self):
         else:
             return f"ListDtype({self.element_type})"
 
+    def __hash__(self):
+        return hash(self._typ)
+
 
 class StructDtype(ExtensionDtype):
 
diff --git a/python/cudf/cudf/tests/test_list.py b/python/cudf/cudf/tests/test_list.py
index 7a256ade844..b5169a411de 100644
--- a/python/cudf/cudf/tests/test_list.py
+++ b/python/cudf/cudf/tests/test_list.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2020, NVIDIA CORPORATION.
+# Copyright (c) 2020-2021, NVIDIA CORPORATION.
 
 import pandas as pd
 import pyarrow as pa
@@ -70,3 +70,22 @@ def test_leaves(data):
     expect = cudf.Series(pa_array, dtype=dtype)
     got = cudf.Series(data).list.leaves
     assert_eq(expect, got)
+
+
+def test_list_to_pandas_nullable_true():
+    df = cudf.DataFrame({"a": cudf.Series([[1, 2, 3]])})
+    actual = df.to_pandas(nullable=True)
+    expected = pd.DataFrame({"a": pd.Series([[1, 2, 3]])})
+
+    assert_eq(actual, expected)
+
+
+def test_listdtype_hash():
+    a = cudf.core.dtypes.ListDtype("int64")
+    b = cudf.core.dtypes.ListDtype("int64")
+
+    assert hash(a) == hash(b)
+
+    c = cudf.core.dtypes.ListDtype("int32")
+
+    assert hash(a) != hash(c)

From f768da7848c7bd3d501bc61c92254b06b7817a9a Mon Sep 17 00:00:00 2001
From: "Robert (Bobby) Evans" <bobby@apache.org>
Date: Wed, 6 Jan 2021 18:22:26 -0600
Subject: [PATCH 12/20] JNI support for creating struct column from existing
 columns and fixed bug in struct with no children(#7084)

The primary goal of this is to add in java APIs to create a struct column from other existing columns.  As a part of this work I found a very small bug in the column_vector constructor that copies data from  a column view for a struct column with no children in it.  Spark supports this use case so I thought it would be good to test/fix the issue.

Authors:
  - Robert (Bobby) Evans <bobby@apache.org>

Approvers:
  - Kuhu Shukla (@kuhushukla)
  - Vukasin Milovanovic (@vuule)
  - Jason Lowe (@jlowe)

URL: https://github.com/rapidsai/cudf/pull/7084
---
 cpp/src/column/column.cu                      |  4 +-
 .../java/ai/rapids/cudf/ColumnVector.java     | 38 +++++++++++++++-
 .../main/java/ai/rapids/cudf/ColumnView.java  | 44 +++++++++++++++++--
 .../java/ai/rapids/cudf/HostColumnVector.java | 12 ++++-
 java/src/main/native/src/ColumnViewJni.cpp    | 26 ++++++++++-
 .../java/ai/rapids/cudf/ColumnVectorTest.java | 28 +++++++++++-
 6 files changed, 142 insertions(+), 10 deletions(-)

diff --git a/cpp/src/column/column.cu b/cpp/src/column/column.cu
index f72a65f2348..796f522d949 100644
--- a/cpp/src/column/column.cu
+++ b/cpp/src/column/column.cu
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019-2020, NVIDIA CORPORATION.
+ * Copyright (c) 2019-2021, NVIDIA CORPORATION.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -266,7 +266,7 @@ struct create_column_from_view {
                        cudf::detail::slice(child, begin, end), stream, mr);
                    });
 
-    auto num_rows = children.empty() ? 0 : children.front()->size();
+    auto num_rows = view.size();
 
     return make_structs_column(num_rows,
                                std::move(children),
diff --git a/java/src/main/java/ai/rapids/cudf/ColumnVector.java b/java/src/main/java/ai/rapids/cudf/ColumnVector.java
index 34a2e2fe374..e698d497b2b 100644
--- a/java/src/main/java/ai/rapids/cudf/ColumnVector.java
+++ b/java/src/main/java/ai/rapids/cudf/ColumnVector.java
@@ -1,6 +1,6 @@
 /*
  *
- *  Copyright (c) 2019-2020, NVIDIA CORPORATION.
+ *  Copyright (c) 2019-2021, NVIDIA CORPORATION.
  *
  *  Licensed under the Apache License, Version 2.0 (the "License");
  *  you may not use this file except in compliance with the License.
@@ -323,6 +323,33 @@ public static ColumnVector fromScalar(Scalar scalar, int rows) {
     return new ColumnVector(columnHandle);
   }
 
+  /**
+   * Create a new struct vector made up of existing columns. Note that this will copy
+   * the contents of the input columns to make a new vector. If you only want to
+   * do a quick temporary computation you can use ColumnView.makeStructView.
+   * @param columns the columns to make the struct from.
+   * @return the new ColumnVector
+   */
+  public static ColumnVector makeStruct(ColumnView... columns) {
+    try (ColumnView cv = ColumnView.makeStructView(columns)) {
+      return cv.copyToColumnVector();
+    }
+  }
+
+  /**
+   * Create a new struct vector made up of existing columns. Note that this will copy
+   * the contents of the input columns to make a new vector. If you only want to
+   * do a quick temporary computation you can use ColumnView.makeStructView.
+   * @param rows the number of rows in the struct. Used for structs with no children.
+   * @param columns the columns to make the struct from.
+   * @return the new ColumnVector
+   */
+  public static ColumnVector makeStruct(long rows, ColumnView... columns) {
+    try (ColumnView cv = ColumnView.makeStructView(rows, columns)) {
+      return cv.copyToColumnVector();
+    }
+  }
+
   /**
    * Create a new vector of length rows, starting at the initialValue and going by step each time.
    * Only numeric types are supported.
@@ -866,6 +893,15 @@ public static ColumnVector fromStructs(HostColumnVector.DataType dataType,
       return host.copyToDevice();
     }
   }
+  /**
+   * This method is evolving, unstable and currently test only.
+   * Please use with caution and expect it to change in the future.
+   */
+  public static ColumnVector emptyStructs(HostColumnVector.DataType dataType, long numRows) {
+    try (HostColumnVector host = HostColumnVector.emptyStructs(dataType, numRows)) {
+      return host.copyToDevice();
+    }
+  }
 
   /**
    * Create a new vector from the given values.
diff --git a/java/src/main/java/ai/rapids/cudf/ColumnView.java b/java/src/main/java/ai/rapids/cudf/ColumnView.java
index e811147f45e..c2110a5f8ff 100644
--- a/java/src/main/java/ai/rapids/cudf/ColumnView.java
+++ b/java/src/main/java/ai/rapids/cudf/ColumnView.java
@@ -1,6 +1,6 @@
 /*
  *
- *  Copyright (c) 2020, NVIDIA CORPORATION.
+ *  Copyright (c) 2020-2021, NVIDIA CORPORATION.
  *
  *  Licensed under the Apache License, Version 2.0 (the "License");
  *  you may not use this file except in compliance with the License.
@@ -2287,6 +2287,42 @@ public final ColumnVector getMapValue(Scalar key) {
     return new ColumnVector(mapLookup(getNativeView(), key.getScalarHandle()));
   }
 
+
+  /**
+   * Create a new struct column view of existing column views. Note that this will NOT copy
+   * the contents of the input columns to make a new vector, but makes a view that must not
+   * outlive the child views that it references. The resulting column cannot be null.
+   * @param rows the number of rows in the struct column. This is needed if no columns
+   *             are provided.
+   * @param columns the columns to add to the struct in the order they should be added
+   * @return the new column view. It is the responsibility of the caller to close this.
+   */
+  public static ColumnView makeStructView(long rows, ColumnView... columns) {
+    long[] handles = new long[columns.length];
+    for (int i = 0; i < columns.length; i++) {
+      ColumnView cv = columns[i];
+      if (rows != cv.getRowCount()) {
+        throw new IllegalArgumentException("All columns must have the same number of rows");
+      }
+      handles[i] = cv.getNativeView();
+    }
+    return new ColumnView(makeStructView(handles, rows));
+  }
+
+  /**
+   * Create a new struct column view of existing column views. Note that this will NOT copy
+   * the contents of the input columns to make a new vector, but makes a view that must not
+   * outlive the child views that it references. The resulting column cannot be null.
+   * @param columns the columns to add to the struct in the order they should be added
+   * @return the new column view. It is the responsibility of the caller to close this.
+   */
+  public static ColumnView makeStructView(ColumnView... columns) {
+    if (columns.length <= 0) {
+      throw new IllegalArgumentException("At least one column is needed to get the row count");
+    }
+    return makeStructView(columns[0].rows, columns);
+  }
+
   /////////////////////////////////////////////////////////////////////////////
   // INTERNAL/NATIVE ACCESS
   /////////////////////////////////////////////////////////////////////////////
@@ -2620,7 +2656,9 @@ private static native long rollingWindow(
   private static native long clamper(long nativeView, long loScalarHandle, long loScalarReplaceHandle,
                                      long hiScalarHandle, long hiScalarReplaceHandle);
 
-  protected native long title(long handle);
+  protected static native long title(long handle);
+
+  private static native long makeStructView(long[] handles, long rowCount);
 
   private static native long isTimestamp(long nativeView, String format);
   /**
@@ -2755,7 +2793,7 @@ static ColumnVector createColumnVector(DType type, int rows, HostMemoryBuffer da
         mainOffsetsDevBuff.copyFromHostBuffer(mainColOffsets, 0, offsetsLen);
       }
       List<DeviceMemoryBuffer> toClose = new ArrayList<>();
-      long[] childHandles = (devChildren.isEmpty()) ? null : new long[devChildren.size()];
+      long[] childHandles = new long[devChildren.size()];
       for (ColumnView.NestedColumnVector ncv : devChildren) {
         toClose.addAll(ncv.getBuffersToClose());
       }
diff --git a/java/src/main/java/ai/rapids/cudf/HostColumnVector.java b/java/src/main/java/ai/rapids/cudf/HostColumnVector.java
index a5689976f0b..559256aa7bf 100644
--- a/java/src/main/java/ai/rapids/cudf/HostColumnVector.java
+++ b/java/src/main/java/ai/rapids/cudf/HostColumnVector.java
@@ -1,6 +1,6 @@
 /*
  *
- *  Copyright (c) 2020, NVIDIA CORPORATION.
+ *  Copyright (c) 2020-2021, NVIDIA CORPORATION.
  *
  *  Licensed under the Apache License, Version 2.0 (the "License");
  *  you may not use this file except in compliance with the License.
@@ -303,6 +303,16 @@ public static HostColumnVector fromStructs(DataType dataType, StructData... valu
     }
   }
 
+  public static HostColumnVector emptyStructs(DataType dataType, long rows) {
+    StructData sd = new StructData();
+    try (ColumnBuilder cb = new ColumnBuilder(dataType, rows)) {
+      for (long i = 0; i < rows; i++) {
+        cb.append(sd);
+      }
+      return cb.build();
+    }
+  }
+
   /**
    * Create a new vector from the given values.
    */
diff --git a/java/src/main/native/src/ColumnViewJni.cpp b/java/src/main/native/src/ColumnViewJni.cpp
index 03a4f2acafd..621344ac38f 100644
--- a/java/src/main/native/src/ColumnViewJni.cpp
+++ b/java/src/main/native/src/ColumnViewJni.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019-2020, NVIDIA CORPORATION.
+ * Copyright (c) 2019-2021, NVIDIA CORPORATION.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -1252,6 +1252,7 @@ JNIEXPORT jlong JNICALL Java_ai_rapids_cudf_ColumnView_bitwiseMergeAndSetValidit
   JNI_NULL_CHECK(env, base_column, "base column native handle is null", 0);
   JNI_NULL_CHECK(env, column_handles, "array of column handles is null", 0);
   try {
+    cudf::jni::auto_set_device(env);
     cudf::column_view *original_column = reinterpret_cast<cudf::column_view *>(base_column);
     std::unique_ptr<cudf::column> copy(new cudf::column(*original_column));
     cudf::jni::native_jpointerArray<cudf::column_view> n_cudf_columns(env, column_handles);
@@ -1333,7 +1334,7 @@ JNIEXPORT jlong JNICALL Java_ai_rapids_cudf_ColumnView_makeCudfColumnView(JNIEnv
       ret.reset(new cudf::column_view(cudf::data_type{cudf::type_id::LIST}, size, nullptr, valid,
         j_null_count, 0, {offsets_column, *children[0]}));
    } else if (n_type == cudf::type_id::STRUCT) {
-     JNI_NULL_CHECK(env, j_children, "children of a list are null", 0);
+     JNI_NULL_CHECK(env, j_children, "children of a struct are null", 0);
      cudf::jni::native_jpointerArray<cudf::column_view> children(env, j_children);
      std::vector<column_view> children_vector(children.size());
      for (int i = 0; i < children.size(); i++) {
@@ -1637,6 +1638,27 @@ JNIEXPORT jlong JNICALL Java_ai_rapids_cudf_ColumnView_title(JNIEnv *env, jobjec
   CATCH_STD(env, 0);
 }
 
+JNIEXPORT jlong JNICALL Java_ai_rapids_cudf_ColumnView_makeStructView(JNIEnv *env, jobject j_object,
+                                                                      jlongArray handles,
+                                                                      jlong row_count) {
+
+  JNI_NULL_CHECK(env, handles, "native view handles are null", 0)
+  try {
+    cudf::jni::auto_set_device(env);
+    std::unique_ptr<cudf::column_view> ret;
+    cudf::jni::native_jpointerArray<cudf::column_view> children(env, handles);
+    std::vector<cudf::column_view> children_vector(children.size());
+    for (int i = 0; i < children.size(); i++) {
+      children_vector[i] = *children[i];
+    }
+    ret.reset(new cudf::column_view(cudf::data_type{cudf::type_id::STRUCT}, row_count, nullptr,
+                nullptr, 0, 0, children_vector));
+
+    return reinterpret_cast<jlong>(ret.release());
+  }
+  CATCH_STD(env, 0);
+}
+
 JNIEXPORT jlong JNICALL Java_ai_rapids_cudf_ColumnView_nansToNulls(JNIEnv *env, jobject j_object,
                                                                    jlong handle) {
 
diff --git a/java/src/test/java/ai/rapids/cudf/ColumnVectorTest.java b/java/src/test/java/ai/rapids/cudf/ColumnVectorTest.java
index d3cde7f77cc..54646583831 100644
--- a/java/src/test/java/ai/rapids/cudf/ColumnVectorTest.java
+++ b/java/src/test/java/ai/rapids/cudf/ColumnVectorTest.java
@@ -1,6 +1,6 @@
 /*
  *
- *  Copyright (c) 2019-2020, NVIDIA CORPORATION.
+ *  Copyright (c) 2019-2021, NVIDIA CORPORATION.
  *
  *  Licensed under the Apache License, Version 2.0 (the "License");
  *  you may not use this file except in compliance with the License.
@@ -3673,4 +3673,30 @@ void testCopyToColumnVector() {
       assertColumnsAreEqual(expected, copiedChildCv);
     }
   }
+
+  @Test
+  void testMakeStructEmpty() {
+    final int numRows = 10;
+    try (ColumnVector expected = ColumnVector.emptyStructs(new StructType(false, new ArrayList<>()), numRows);
+         ColumnVector created = ColumnVector.makeStruct(numRows)) {
+      assertColumnsAreEqual(expected, created);
+    }
+  }
+
+  @Test
+  void testMakeStruct() {
+    try (ColumnVector expected = ColumnVector.fromStructs(new StructType(false,
+            Arrays.asList(
+                new BasicType(false, DType.INT32),
+                new BasicType(false, DType.INT32),
+                new BasicType(false, DType.INT32))),
+        new HostColumnVector.StructData(1, 2, 3),
+        new HostColumnVector.StructData(4, 5, 6));
+         ColumnVector child1 = ColumnVector.fromInts(1, 4);
+         ColumnVector child2 = ColumnVector.fromInts(2, 5);
+         ColumnVector child3 = ColumnVector.fromInts(3, 6);
+         ColumnVector created = ColumnVector.makeStruct(child1, child2, child3)) {
+      assertColumnsAreEqual(expected, created);
+    }
+  }
 }

From 9439ed8fca9a89ab460608815c29269335ba9fd1 Mon Sep 17 00:00:00 2001
From: GALI PREM SAGAR <sagarprem75@gmail.com>
Date: Thu, 7 Jan 2021 10:22:28 -0600
Subject: [PATCH 13/20] Handle `nan` values correctly in
 `Series.one_hot_encoding`(#7059)

Fixes: #7056

This PR handles `nan` values separately in `one_hot_encoding` when the given input category is `None`. Previously we were combining both `nan` & `<NA>` values to be the same when cat is `None`.

Authors:
  - galipremsagar <sagarprem75@gmail.com>

Approvers:
  - Keith Kraus (@kkraus14)

URL: https://github.com/rapidsai/cudf/pull/7059
---
 python/cudf/cudf/core/series.py             |  9 ++++++++-
 python/cudf/cudf/tests/test_label_encode.py |  5 -----
 python/cudf/cudf/tests/test_onehot.py       | 18 ++++++++++++++++--
 3 files changed, 24 insertions(+), 8 deletions(-)

diff --git a/python/cudf/cudf/core/series.py b/python/cudf/cudf/core/series.py
index 28fb10ea334..9602cf8d473 100644
--- a/python/cudf/cudf/core/series.py
+++ b/python/cudf/cudf/core/series.py
@@ -2448,7 +2448,14 @@ def one_hot_encoding(self, cats, dtype="float64"):
 
         def encode(cat):
             if cat is None:
-                return self.isnull()
+                if self.dtype.kind == "f":
+                    # Need to ignore `np.nan` values incase
+                    # of a float column
+                    return self.__class__(
+                        libcudf.unary.is_null((self._column))
+                    )
+                else:
+                    return self.isnull()
             elif np.issubdtype(type(cat), np.floating) and np.isnan(cat):
                 return self.__class__(libcudf.unary.is_nan(self._column))
             else:
diff --git a/python/cudf/cudf/tests/test_label_encode.py b/python/cudf/cudf/tests/test_label_encode.py
index bb4921aac8c..29a787768f2 100644
--- a/python/cudf/cudf/tests/test_label_encode.py
+++ b/python/cudf/cudf/tests/test_label_encode.py
@@ -123,8 +123,3 @@ def test_label_encode_dtype(ncats, cat_dtype):
     cats = s.unique().astype(s.dtype)
     encoded_col = s.label_encoding(cats=cats)
     np.testing.assert_equal(encoded_col.dtype, cat_dtype)
-
-
-if __name__ == "__main__":
-    test_label_encode()
-    test_label_encode_drop_one()
diff --git a/python/cudf/cudf/tests/test_onehot.py b/python/cudf/cudf/tests/test_onehot.py
index 2c87508c3f5..8b5b0609f31 100644
--- a/python/cudf/cudf/tests/test_onehot.py
+++ b/python/cudf/cudf/tests/test_onehot.py
@@ -189,5 +189,19 @@ def test_get_dummies_prefix_sep(prefix, prefix_sep):
     utils.assert_eq(encoded_expected, encoded_actual)
 
 
-if __name__ == "__main__":
-    test_onehot_random()
+def test_get_dummies_with_nan():
+    df = cudf.DataFrame(
+        {"a": cudf.Series([1, 2, np.nan, None], nan_as_null=False)}
+    )
+    expected = cudf.DataFrame(
+        {
+            "a_1.0": [1, 0, 0, 0],
+            "a_2.0": [0, 1, 0, 0],
+            "a_nan": [0, 0, 1, 0],
+            "a_null": [0, 0, 0, 1],
+        },
+        dtype="uint8",
+    )
+    actual = cudf.get_dummies(df, dummy_na=True, columns=["a"])
+
+    utils.assert_eq(expected, actual)

From ee65a4715b14adacd6d85e14521ce1d4b247f3bc Mon Sep 17 00:00:00 2001
From: GALI PREM SAGAR <sagarprem75@gmail.com>
Date: Thu, 7 Jan 2021 11:07:25 -0600
Subject: [PATCH 14/20] Add `pyorc` to dev environment(#7085)

This PR adds `pyorc` package to dev environment yml files.

Authors:
  - galipremsagar <sagarprem75@gmail.com>

Approvers:
  - Christopher Harris (@cwharris)
  - AJ Schmidt (@ajschmidt8)

URL: https://github.com/rapidsai/cudf/pull/7085
---
 conda/environments/cudf_dev_cuda10.1.yml | 3 +++
 conda/environments/cudf_dev_cuda10.2.yml | 3 +++
 conda/environments/cudf_dev_cuda11.0.yml | 3 +++
 3 files changed, 9 insertions(+)

diff --git a/conda/environments/cudf_dev_cuda10.1.yml b/conda/environments/cudf_dev_cuda10.1.yml
index 3de28fe30b0..24882d9b3e2 100644
--- a/conda/environments/cudf_dev_cuda10.1.yml
+++ b/conda/environments/cudf_dev_cuda10.1.yml
@@ -1,3 +1,5 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+
 name: cudf_dev
 channels:
   - rapidsai
@@ -60,3 +62,4 @@ dependencies:
       - git+https://github.com/dask/dask.git@master
       - git+https://github.com/dask/distributed.git@master
       - git+https://github.com/python-streamz/streamz.git
+      - pyorc
diff --git a/conda/environments/cudf_dev_cuda10.2.yml b/conda/environments/cudf_dev_cuda10.2.yml
index 7055228943a..49675fe2154 100644
--- a/conda/environments/cudf_dev_cuda10.2.yml
+++ b/conda/environments/cudf_dev_cuda10.2.yml
@@ -1,3 +1,5 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+
 name: cudf_dev
 channels:
   - rapidsai
@@ -60,3 +62,4 @@ dependencies:
       - git+https://github.com/dask/dask.git@master
       - git+https://github.com/dask/distributed.git@master
       - git+https://github.com/python-streamz/streamz.git
+      - pyorc
diff --git a/conda/environments/cudf_dev_cuda11.0.yml b/conda/environments/cudf_dev_cuda11.0.yml
index 497d8feefea..2917c2c3ce0 100644
--- a/conda/environments/cudf_dev_cuda11.0.yml
+++ b/conda/environments/cudf_dev_cuda11.0.yml
@@ -1,3 +1,5 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+
 name: cudf_dev
 channels:
   - rapidsai
@@ -60,3 +62,4 @@ dependencies:
       - git+https://github.com/dask/dask.git@master
       - git+https://github.com/dask/distributed.git@master
       - git+https://github.com/python-streamz/streamz.git
+      - pyorc

From aa38f85b38cde2c9dc4c5cccd0286d0212e233fb Mon Sep 17 00:00:00 2001
From: GALI PREM SAGAR <sagarprem75@gmail.com>
Date: Thu, 7 Jan 2021 13:37:25 -0600
Subject: [PATCH 15/20] Add informative error message for `sep` in CSV
 writer(#7095)

Fixes: #7091

This PR introduces validation and throwing of informative error messages for the `sep` parameter in csv writer.

Authors:
  - galipremsagar <sagarprem75@gmail.com>

Approvers:
  - Keith Kraus (@kkraus14)
  - Vukasin Milovanovic (@vuule)

URL: https://github.com/rapidsai/cudf/pull/7095
---
 python/cudf/cudf/io/csv.py         |  7 ++++++-
 python/cudf/cudf/tests/test_csv.py | 24 ++++++++++++++++++++++--
 2 files changed, 28 insertions(+), 3 deletions(-)

diff --git a/python/cudf/cudf/io/csv.py b/python/cudf/cudf/io/csv.py
index 2766e0d79d0..fd040181ca8 100644
--- a/python/cudf/cudf/io/csv.py
+++ b/python/cudf/cudf/io/csv.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2018-2020, NVIDIA CORPORATION.
+# Copyright (c) 2018-2021, NVIDIA CORPORATION.
 
 from io import BytesIO, StringIO
 
@@ -119,6 +119,11 @@ def to_csv(
 ):
     """{docstring}"""
 
+    if not isinstance(sep, str):
+        raise TypeError(f'"sep" must be string, not {type(sep).__name__}')
+    elif len(sep) > 1:
+        raise TypeError('"sep" must be a 1-character string')
+
     return_as_string = False
     if path_or_buf is None:
         path_or_buf = StringIO()
diff --git a/python/cudf/cudf/tests/test_csv.py b/python/cudf/cudf/tests/test_csv.py
index 94a5f061383..0ea8b3add4b 100644
--- a/python/cudf/cudf/tests/test_csv.py
+++ b/python/cudf/cudf/tests/test_csv.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2018-2020, NVIDIA CORPORATION.
+# Copyright (c) 2018-2021, NVIDIA CORPORATION.
 
 import csv
 import gzip
@@ -15,7 +15,7 @@
 
 import cudf
 from cudf import read_csv
-from cudf.tests.utils import assert_eq
+from cudf.tests.utils import assert_eq, assert_exceptions_equal
 
 
 def make_numeric_dataframe(nrows, dtype):
@@ -1935,3 +1935,23 @@ def test_na_filter_empty_fields():
         StringIO(buffer), keep_default_na=False, na_values=test_na
     )
     assert_eq(pdf, gdf)
+
+
+def test_csv_sep_error():
+    pdf = pd.DataFrame({"a": [1, 2, 3]})
+    gdf = cudf.DataFrame({"a": [1, 2, 3]})
+    assert_exceptions_equal(
+        lfunc=pdf.to_csv,
+        rfunc=gdf.to_csv,
+        lfunc_args_and_kwargs=([], {"sep": "abc"}),
+        rfunc_args_and_kwargs=([], {"sep": "abc"}),
+        expected_error_message='"sep" must be a 1-character string',
+    )
+
+    assert_exceptions_equal(
+        lfunc=pdf.to_csv,
+        rfunc=gdf.to_csv,
+        lfunc_args_and_kwargs=([], {"sep": 1}),
+        rfunc_args_and_kwargs=([], {"sep": 1}),
+        expected_error_message='"sep" must be string, not int',
+    )

From 30e154c2647594b1abda7d9e14d2d54857b3e9ca Mon Sep 17 00:00:00 2001
From: Alfred Xu <lovedreamf@gmail.com>
Date: Fri, 8 Jan 2021 23:07:25 +0800
Subject: [PATCH 16/20] Add Java tests for decimal casts(#7051)

This pull request attempts to verify the support of decimal cast in terms of java package.

Authors:
  - sperlingxx <lovedreamf@gmail.com>

Approvers:
  - Jason Lowe (@jlowe)

URL: https://github.com/rapidsai/cudf/pull/7051
---
 CHANGELOG.md                                  |   1 +
 .../java/ai/rapids/cudf/ColumnVectorTest.java | 131 ++++++++++++++++++
 2 files changed, 132 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index e895a657de8..04ffc5868e8 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -22,6 +22,7 @@
 - PR #6939 Use simplified `rmm::exec_policy`
 - PR #6512 Refactor rolling.cu to reduce compile time
 - PR #6982 Disable some pragma unroll statements in thrust `sort.h`
+- PR #7051 Verify decimal cast in java package
 
 ## Bug Fixes
 
diff --git a/java/src/test/java/ai/rapids/cudf/ColumnVectorTest.java b/java/src/test/java/ai/rapids/cudf/ColumnVectorTest.java
index 54646583831..738bacfe130 100644
--- a/java/src/test/java/ai/rapids/cudf/ColumnVectorTest.java
+++ b/java/src/test/java/ai/rapids/cudf/ColumnVectorTest.java
@@ -2215,6 +2215,137 @@ private static void testCastFixedWidthToStringsAndBack(DType type, Supplier<Colu
     }
   }
 
+  @Test
+  void testCastIntToDecimal() {
+    testCastNumericToDecimalsAndBack(DType.INT32, true, 0,
+        () -> ColumnVector.fromBoxedInts(1, -21, 345, null, 8008, Integer.MIN_VALUE, Integer.MAX_VALUE),
+        () -> ColumnVector.fromBoxedInts(1, -21, 345, null, 8008, Integer.MIN_VALUE, Integer.MAX_VALUE),
+        new Long[]{1L, -21L, 345L, null, 8008L, (long) Integer.MIN_VALUE, (long) Integer.MAX_VALUE}
+    );
+    testCastNumericToDecimalsAndBack(DType.INT32, false, -2,
+        () -> ColumnVector.fromBoxedInts(1, -21, 345, null, 8008, 0, 123456),
+        () -> ColumnVector.fromBoxedInts(1, -21, 345, null, 8008, 0, 123456),
+        new Long[]{100L, -2100L, 34500L, null, 800800L, 0L, 12345600L}
+    );
+    testCastNumericToDecimalsAndBack(DType.INT32, false, 2,
+        () -> ColumnVector.fromBoxedInts(1, -21, 345, null, 8008, 0, 123456),
+        () -> ColumnVector.fromBoxedInts(0, 0, 300, null, 8000, 0, 123400),
+        new Long[]{0L, 0L, 3L, null, 80L, 0L, 1234L}
+    );
+  }
+
+  @Test
+  void testCastLongToDecimal() {
+    testCastNumericToDecimalsAndBack(DType.INT64, false, 0,
+        () -> ColumnVector.fromBoxedLongs(1L, -21L, 345L, null, 8008L, Long.MIN_VALUE, Long.MAX_VALUE),
+        () -> ColumnVector.fromBoxedLongs(1L, -21L, 345L, null, 8008L, Long.MIN_VALUE, Long.MAX_VALUE),
+        new Long[]{1L, -21L, 345L, null, 8008L, Long.MIN_VALUE, Long.MAX_VALUE}
+    );
+    testCastNumericToDecimalsAndBack(DType.INT64, false, -1,
+        () -> ColumnVector.fromBoxedLongs(1L, -21L, 345L, null, 8008L, 0L, 123456L),
+        () -> ColumnVector.fromBoxedLongs(1L, -21L, 345L, null, 8008L, 0L, 123456L),
+        new Long[]{10L, -210L, 3450L, null, 80080L, 0L, 1234560L}
+    );
+    testCastNumericToDecimalsAndBack(DType.INT64, false, 1,
+        () -> ColumnVector.fromBoxedLongs(1L, -21L, 345L, null, 8018L, 0L, 123456L),
+        () -> ColumnVector.fromBoxedLongs(0L, -20L, 340L, null, 8010L, 0L, 123450L),
+        new Long[]{0L, -2L, 34L, null, 801L, 0L, 12345L}
+    );
+  }
+
+  @Test
+  void testCastFloatToDecimal() {
+    testCastNumericToDecimalsAndBack(DType.FLOAT32, true, 0,
+        () -> ColumnVector.fromBoxedFloats(1.0f, 2.1f, -3.23f, null, 2.41281f, 1378952.001f),
+        () -> ColumnVector.fromBoxedFloats(1f, 2f, -3f, null, 2f, 1378952f),
+        new Long[]{1L, 2L, -3L, null, 2L, 1378952L}
+    );
+    testCastNumericToDecimalsAndBack(DType.FLOAT32, true, -1,
+        () -> ColumnVector.fromBoxedFloats(1.0f, 2.1f, -3.23f, null, 2.41281f, 1378952.001f),
+        () -> ColumnVector.fromBoxedFloats(1f, 2.1f, -3.2f, null, 2.4f, 1378952f),
+        new Long[]{10L, 21L, -32L, null, 24L, 13789520L}
+    );
+    testCastNumericToDecimalsAndBack(DType.FLOAT32, true, 1,
+        () -> ColumnVector.fromBoxedFloats(1.0f, 21.1f, -300.23f, null, 24128.1f, 1378952.001f),
+        () -> ColumnVector.fromBoxedFloats(0f, 20f, -300f, null, 24120f, 1378950f),
+        new Long[]{0L, 2L, -30L, null, 2412L, 137895L}
+    );
+  }
+
+  @Test
+  void testCastDoubleToDecimal() {
+    testCastNumericToDecimalsAndBack(DType.FLOAT64, false, 0,
+        () -> ColumnVector.fromBoxedDoubles(1.0, 2.1, -3.23, null, 2.41281, (double) Long.MAX_VALUE),
+        () -> ColumnVector.fromBoxedDoubles(1.0, 2.0, -3.0, null, 2.0, (double) Long.MAX_VALUE),
+        new Long[]{1L, 2L, -3L, null, 2L, Long.MAX_VALUE}
+    );
+    testCastNumericToDecimalsAndBack(DType.FLOAT64, false, -2,
+        () -> ColumnVector.fromBoxedDoubles(1.0, 2.1, -3.23, null, 2.41281, -55.01999),
+        () -> ColumnVector.fromBoxedDoubles(1.0, 2.1, -3.23, null, 2.41, -55.01),
+        new Long[]{100L, 210L, -323L, null, 241L, -5501L}
+    );
+    testCastNumericToDecimalsAndBack(DType.FLOAT64, false, 1,
+        () -> ColumnVector.fromBoxedDoubles(1.0, 23.1, -3089.23, null, 200.41281, -199.01999),
+        () -> ColumnVector.fromBoxedDoubles(0.0, 20.0, -3080.0, null, 200.0, -190.0),
+        new Long[]{0L, 2L, -308L, null, 20L, -19L}
+    );
+  }
+
+  @Test
+  void testCastDecimalToDecimal() {
+    // DECIMAL32(scale: 0) -> DECIMAL32(scale: 0)
+    testCastNumericToDecimalsAndBack(DType.create(DType.DTypeEnum.DECIMAL32, 0), true, -0,
+        () -> ColumnVector.decimalFromInts(0, 1, 12, -234, 5678, Integer.MIN_VALUE / 100),
+        () -> ColumnVector.decimalFromInts(0, 1, 12, -234, 5678, Integer.MIN_VALUE / 100),
+        new Long[]{1L, 12L, -234L, 5678L, (long) Integer.MIN_VALUE / 100}
+    );
+    // DECIMAL32(scale: 0) -> DECIMAL64(scale: -2)
+    testCastNumericToDecimalsAndBack(DType.create(DType.DTypeEnum.DECIMAL32, 0), false, -2,
+        () -> ColumnVector.decimalFromInts(0, 1, 12, -234, 5678, Integer.MIN_VALUE / 100),
+        () -> ColumnVector.decimalFromInts(0, 1, 12, -234, 5678, Integer.MIN_VALUE / 100),
+        new Long[]{100L, 1200L, -23400L, 567800L, (long) Integer.MIN_VALUE / 100 * 100}
+    );
+    // DECIMAL64(scale: -3) -> DECIMAL64(scale: -1)
+    DType dt = DType.create(DType.DTypeEnum.DECIMAL64, -3);
+    testCastNumericToDecimalsAndBack(dt, false, -1,
+        () -> ColumnVector.decimalFromDoubles(dt, RoundingMode.UNNECESSARY, -1000.1, 1.222, 0.03, -4.678, 16789431.0),
+        () -> ColumnVector.decimalFromDoubles(dt, RoundingMode.UNNECESSARY, -1000.1, 1.2, 0, -4.6, 16789431.0),
+        new Long[]{-10001L, 12L, 0L, -46L, 167894310L}
+    );
+    // DECIMAL64(scale: -3) -> DECIMAL64(scale: 2)
+    DType dt2 = DType.create(DType.DTypeEnum.DECIMAL64, -3);
+    testCastNumericToDecimalsAndBack(dt2, false, 2,
+        () -> ColumnVector.decimalFromDoubles(dt2, RoundingMode.UNNECESSARY, -1013.1, 14.222, 780.03, -4.678, 16789431.0),
+        () -> ColumnVector.decimalFromDoubles(dt2, RoundingMode.UNNECESSARY, -1000, 0, 700, 0, 16789400),
+        new Long[]{-10L, 0L, 7L, 0L, 167894L}
+    );
+    // DECIMAL64(scale: -3) -> DECIMAL32(scale: -3)
+    testCastNumericToDecimalsAndBack(dt2, true, -3,
+        () -> ColumnVector.decimalFromDoubles(dt2, RoundingMode.UNNECESSARY, -1013.1, 14.222, 780.03, -4.678, 16789.0),
+        () -> ColumnVector.decimalFromDoubles(dt2, RoundingMode.UNNECESSARY, -1013.1, 14.222, 780.03, -4.678, 16789.0),
+        new Long[]{-1013100L, 14222L, 780030L, -4678L, 16789000L}
+    );
+  }
+
+  private static void testCastNumericToDecimalsAndBack(DType sourceType, boolean isDec32, int scale,
+                                                       Supplier<ColumnVector> sourceData,
+                                                       Supplier<ColumnVector> returnData,
+                                                       Long[] unscaledDecimal) {
+    DType decimalType = DType.create(isDec32 ? DType.DTypeEnum.DECIMAL32 : DType.DTypeEnum.DECIMAL64, scale);
+    try (ColumnVector sourceColumn = sourceData.get();
+         ColumnVector expectedColumn = returnData.get();
+         ColumnVector decimalColumn = sourceColumn.castTo(decimalType);
+         HostColumnVector hostDecimalColumn = decimalColumn.copyToHost();
+         ColumnVector returnColumn = decimalColumn.castTo(sourceType)) {
+      for (int i = 0; i < sourceColumn.rows; i++) {
+        Long actual = hostDecimalColumn.isNull(i) ? null :
+            (isDec32 ? hostDecimalColumn.getInt(i) : hostDecimalColumn.getLong(i));
+        assertEquals(unscaledDecimal[i], actual);
+      }
+      assertColumnsAreEqual(expectedColumn, returnColumn);
+    }
+  }
+
   @Test
   void testIsTimestamp() {
       final String[] TIMESTAMP_STRINGS = {

From 04aa30cf11a930a04675df40429282fa6a41c5b6 Mon Sep 17 00:00:00 2001
From: Ashwin Srinath <3190405+shwina@users.noreply.github.com>
Date: Fri, 8 Jan 2021 17:52:25 -0500
Subject: [PATCH 17/20] Add groupby docs(#7100)

Closes #6858. Adds a "GroupBy" page to our docs.

Authors:
  - Ashwin Srinath <shwina@users.noreply.github.com>

Approvers:
  - Keith Kraus (@kkraus14)

URL: https://github.com/rapidsai/cudf/pull/7100
---
 docs/cudf/source/groupby.md | 168 ++++++++++++++++++++++++++++++++++++
 docs/cudf/source/index.rst  |   1 +
 2 files changed, 169 insertions(+)
 create mode 100644 docs/cudf/source/groupby.md

diff --git a/docs/cudf/source/groupby.md b/docs/cudf/source/groupby.md
new file mode 100644
index 00000000000..4d775e3d51a
--- /dev/null
+++ b/docs/cudf/source/groupby.md
@@ -0,0 +1,168 @@
+GroupBy
+=======
+
+cuDF supports a small (but important) subset of
+Pandas' [groupby API](https://pandas.pydata.org/pandas-docs/stable/user_guide/groupby.html).
+
+## Summary of supported operations
+
+1. Grouping by one or more columns
+1. Basic aggregations such as "sum", "mean", etc.
+1. Quantile aggregation
+1. A "collect" or `list` aggregation for collecting values in a group into lists
+1. Automatic exclusion of columns with unsupported dtypes ("nuisance" columns) when aggregating
+1. Iterating over the groups of a GroupBy object
+1. `GroupBy.groups` API that returns a mapping of group keys to row labels
+1. `GroupBy.apply` API for performing arbitrary operations on each group. Note that
+   this has very limited functionality compared to the equivalent Pandas function.
+   See the section on [apply](#groupby-apply) for more details.
+1. `GroupBy.pipe` similar to [Pandas](https://pandas.pydata.org/pandas-docs/stable/user_guide/groupby.html#piping-function-calls).
+
+## Grouping
+
+A GroupBy object is created by grouping the values of a `Series` or `DataFrame`
+by one or more columns:
+
+```python
+import cudf
+
+>>> df = cudf.DataFrame({'a': [1, 1, 1, 2, 2], 'b': [1, 1, 2, 2, 3], 'c': [1, 2, 3, 4, 5]})
+>>> df
+>>> gb1 = df.groupby('a')  # grouping by a single column
+>>> gb2 = df.groupby(['a', 'b'])  # grouping by multiple columns
+>>> gb3 = df.groupby(cudf.Series(['a', 'a', 'b', 'b', 'b']))  # grouping by an external column
+```
+
+### Grouping by index levels
+
+You can also group by one or more levels of a MultiIndex:
+
+```python
+>>> df = cudf.DataFrame(
+...     {'a': [1, 1, 1, 2, 2], 'b': [1, 1, 2, 2, 3], 'c': [1, 2, 3, 4, 5]}
+... ).set_index(['a', 'b'])
+...
+>>> df.groupby(level='a')
+```
+
+### The `Grouper` object
+
+A `Grouper` can be used to disambiguate between columns and levels when they have the same name:
+
+```python
+>>> df
+   b  c
+b
+1  1  1
+1  1  2
+1  2  3
+2  2  4
+2  3  5
+>>> df.groupby('b', level='b')  # ValueError: Cannot specify both by and level
+>>> df.groupby([cudf.Grouper(key='b'), cudf.Grouper(level='b')])  # OK
+```
+
+## Aggregation
+
+Aggregations on groups is supported via the `agg` method:
+
+```
+>>> df
+   a  b  c
+0  1  1  1
+1  1  1  2
+2  1  2  3
+3  2  2  4
+4  2  3  5
+>>> df.groupby('a').agg('sum')
+   b  c
+a
+1  4  6
+2  5  9
+>>> df.groupby('a').agg({'b': ['sum', 'min'], 'c': 'mean'})
+    b        c
+  sum min mean
+a
+1   4   1  2.0
+2   5   2  4.5
+```
+
+The following table summarizes the available aggregations and the types that support them:
+
+| Aggregations\dtypes | Numeric  | Datetime | String   | Categorical | List | Struct |
+| ------------------- | -------- | -------  | -------- | ----------- | ---- | ------ |
+| count               | ✅       | ✅       | ✅       | ✅          |      |        |
+| size                | ✅       | ✅       | ✅       | ✅          |      |        |
+| sum                 | ✅       | ✅       |          |             |      |        |
+| idxmin              | ✅       | ✅       |          |             |      |        |
+| idxmax              | ✅       | ✅       |          |             |      |        |
+| min                 | ✅       | ✅       | ✅       |             |      |        |
+| max                 | ✅       | ✅       | ✅       |             |      |        |
+| mean                | ✅       | ✅       |          |             |      |        |
+| var                 | ✅       | ✅       |          |             |      |        |
+| std                 | ✅       | ✅       |          |             |      |        |
+| quantile            | ✅       | ✅       |          |             |      |        |
+| median              | ✅       | ✅       |          |             |      |        |
+| nunique             | ✅       | ✅       | ✅       | ✅          |      |        |
+| nth                 | ✅       | ✅       | ✅       |             |      |        |
+| collect             | ✅       | ✅       | ✅       |             | ✅   |        |
+
+## GroupBy apply
+
+To apply function on each group, use the `GroupBy.apply()` method:
+
+```python
+>>> df
+   a  b  c
+0  1  1  1
+1  1  1  2
+2  1  2  3
+3  2  2  4
+4  2  3  5
+>>> df.groupby('a').apply(lambda x: x.max() - x.min())
+   a  b  c
+a
+0  0  1  2
+1  0  1  1
+```
+
+### Limitations
+
+* `apply` works by applying the provided function to each group sequentially,
+  and concatenating the results together. **This can be very slow**, especially
+  for a large number of small groups. For a small number of large groups, it
+  can give acceptable performance
+
+* The results may not always match Pandas exactly. For example, cuDF may return
+  a `DataFrame` containing a single column where Pandas returns a `Series`.
+  Some post-processing may be required to match Pandas behavior.
+
+* cuDF does not support some of the exceptional cases that Pandas supports with
+  `apply`, such as [`describe`](https://pandas.pydata.org/pandas-docs/stable/user_guide/groupby.html#flexible-apply).
+
+## Rolling window calculations
+
+Use the `GroupBy.rolling()` method to perform rolling window calculations on each group:
+
+```python
+>>> df
+   a  b  c
+0  1  1  1
+1  1  1  2
+2  1  2  3
+3  2  2  4
+4  2  3  5
+```
+
+Rolling window sum on each group with a window size of 2:
+
+```python
+>>> df.groupby('a').rolling(2).sum()
+        a     b     c
+a
+1 0  <NA>  <NA>  <NA>
+  1     2     2     3
+  2     2     3     5
+2 3  <NA>  <NA>  <NA>
+  4     4     5     9
+```
diff --git a/docs/cudf/source/index.rst b/docs/cudf/source/index.rst
index 8a100487374..65cdd619fe0 100644
--- a/docs/cudf/source/index.rst
+++ b/docs/cudf/source/index.rst
@@ -8,6 +8,7 @@ Welcome to cuDF's documentation!
    api.rst
    10min.ipynb
    basics.rst
+   groupby.md
    dask-cudf.md
    10min-cudf-cupy.ipynb
    guide-to-udfs.ipynb

From 11ebc3e089b3da786d6d6cf71596d8d2404a6d04 Mon Sep 17 00:00:00 2001
From: "Robert (Bobby) Evans" <bobby@apache.org>
Date: Mon, 11 Jan 2021 13:22:25 -0600
Subject: [PATCH 18/20] Adds in JNI support for creating an list column from
 existing columns(#7112)

Adds in java support to be able to create a list column from other existing columns.

Authors:
  - Robert (Bobby) Evans <bobby@apache.org>

Approvers:
  - Jason Lowe (@jlowe)

URL: https://github.com/rapidsai/cudf/pull/7112
---
 .../java/ai/rapids/cudf/ColumnVector.java     | 51 ++++++++++++++++++-
 java/src/main/native/src/ColumnVectorJni.cpp  | 47 ++++++++++++++++-
 .../java/ai/rapids/cudf/ColumnVectorTest.java | 36 +++++++++++++
 3 files changed, 132 insertions(+), 2 deletions(-)

diff --git a/java/src/main/java/ai/rapids/cudf/ColumnVector.java b/java/src/main/java/ai/rapids/cudf/ColumnVector.java
index e698d497b2b..88c024a437b 100644
--- a/java/src/main/java/ai/rapids/cudf/ColumnVector.java
+++ b/java/src/main/java/ai/rapids/cudf/ColumnVector.java
@@ -19,7 +19,6 @@
 package ai.rapids.cudf;
 
 import ai.rapids.cudf.HostColumnVector.Builder;
-import ai.rapids.cudf.WindowOptions.FrameType;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -350,6 +349,53 @@ public static ColumnVector makeStruct(long rows, ColumnView... columns) {
     }
   }
 
+  /**
+   * Create a LIST column from the given columns. Each list in the returned column will have the
+   * same number of entries in it as columns passed into this method. Be careful about the
+   * number of rows passed in as there are limits on the maximum output size supported for
+   * column lists.
+   * @param columns the columns to make up the list column, in the order they will appear in the
+   *                resulting lists.
+   * @return the new LIST ColumnVector
+   */
+  public static ColumnVector makeList(ColumnView... columns) {
+    if (columns.length <= 0) {
+      throw new IllegalArgumentException("At least one column is needed to get the row count");
+    }
+    return makeList(columns[0].getRowCount(), columns[0].getType(), columns);
+  }
+
+  /**
+   * Create a LIST column from the given columns. Each list in the returned column will have the
+   * same number of entries in it as columns passed into this method. Be careful about the
+   * number of rows passed in as there are limits on the maximum output size supported for
+   * column lists.
+   * @param rows the number of rows to create, for the special case of an empty list.
+   * @param type the type of the child column, for the special case of an empty list.
+   * @param columns the columns to make up the list column, in the order they will appear in the
+   *                resulting lists.
+   * @return the new LIST ColumnVector
+   */
+  public static ColumnVector makeList(long rows, DType type, ColumnView... columns) {
+    long[] handles = new long[columns.length];
+    for (int i = 0; i < columns.length; i++) {
+      ColumnView cv = columns[i];
+      if (rows != cv.getRowCount()) {
+        throw new IllegalArgumentException("All columns must have the same number of rows");
+      }
+      if (!type.equals(cv.getType())) {
+        throw new IllegalArgumentException("All columns must have the same type");
+      }
+
+      handles[i] = cv.getNativeView();
+    }
+    if (columns.length == 0 && type.isNestedType()) {
+      throw new IllegalArgumentException(
+          "Creating an empty list column of nested types is not currently supported");
+    }
+    return new ColumnVector(makeList(handles, type.typeId.nativeId, type.getScale(), rows));
+  }
+
   /**
    * Create a new vector of length rows, starting at the initialValue and going by step each time.
    * Only numeric types are supported.
@@ -571,6 +617,9 @@ public ColumnVector castTo(DType type) {
 
   private static native long fromScalar(long scalarHandle, int rowCount) throws CudfException;
 
+  private static native long makeList(long[] handles, long typeHandle, int scale, long rows)
+      throws CudfException;
+
   private static native long concatenate(long[] viewHandles) throws CudfException;
 
   /**
diff --git a/java/src/main/native/src/ColumnVectorJni.cpp b/java/src/main/native/src/ColumnVectorJni.cpp
index 8aadb4011e1..3bce4912fa4 100644
--- a/java/src/main/native/src/ColumnVectorJni.cpp
+++ b/java/src/main/native/src/ColumnVectorJni.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019-2020, NVIDIA CORPORATION.
+ * Copyright (c) 2019-2021, NVIDIA CORPORATION.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -18,9 +18,11 @@
 #include <cudf/concatenate.hpp>
 #include <cudf/filling.hpp>
 #include <cudf/hashing.hpp>
+#include <cudf/reshape.hpp>
 #include <cudf/utilities/bit.hpp>
 #include <cudf/lists/detail/concatenate.hpp>
 #include <cudf/lists/lists_column_view.hpp>
+#include <cudf/scalar/scalar_factories.hpp>
 #include <cudf/structs/structs_column_view.hpp>
 
 #include "cudf_jni_apis.hpp"
@@ -48,6 +50,49 @@ JNIEXPORT jlong JNICALL Java_ai_rapids_cudf_ColumnVector_sequence(JNIEnv *env, j
   CATCH_STD(env, 0);
 }
 
+JNIEXPORT jlong JNICALL Java_ai_rapids_cudf_ColumnVector_makeList(JNIEnv *env, jobject j_object,
+                                                                  jlongArray handles,
+                                                                  jlong j_type,
+                                                                  jint scale,
+                                                                  jlong row_count) {
+  using ScalarType = cudf::scalar_type_t<cudf::size_type>;
+  JNI_NULL_CHECK(env, handles, "native view handles are null", 0)
+  try {
+    cudf::jni::auto_set_device(env);
+    std::unique_ptr<cudf::column> ret;
+    cudf::jni::native_jpointerArray<cudf::column_view> children(env, handles);
+    std::vector<cudf::column_view> children_vector(children.size());
+    for (int i = 0; i < children.size(); i++) {
+      children_vector[i] = *children[i];
+    }
+    auto zero = cudf::make_numeric_scalar(cudf::data_type(cudf::type_id::INT32));
+    zero->set_valid(true);
+    static_cast<ScalarType *>(zero.get())->set_value(0);
+
+    if (children.size() == 0) {
+      // special case because cudf::interleave_columns does not support no columns
+      auto offsets = cudf::make_column_from_scalar(*zero, row_count + 1);
+      cudf::type_id n_type = static_cast<cudf::type_id>(j_type);
+      cudf::data_type n_data_type = cudf::jni::make_data_type(j_type, scale);
+      auto empty_col = cudf::make_empty_column(n_data_type);
+      ret = cudf::make_lists_column(row_count, std::move(offsets), std::move(empty_col),
+              0, rmm::device_buffer());
+    } else {
+      auto count = cudf::make_numeric_scalar(cudf::data_type(cudf::type_id::INT32));
+      count->set_valid(true);
+      static_cast<ScalarType *>(count.get())->set_value(children.size());
+
+      std::unique_ptr<cudf::column> offsets = cudf::sequence(row_count + 1, *zero, *count);
+      auto data_col = cudf::interleave_columns(cudf::table_view(children_vector));
+      ret = cudf::make_lists_column(row_count, std::move(offsets), std::move(data_col),
+              0, rmm::device_buffer());
+    }
+
+    return reinterpret_cast<jlong>(ret.release());
+  }
+  CATCH_STD(env, 0);
+}
+
 JNIEXPORT jlong JNICALL Java_ai_rapids_cudf_ColumnVector_fromScalar(JNIEnv *env, jclass,
                                                                     jlong j_scalar,
                                                                     jint row_count) {
diff --git a/java/src/test/java/ai/rapids/cudf/ColumnVectorTest.java b/java/src/test/java/ai/rapids/cudf/ColumnVectorTest.java
index 738bacfe130..88ff50959f7 100644
--- a/java/src/test/java/ai/rapids/cudf/ColumnVectorTest.java
+++ b/java/src/test/java/ai/rapids/cudf/ColumnVectorTest.java
@@ -3830,4 +3830,40 @@ void testMakeStruct() {
       assertColumnsAreEqual(expected, created);
     }
   }
+
+  @Test
+  void testMakeListEmpty() {
+    final int numRows = 10;
+    try (ColumnVector expected =
+             ColumnVector.fromLists(
+                 new ListType(false, new BasicType(false, DType.STRING)),
+                 Arrays.asList(),
+                 Arrays.asList(),
+                 Arrays.asList(),
+                 Arrays.asList(),
+                 Arrays.asList(),
+                 Arrays.asList(),
+                 Arrays.asList(),
+                 Arrays.asList(),
+                 Arrays.asList(),
+                 Arrays.asList());
+         ColumnVector created = ColumnVector.makeList(numRows, DType.STRING)) {
+      assertColumnsAreEqual(expected, created);
+    }
+  }
+
+  @Test
+  void testMakeList() {
+    try (ColumnVector expected =
+             ColumnVector.fromLists(
+                 new ListType(false, new BasicType(false, DType.INT32)),
+                 Arrays.asList(1, 3, 5),
+                 Arrays.asList(2, 4, 6));
+         ColumnVector child1 = ColumnVector.fromInts(1, 2);
+         ColumnVector child2 = ColumnVector.fromInts(3, 4);
+         ColumnVector child3 = ColumnVector.fromInts(5, 6);
+         ColumnVector created = ColumnVector.makeList(child1, child2, child3)) {
+      assertColumnsAreEqual(expected, created);
+    }
+  }
 }

From 87e414c5bb5998c552273a5b8aab8f0a7b3750c4 Mon Sep 17 00:00:00 2001
From: Karthikeyan <6488848+karthikeyann@users.noreply.github.com>
Date: Tue, 12 Jan 2021 06:22:26 +0530
Subject: [PATCH 19/20] Add segmented_gather(list_column, gather_list)(#7003)

closes https://github.com/rapidsai/cudf/issues/6542

- [x] Add segmented_gather(list, list)
- [x] Add unit tests
- [x] Documentation

Authors:
  - Karthikeyan Natarajan <karthikeyann@users.noreply.github.com>
  - Karthikeyan <6488848+karthikeyann@users.noreply.github.com>

Approvers:
  - Vukasin Milovanovic (@vuule)
  - @nvdbaranec
  - AJ Schmidt (@ajschmidt8)
  - Jake Hemstad (@jrhemstad)

URL: https://github.com/rapidsai/cudf/pull/7003
---
 conda/recipes/libcudf/meta.yaml               |   1 +
 cpp/include/cudf/lists/detail/gather.cuh      |  15 +-
 cpp/include/cudf/lists/gather.hpp             |  63 +++
 cpp/src/copying/gather.cu                     |   2 -
 cpp/src/lists/copying/segmented_gather.cu     | 104 ++++
 cpp/tests/CMakeLists.txt                      |   1 +
 .../copying/segmented_gather_list_tests.cpp   | 475 ++++++++++++++++++
 7 files changed, 658 insertions(+), 3 deletions(-)
 create mode 100644 cpp/include/cudf/lists/gather.hpp
 create mode 100644 cpp/src/lists/copying/segmented_gather.cu
 create mode 100644 cpp/tests/copying/segmented_gather_list_tests.cpp

diff --git a/conda/recipes/libcudf/meta.yaml b/conda/recipes/libcudf/meta.yaml
index 0816799948e..0c820b76050 100644
--- a/conda/recipes/libcudf/meta.yaml
+++ b/conda/recipes/libcudf/meta.yaml
@@ -123,6 +123,7 @@ test:
     - test -f $PREFIX/include/cudf/lists/detail/concatenate.hpp
     - test -f $PREFIX/include/cudf/lists/detail/copying.hpp
     - test -f $PREFIX/include/cudf/lists/extract.hpp
+    - test -f $PREFIX/include/cudf/lists/gather.hpp
     - test -f $PREFIX/include/cudf/lists/lists_column_view.hpp
     - test -f $PREFIX/include/cudf/merge.hpp
     - test -f $PREFIX/include/cudf/null_mask.hpp
diff --git a/cpp/include/cudf/lists/detail/gather.cuh b/cpp/include/cudf/lists/detail/gather.cuh
index 0dcc4e2b37c..256e4e87a08 100644
--- a/cpp/include/cudf/lists/detail/gather.cuh
+++ b/cpp/include/cudf/lists/detail/gather.cuh
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2020, NVIDIA CORPORATION.
+ * Copyright (c) 2020-2021, NVIDIA CORPORATION.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -285,6 +285,19 @@ std::unique_ptr<column> gather_list_leaf(
   rmm::cuda_stream_view stream,
   rmm::mr::device_memory_resource* mr = rmm::mr::get_current_device_resource());
 
+/**
+ * @copydoc cudf::lists::segmented_gather(lists_column_view const& source_column,
+ *                                        lists_column_view const& gather_map_list,
+ *                                        rmm::mr::device_memory_resource* mr)
+ *
+ * @param stream CUDA stream on which to execute kernels
+ */
+std::unique_ptr<column> segmented_gather(
+  lists_column_view const& source_column,
+  lists_column_view const& gather_map_list,
+  rmm::cuda_stream_view stream        = rmm::cuda_stream_default,
+  rmm::mr::device_memory_resource* mr = rmm::mr::get_current_device_resource());
+
 }  // namespace detail
 }  // namespace lists
 }  // namespace cudf
diff --git a/cpp/include/cudf/lists/gather.hpp b/cpp/include/cudf/lists/gather.hpp
new file mode 100644
index 00000000000..66d7fb137b2
--- /dev/null
+++ b/cpp/include/cudf/lists/gather.hpp
@@ -0,0 +1,63 @@
+/*
+ * Copyright (c) 2021, NVIDIA CORPORATION.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#pragma once
+
+#include <cudf/column/column.hpp>
+#include <cudf/column/column_view.hpp>
+#include <cudf/lists/lists_column_view.hpp>
+
+namespace cudf {
+namespace lists {
+/**
+ * @addtogroup lists_gather
+ * @{
+ * @file
+ */
+
+/**
+ * @brief Segmented gather of the elements within a list element in each row of a list column.
+ *
+ * `source_column` with any depth and `gather_map_list` with depth 1 are only supported.
+ *
+ * * @code{.pseudo}
+ * source_column   : [{"a", "b", "c", "d"}, {"1", "2", "3", "4"}, {"x", "y", "z"}]
+ * gather_map_list : [{0, 1, 3, 2}, {1, 3, 2}, {}]
+ *
+ * result          : [{"a", "b", "d", "c"}, {"2", "4", "3"}, {}]
+ * @endcode
+ *
+ * @throws cudf::logic_error if `gather_map_list` size is not same as `source_column` size.
+ * @throws cudf::logic_error if gather_map contains null values.
+ * @throws cudf::logic_error if gather_map is not list column of an index type.
+ *
+ * If indices in `gather_map_list` are outside the range `[-n, n)`, where `n` is the number of
+ * elements in corresponding row of the source column, the behavior is undefined.
+ *
+ * @param source_column View into the list column to gather from
+ * @param gather_map_list View into a non-nullable list column of integral indices that maps the
+ * element in list of each row in the source columns to rows of lists in the destination columns.
+ * @param mr Device memory resource to allocate any returned objects
+ * @return column with elements in list of rows gathered based on `gather_map_list`
+ *
+ */
+std::unique_ptr<column> segmented_gather(
+  lists_column_view const& source_column,
+  lists_column_view const& gather_map_list,
+  rmm::mr::device_memory_resource* mr = rmm::mr::get_current_device_resource());
+
+/** @} */  // end of group
+}  // namespace lists
+}  // namespace cudf
diff --git a/cpp/src/copying/gather.cu b/cpp/src/copying/gather.cu
index 87721184536..dc153e9395d 100644
--- a/cpp/src/copying/gather.cu
+++ b/cpp/src/copying/gather.cu
@@ -25,8 +25,6 @@
 
 #include <rmm/cuda_stream_view.hpp>
 
-#include <thrust/count.h>
-
 namespace cudf {
 namespace detail {
 
diff --git a/cpp/src/lists/copying/segmented_gather.cu b/cpp/src/lists/copying/segmented_gather.cu
new file mode 100644
index 00000000000..407e41d4a7f
--- /dev/null
+++ b/cpp/src/lists/copying/segmented_gather.cu
@@ -0,0 +1,104 @@
+/*
+ * Copyright (c) 2020-2021, NVIDIA CORPORATION.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include <cudf/detail/gather.cuh>
+#include <cudf/detail/gather.hpp>
+#include <cudf/detail/indexalator.cuh>
+#include <cudf/detail/null_mask.hpp>
+#include <cudf/lists/detail/gather.cuh>
+
+#include <thrust/binary_search.h>
+#include <rmm/cuda_stream_view.hpp>
+
+namespace cudf {
+namespace lists {
+namespace detail {
+
+std::unique_ptr<column> segmented_gather(lists_column_view const& value_column,
+                                         lists_column_view const& gather_map,
+                                         rmm::cuda_stream_view stream,
+                                         rmm::mr::device_memory_resource* mr)
+{
+  CUDF_EXPECTS(is_index_type(gather_map.child().type()),
+               "Gather map should be list column of index type");
+  CUDF_EXPECTS(!gather_map.has_nulls(), "Gather map contains nulls");
+  CUDF_EXPECTS(value_column.size() == gather_map.size(),
+               "Gather map and list column should be same size");
+
+  auto gather_map_sliced_child = gather_map.get_sliced_child(stream);
+  auto const gather_map_size   = gather_map_sliced_child.size();
+  auto gather_index_begin      = gather_map.offsets().begin<size_type>() + 1 + gather_map.offset();
+  auto gather_index_end        = gather_index_begin + gather_map.size();
+  auto value_offsets           = value_column.offsets().begin<size_type>() + value_column.offset();
+  auto map_begin = cudf::detail::indexalator_factory::make_input_iterator(gather_map_sliced_child);
+
+  // Calculate Flattened gather indices  (value_offset[row]+sub_index
+  auto transformer = [value_offsets, map_begin, gather_index_begin, gather_index_end] __device__(
+                       size_type index) -> size_type {
+    // Get each row's offset. (Each row is a list).
+    auto offset_idx =
+      thrust::upper_bound(
+        thrust::seq, gather_index_begin, gather_index_end, gather_index_begin[-1] + index) -
+      gather_index_begin;
+    // Get each sub_index in list in each row of gather_map.
+    auto sub_index         = map_begin[index];
+    auto list_size         = value_offsets[offset_idx + 1] - value_offsets[offset_idx];
+    auto wrapped_sub_index = (sub_index % list_size + list_size) % list_size;
+    // Add sub_index to value_column offsets, to get gather indices of child of value_column
+    return value_offsets[offset_idx] + wrapped_sub_index - value_offsets[0];
+  };
+  auto child_gather_index_begin =
+    thrust::make_transform_iterator(thrust::make_counting_iterator<size_type>(0), transformer);
+  auto child_gather_index_end = child_gather_index_begin + gather_map_size;
+
+  // Call gather on child of value_column
+  auto child_table = cudf::detail::gather(table_view({value_column.get_sliced_child(stream)}),
+                                          child_gather_index_begin,
+                                          child_gather_index_end,
+                                          out_of_bounds_policy::DONT_CHECK,
+                                          stream,
+                                          mr);
+  auto child       = std::move(child_table->release().front());
+
+  // Create list offsets from gather_map.
+  auto output_offset = cudf::allocate_like(
+    gather_map.offsets(), gather_map.size() + 1, mask_allocation_policy::RETAIN, mr);
+  auto output_offset_view = output_offset->mutable_view();
+  cudf::copy_range_in_place(gather_map.offsets(),
+                            output_offset_view,
+                            gather_map.offset(),
+                            gather_map.offset() + output_offset_view.size(),
+                            0);
+  // Assemble list column & return
+  auto null_mask       = cudf::detail::copy_bitmask(value_column.parent(), stream, mr);
+  size_type null_count = value_column.null_count();
+  return make_lists_column(gather_map.size(),
+                           std::move(output_offset),
+                           std::move(child),
+                           null_count,
+                           std::move(null_mask));
+}
+
+}  // namespace detail
+
+std::unique_ptr<column> segmented_gather(lists_column_view const& source_column,
+                                         lists_column_view const& gather_map_list,
+                                         rmm::mr::device_memory_resource* mr)
+{
+  return detail::segmented_gather(source_column, gather_map_list, rmm::cuda_stream_default, mr);
+}
+
+}  // namespace lists
+}  // namespace cudf
diff --git a/cpp/tests/CMakeLists.txt b/cpp/tests/CMakeLists.txt
index 2592b5e4221..0d958f47b6b 100644
--- a/cpp/tests/CMakeLists.txt
+++ b/cpp/tests/CMakeLists.txt
@@ -376,6 +376,7 @@ set(COPYING_TEST_SRC
     "${CMAKE_CURRENT_SOURCE_DIR}/copying/gather_tests.cu"
     "${CMAKE_CURRENT_SOURCE_DIR}/copying/gather_str_tests.cu"
     "${CMAKE_CURRENT_SOURCE_DIR}/copying/gather_list_tests.cu"
+    "${CMAKE_CURRENT_SOURCE_DIR}/copying/segmented_gather_list_tests.cpp"
     "${CMAKE_CURRENT_SOURCE_DIR}/copying/gather_struct_tests.cu"
     "${CMAKE_CURRENT_SOURCE_DIR}/copying/detail_gather_tests.cu"
     "${CMAKE_CURRENT_SOURCE_DIR}/copying/scatter_tests.cpp"
diff --git a/cpp/tests/copying/segmented_gather_list_tests.cpp b/cpp/tests/copying/segmented_gather_list_tests.cpp
new file mode 100644
index 00000000000..b1abec9aa55
--- /dev/null
+++ b/cpp/tests/copying/segmented_gather_list_tests.cpp
@@ -0,0 +1,475 @@
+/*
+ * Copyright (c) 2020, NVIDIA CORPORATION.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include <cudf/column/column_view.hpp>
+#include <cudf/copying.hpp>
+#include <cudf/lists/detail/gather.cuh>
+#include <cudf/lists/lists_column_view.hpp>
+
+#include <cudf_test/base_fixture.hpp>
+#include <cudf_test/column_utilities.hpp>
+#include <cudf_test/column_wrapper.hpp>
+#include <cudf_test/type_lists.hpp>
+
+template <typename T>
+class SegmentedGatherTest : public cudf::test::BaseFixture {
+};
+using FixedWidthTypesNotBool = cudf::test::Concat<cudf::test::IntegralTypesNotBool,
+                                                  cudf::test::FloatingPointTypes,
+                                                  cudf::test::DurationTypes,
+                                                  cudf::test::TimestampTypes>;
+TYPED_TEST_CASE(SegmentedGatherTest, FixedWidthTypesNotBool);
+
+class SegmentedGatherTestList : public cudf::test::BaseFixture {
+};
+
+// to disambiguate between {} == 0 and {} == List{0}
+// Also, see note about compiler issues when declaring nested
+// empty lists in lists_column_wrapper documentation
+template <typename T>
+using LCW = cudf::test::lists_column_wrapper<T, int32_t>;
+using cudf::lists_column_view;
+
+TYPED_TEST(SegmentedGatherTest, Gather)
+{
+  using T = TypeParam;
+
+  // List<T>
+  LCW<T> list{{1, 2, 3, 4}, {5}, {6, 7}, {8, 9, 10}};
+  LCW<int> gather_map{{3, 2, 1, 0}, {0}, {0, 1}, {0, 2, 1}};
+  LCW<T> expected{{4, 3, 2, 1}, {5}, {6, 7}, {8, 10, 9}};
+
+  auto results =
+    cudf::lists::detail::segmented_gather(lists_column_view{list}, lists_column_view{gather_map});
+
+  CUDF_TEST_EXPECT_COLUMNS_EQUAL(results->view(), expected);
+}
+
+TYPED_TEST(SegmentedGatherTest, GatherNothing)
+{
+  using T = TypeParam;
+  using namespace cudf;
+
+  // List<T>
+  {
+    LCW<T> list{{1, 2, 3, 4}, {5}, {6, 7}, {8, 9, 10}};
+    LCW<int> gather_map{LCW<int>{}, LCW<int>{}, LCW<int>{}, LCW<int>{}};
+
+    auto results =
+      cudf::lists::detail::segmented_gather(lists_column_view{list}, lists_column_view{gather_map});
+
+    LCW<T> expected{LCW<T>{}, LCW<T>{}, LCW<T>{}, LCW<T>{}};
+    CUDF_TEST_EXPECT_COLUMNS_EQUAL(*results, expected);
+  }
+  // List<List<T>>
+  {
+    LCW<T> list{{{1, 2, 3, 4}, {5}}, {{6, 7}}, {{}, {8, 9, 10}}};
+    LCW<int> gather_map{LCW<int>{}, LCW<int>{}, LCW<int>{}};
+
+    auto results =
+      cudf::lists::detail::segmented_gather(lists_column_view{list}, lists_column_view{gather_map});
+
+    // hack to get column of empty list of list
+    LCW<T> expected_dummy{{{1, 2, 3, 4}, {5}}, LCW<T>{}, LCW<T>{}, LCW<T>{}};
+    auto expected = cudf::split(expected_dummy, {1})[1];
+    CUDF_TEST_EXPECT_COLUMNS_EQUAL(*results, expected);
+  }
+
+  // List<List<List<T>>>
+  {
+    LCW<T> list{{{{1, 2, 3, 4}, {5}}}, {{{6, 7}, {8, 9, 10}}}};
+    LCW<int> gather_map{LCW<int>{}, LCW<int>{}};
+
+    auto results =
+      cudf::lists::detail::segmented_gather(lists_column_view{list}, lists_column_view{gather_map});
+
+    LCW<T> expected_dummy{{{{1, 2, 3, 4}}},  // hack to get column of empty list of list of list
+                          LCW<T>{},
+                          LCW<T>{}};
+    auto expected = cudf::split(expected_dummy, {1})[1];
+    CUDF_TEST_EXPECT_COLUMNS_EQUAL(*results, expected);
+
+    // the result should preserve the full List<List<List<int>>> hierarchy
+    // even though it is empty past the first level
+    cudf::lists_column_view lcv(results->view());
+    EXPECT_EQ(lcv.size(), 2);
+    EXPECT_EQ(lcv.child().type().id(), type_id::LIST);
+    EXPECT_EQ(lcv.child().size(), 0);
+    EXPECT_EQ(lists_column_view(lcv.child()).child().type().id(), type_id::LIST);
+    EXPECT_EQ(lists_column_view(lcv.child()).child().size(), 0);
+    EXPECT_EQ(lists_column_view(lists_column_view(lcv.child()).child()).child().type().id(),
+              type_to_id<T>());
+    EXPECT_EQ(lists_column_view(lists_column_view(lcv.child()).child()).child().size(), 0);
+  }
+}
+
+TYPED_TEST(SegmentedGatherTest, GatherNulls)
+{
+  using T = TypeParam;
+
+  auto valids = cudf::test::make_counting_transform_iterator(
+    0, [](auto i) { return i % 2 == 0 ? true : false; });
+
+  // List<T>
+  LCW<T> list{{{1, 2, 3, 4}, valids}, {5}, {{6, 7}, valids}, {{8, 9, 10}, valids}};
+  LCW<int> gather_map{{0, 1}, LCW<int>{}, {1}, {2, 1, 0}};
+
+  auto results =
+    cudf::lists::detail::segmented_gather(lists_column_view{list}, lists_column_view{gather_map});
+
+  LCW<T> expected{{{1, 2}, valids}, LCW<T>{}, {{7}, valids + 1}, {{10, 9, 8}, valids}};
+  CUDF_TEST_EXPECT_COLUMNS_EQUAL(results->view(), expected);
+}
+
+TYPED_TEST(SegmentedGatherTest, GatherNested)
+{
+  using T = TypeParam;
+
+  // List<List<T>>
+  {
+    LCW<T> list{{{2, 3}, {4, 5}},
+                {{6, 7, 8}, {9, 10, 11}, {12, 13, 14}},
+                {{15, 16}, {17, 18}, {17, 18}, {17, 18}, {-17, -18}}};
+    LCW<int> gather_map{{0, 2, -2}, {1}, {1, 0, -1, 5}};
+
+    auto results =
+      cudf::lists::detail::segmented_gather(lists_column_view{list}, lists_column_view{gather_map});
+
+    LCW<T> expected{
+      {{2, 3}, {2, 3}, {2, 3}}, {{9, 10, 11}}, {{17, 18}, {15, 16}, {-17, -18}, {15, 16}}};
+    CUDF_TEST_EXPECT_COLUMNS_EQUAL(results->view(), expected);
+  }
+
+  // List<List<List<T>>>
+  {
+    LCW<T> list{{{{2, 3}, {4, 5}}, {{6, 7, 8}, {9, 10, 11}, {12, 13, 14}}},
+                {{{15, 16}, {17, 18}, {17, 18}, {17, 18}, {17, 18}}},
+                {{LCW<T>{0}}},
+                {{{10}, {20, 30, 40, 50}, {60, 70, 80}},
+                 {{0, 1, 3}, {5}},
+                 {{11, 12, 13, 14, 15}, {16, 17}, {0}}},
+                {{{10, 20}}, {LCW<T>{30}}, {{40, 50}, {60, 70, 80}}}};
+    LCW<int> gather_map{{1}, LCW<int>{}, {0}, {1}, {0, -1, 1}};
+
+    auto results =
+      cudf::lists::detail::segmented_gather(lists_column_view{list}, lists_column_view{gather_map});
+
+    LCW<T> expected{{{{6, 7, 8}, {9, 10, 11}, {12, 13, 14}}},
+                    LCW<T>{},
+                    {{LCW<T>{0}}},
+                    {{{0, 1, 3}, {5}}},
+                    {{{10, 20}}, {{40, 50}, {60, 70, 80}}, {LCW<T>{30}}}};
+    CUDF_TEST_EXPECT_COLUMNS_EQUAL(results->view(), expected);
+  }
+}
+
+TYPED_TEST(SegmentedGatherTest, GatherOutOfOrder)
+{
+  using T = TypeParam;
+
+  // List<List<T>>
+  {
+    LCW<T> list{{{2, 3}, {4, 5}},
+                {{6, 7, 8}, {9, 10, 11}, {12, 13, 14}},
+                {{15, 16}, {17, 18}, {17, 18}, {17, 18}, {17, 18}}};
+    LCW<int> gather_map{{1, 0}, {1, 2, 0}, {5, 4, 3, 2, 1, 0}};
+
+    auto results =
+      cudf::lists::detail::segmented_gather(lists_column_view{list}, lists_column_view{gather_map});
+
+    LCW<T> expected{{{4, 5}, {2, 3}},
+                    {{9, 10, 11}, {12, 13, 14}, {6, 7, 8}},
+                    {{15, 16}, {17, 18}, {17, 18}, {17, 18}, {17, 18}, {15, 16}}};
+
+    CUDF_TEST_EXPECT_COLUMNS_EQUAL(results->view(), expected);
+  }
+}
+
+TYPED_TEST(SegmentedGatherTest, GatherNegatives)
+{
+  using T = TypeParam;
+
+  // List<List<T>>
+  {
+    LCW<T> list{{{2, 3}, {4, 5}},
+                {{6, 7, 8}, {9, 10, 11}, {12, 13, 14}},
+                {{15, 16}, {17, 18}, {17, 18}, {17, 18}, {17, 18}}};
+    LCW<int> gather_map{{-1, 0}, {-2, -1, 0}, {-5, -4, -3, -2, -1, 0}};
+
+    auto results =
+      cudf::lists::detail::segmented_gather(lists_column_view{list}, lists_column_view{gather_map});
+
+    LCW<T> expected{{{4, 5}, {2, 3}},
+                    {{9, 10, 11}, {12, 13, 14}, {6, 7, 8}},
+                    {{15, 16}, {17, 18}, {17, 18}, {17, 18}, {17, 18}, {15, 16}}};
+
+    CUDF_TEST_EXPECT_COLUMNS_EQUAL(results->view(), expected);
+  }
+}
+
+TYPED_TEST(SegmentedGatherTest, GatherNestedNulls)
+{
+  using T = TypeParam;
+
+  auto valids = cudf::test::make_counting_transform_iterator(
+    0, [](auto i) { return i % 2 == 0 ? true : false; });
+
+  // List<List<T>>
+  {
+    LCW<T> list{{{{2, 3}, valids}, {4, 5}},
+                {{{6, 7, 8}, {9, 10, 11}, {12, 13, 14}}, valids},
+                {{15, 16}, {17, 18}, {17, 18}, {17, 18}, {17, 18}},
+                {{{{25, 26}, valids}, {27, 28}, {{29, 30}, valids}, {31, 32}, {33, 34}}, valids}};
+
+    LCW<int> gather_map{{0, 1}, {0, 2}, LCW<int>{}, {0, 1, 4}};
+
+    auto results =
+      cudf::lists::detail::segmented_gather(lists_column_view{list}, lists_column_view{gather_map});
+
+    auto trues = cudf::test::make_counting_transform_iterator(0, [](auto i) { return true; });
+
+    LCW<T> expected{{{{2, 3}, valids}, {4, 5}},
+                    {{{6, 7, 8}, {12, 13, 14}}, trues},
+                    LCW<T>{},
+                    {{{{25, 26}, valids}, {27, 28}, {33, 34}}, valids}};
+
+    CUDF_TEST_EXPECT_COLUMNS_EQUAL(results->view(), expected);
+  }
+
+  // List<List<List<List<T>>>>
+  {
+    LCW<T> list{{{{{2, 3}, {4, 5}}, {{6, 7, 8}, {9, 10, 11}, {12, 13, 14}}},
+                 {{{15, 16}, {{27, 28}, valids}, {{37, 38}, valids}, {47, 48}, {57, 58}}},
+                 {{LCW<T>{0}}},
+                 {{{10}, {20, 30, 40, 50}, {60, 70, 80}},
+                  {{0, 1, 3}, {5}},
+                  {{11, 12, 13, 14, 15}, {16, 17}, {0}}},
+                 {{{{{10, 20}, valids}}, {LCW<T>{30}}, {{40, 50}, {60, 70, 80}}}, valids}}};
+
+    LCW<int> gather_map{{1, 2, 4}};
+
+    auto results =
+      cudf::lists::detail::segmented_gather(lists_column_view{list}, lists_column_view{gather_map});
+
+    LCW<T> expected{{{{{15, 16}, {{27, 28}, valids}, {{37, 38}, valids}, {47, 48}, {57, 58}}},
+                     {{LCW<T>{0}}},
+                     {{{{{10, 20}, valids}}, {LCW<T>{30}}, {{40, 50}, {60, 70, 80}}}, valids}}};
+
+    CUDF_TEST_EXPECT_COLUMNS_EQUAL(results->view(), expected);
+  }
+}
+
+TYPED_TEST(SegmentedGatherTest, GatherNestedWithEmpties)
+{
+  using T = TypeParam;
+
+  LCW<T> list{{{2, 3}, LCW<T>{}}, {{6, 7, 8}, {9, 10, 11}, {12, 13, 14}}, {LCW<T>{}}};
+  LCW<int> gather_map{LCW<int>{0}, LCW<int>{0}, LCW<int>{0}};
+
+  auto results =
+    cudf::lists::detail::segmented_gather(lists_column_view{list}, lists_column_view{gather_map});
+
+  // skip one null, gather one null.
+  LCW<T> expected{{{2, 3}}, {{6, 7, 8}}, {LCW<T>{}}};
+
+  CUDF_TEST_EXPECT_COLUMNS_EQUAL(results->view(), expected);
+}
+
+TYPED_TEST(SegmentedGatherTest, GatherSliced)
+{
+  using T = TypeParam;
+  {
+    LCW<T> a{
+      {{1, 1, 1}, {2, 2}, {3, 3}},
+      {{4, 4, 4}, {5, 5}, {6, 6}},
+      {{7, 7, 7}, {8, 8}, {9, 9}},
+      {{10, 10, 10}, {11, 11}, {12, 12}},
+      {{20, 20, 20, 20}, {25}},
+      {{30, 30, 30, 30}, {40}},
+      {{50, 50, 50, 50}, {6, 13}},
+      {{70, 70, 70, 70}, {80}},
+    };
+    auto split_a = cudf::split(a, {3});
+
+    auto result0 = cudf::lists::detail::segmented_gather(
+      lists_column_view{split_a[0]}, lists_column_view{LCW<int>{{1, 2}, {0, 2}, {0, 1}}});
+    LCW<T> expected0{
+      {{2, 2}, {3, 3}},
+      {{4, 4, 4}, {6, 6}},
+      {{7, 7, 7}, {8, 8}},
+    };
+    CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected0, result0->view());
+
+    auto result1 = cudf::lists::detail::segmented_gather(
+      lists_column_view{split_a[1]},
+      lists_column_view{LCW<int>{{0, 1}, LCW<int>{}, LCW<int>{}, {0, 1}, LCW<int>{}}});
+    LCW<T> expected1{
+      {{10, 10, 10}, {11, 11}}, LCW<T>{}, LCW<T>{}, {{50, 50, 50, 50}, {6, 13}}, LCW<T>{}};
+    CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected1, result1->view());
+  }
+
+  auto valids = cudf::test::make_counting_transform_iterator(0, [](auto i) { return i % 2 == 0; });
+
+  // List<List<List<T>>>
+  {
+    LCW<T> list{
+      // slice 0
+      {{{2, 3}, {4, 5}}, {{6, 7, 8}, {9, 10, 11}, {12, 13, 14}}},
+
+      {{{15, 16}, {{27, 28}, valids}, {{37, 38}, valids}, {47, 48}, {57, 58}},
+       {{11, 12}, {{42, 43, 44}, valids}, {{77, 78}, valids}}},
+
+      // slice 1
+      {{LCW<T>{0}}},
+      {{{10}, {20, 30, 40, 50}, {60, 70, 80}},
+       {{0, 1, 3}, {5}},
+       {{11, 12, 13, 14, 15}, {16, 17}, {0}}},
+      {{{{1, 6}, {60, 70, 80, 100}}, {{10, 11, 13}, {15}}, {{11, 12, 13, 14, 15}}}, valids},
+
+      // slice 2
+      {{{{{10, 20}, valids}}, {LCW<T>{30}}, {{40, 50}, {60, 70, 80}}}, valids},
+      {{{{10, 20, 30}}, {LCW<T>{30}}, {{{20, 30}, valids}, {62, 72, 82}}}, valids}};
+
+    auto sliced = cudf::slice(list, {0, 1, 2, 5, 5, 7});
+
+    // gather from slice 0
+    {
+      LCW<int> map{{0, 1}};
+      auto result =
+        cudf::lists::detail::segmented_gather(lists_column_view{sliced[0]}, lists_column_view{map});
+      LCW<T> expected{{{{2, 3}, {4, 5}}, {{6, 7, 8}, {9, 10, 11}, {12, 13, 14}}}};
+      CUDF_TEST_EXPECT_COLUMNS_EQUIVALENT(expected, result->view());
+    }
+
+    // gather from slice 1
+    {
+      LCW<int16_t> map{{0}, {1, 2, 0, 1}, {0, 1, 2}};
+      auto result =
+        cudf::lists::detail::segmented_gather(lists_column_view{sliced[1]}, lists_column_view{map});
+      LCW<T> expected{
+        {{LCW<T>{0}}},
+
+        {{{0, 1, 3}, {5}},
+         {{11, 12, 13, 14, 15}, {16, 17}, {0}},
+         {{10}, {20, 30, 40, 50}, {60, 70, 80}},
+         {{0, 1, 3}, {5}}},
+
+        {{{{1, 6}, {60, 70, 80, 100}}, {{10, 11, 13}, {15}}, {{11, 12, 13, 14, 15}}}, valids},
+      };
+      CUDF_TEST_EXPECT_COLUMNS_EQUIVALENT(expected, result->view());
+    }
+
+    // gather from slice 2
+    {
+      LCW<int> map{{1, 0, 0, 1, 1, 0}, {1, 0, 0, 1, 1, 2}};
+      auto result =
+        cudf::lists::detail::segmented_gather(lists_column_view{sliced[2]}, lists_column_view{map});
+      std::vector<bool> expected_valids = {false, true, true, false, false, true};
+
+      LCW<T> expected{{{{LCW<T>{30}},
+                        {{{10, 20}, valids}},
+                        {{{10, 20}, valids}},
+                        {LCW<T>{30}},
+                        {LCW<T>{30}},
+                        {{{10, 20}, valids}}},
+                       expected_valids.begin()},
+                      {{{LCW<T>{30}},
+                        {{10, 20, 30}},
+                        {{10, 20, 30}},
+                        {LCW<T>{30}},
+                        {LCW<T>{30}},
+                        {{{20, 30}, valids}, {62, 72, 82}}},
+                       expected_valids.begin()}};
+      CUDF_TEST_EXPECT_COLUMNS_EQUIVALENT(expected, result->view());
+    }
+  }
+}
+
+using SegmentedGatherTestString = SegmentedGatherTest<cudf::string_view>;
+TEST_F(SegmentedGatherTestString, StringGather)
+{
+  using T = cudf::string_view;
+  // List<T>
+  LCW<T> list{{"a", "b", "c", "d"}, {"1", "22", "333", "4"}, {"x", "y", "z"}};
+  LCW<int8_t> gather_map{{0, 1, 3, 2}, {1, 0, 3, 2}, LCW<int8_t>{}};
+  LCW<T> expected{{"a", "b", "d", "c"}, {"22", "1", "4", "333"}, LCW<T>{}};
+
+  auto result =
+    cudf::lists::detail::segmented_gather(lists_column_view{list}, lists_column_view{gather_map});
+  CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, result->view());
+}
+
+using SegmentedGatherTestFloat = SegmentedGatherTest<float>;
+TEST_F(SegmentedGatherTestFloat, GatherMapSliced)
+{
+  using T = float;
+
+  // List<T>
+  LCW<T> list{{1, 2, 3, 4}, {5}, {6, 7}, {8, 9, 10}, {11, 12}, {13, 14, 15, 16}};
+  LCW<int> gather_map{{3, 2, 1, 0}, {0}, {0, 1}, {0, 2, 1}, {0}, {1}};
+  // gather_map.offset: 0, 4, 5, 7, 10, 11, 12
+  LCW<T> expected{{4, 3, 2, 1}, {5}, {6, 7}, {8, 10, 9}, {11}, {14}};
+
+  auto results =
+    cudf::lists::detail::segmented_gather(lists_column_view{list}, lists_column_view{gather_map});
+
+  CUDF_TEST_EXPECT_COLUMNS_EQUAL(results->view(), expected);
+  auto sliced  = cudf::split(list, {1, 4});
+  auto split_m = cudf::split(gather_map, {1, 4});
+  auto split_e = cudf::split(expected, {1, 4});
+
+  auto result0 = cudf::lists::detail::segmented_gather(lists_column_view{sliced[0]},
+                                                       lists_column_view{split_m[0]});
+  CUDF_TEST_EXPECT_COLUMNS_EQUIVALENT(split_e[0], result0->view());
+  auto result1 = cudf::lists::detail::segmented_gather(lists_column_view{sliced[1]},
+                                                       lists_column_view{split_m[1]});
+  CUDF_TEST_EXPECT_COLUMNS_EQUIVALENT(split_e[1], result1->view());
+  auto result2 = cudf::lists::detail::segmented_gather(lists_column_view{sliced[2]},
+                                                       lists_column_view{split_m[2]});
+  CUDF_TEST_EXPECT_COLUMNS_EQUIVALENT(split_e[2], result2->view());
+}
+
+TEST_F(SegmentedGatherTestFloat, Fails)
+{
+  using T = float;
+  // List<T>
+  LCW<T> list{{1, 2, 3, 4}, {5}, {6, 7}, {8, 9, 10}};
+  LCW<int8_t> size_mismatch_map{{3, 2, 1, 0}, {0}, {0, 1}};
+  cudf::test::fixed_width_column_wrapper<int> nonlist_map0{1, 2, 0, 1};
+  cudf::test::strings_column_wrapper nonlist_map1{"1", "2", "0", "1"};
+  LCW<cudf::string_view> nonlist_map2{{"1", "2", "0", "1"}};
+
+  CUDF_EXPECT_THROW_MESSAGE(
+    cudf::lists::detail::segmented_gather(lists_column_view{list}, lists_column_view{nonlist_map0}),
+    "lists_column_view only supports lists");
+
+  CUDF_EXPECT_THROW_MESSAGE(
+    cudf::lists::detail::segmented_gather(lists_column_view{list}, lists_column_view{nonlist_map1}),
+    "lists_column_view only supports lists");
+
+  CUDF_EXPECT_THROW_MESSAGE(
+    cudf::lists::detail::segmented_gather(lists_column_view{list}, lists_column_view{nonlist_map2}),
+    "Gather map should be list column of index type");
+
+  auto valids = cudf::test::make_counting_transform_iterator(0, [](auto i) { return i % 2 == 0; });
+  LCW<int8_t> nulls_map{{{3, 2, 1, 0}, {0}, {0}, {0, 1}}, valids};
+  CUDF_EXPECT_THROW_MESSAGE(
+    cudf::lists::detail::segmented_gather(lists_column_view{list}, lists_column_view{nulls_map}),
+    "Gather map contains nulls");
+
+  CUDF_EXPECT_THROW_MESSAGE(cudf::lists::detail::segmented_gather(
+                              lists_column_view{list}, lists_column_view{size_mismatch_map}),
+                            "Gather map and list column should be same size");
+}

From 9a6657692c509bf0d01a4bac42846e040fbd9752 Mon Sep 17 00:00:00 2001
From: Alfred Xu <lovedreamf@gmail.com>
Date: Tue, 12 Jan 2021 22:37:25 +0800
Subject: [PATCH 20/20] verify window operations on decimal with java
 tests(#7120)

This pull request is to verify window operations on decimal columns in java package, which is required by spark-rapids on [issue 1333](https://github.com/NVIDIA/spark-rapids/issues/1333).

Authors:
  - sperlingxx <lovedreamf@gmail.com>

Approvers:
  - Robert (Bobby) Evans (@revans2)

URL: https://github.com/rapidsai/cudf/pull/7120
---
 CHANGELOG.md                                  |   1 +
 .../test/java/ai/rapids/cudf/TableTest.java   | 322 +++++++++++++-----
 2 files changed, 236 insertions(+), 87 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 04ffc5868e8..2d218b5a853 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -23,6 +23,7 @@
 - PR #6512 Refactor rolling.cu to reduce compile time
 - PR #6982 Disable some pragma unroll statements in thrust `sort.h`
 - PR #7051 Verify decimal cast in java package
+- PR #7120 Verify window operations on decimal in java package
 
 ## Bug Fixes
 
diff --git a/java/src/test/java/ai/rapids/cudf/TableTest.java b/java/src/test/java/ai/rapids/cudf/TableTest.java
index b8327baec85..34790f74435 100644
--- a/java/src/test/java/ai/rapids/cudf/TableTest.java
+++ b/java/src/test/java/ai/rapids/cudf/TableTest.java
@@ -2249,25 +2249,34 @@ void testGroupByCount() {
 
   @Test
   void testWindowingCount() {
-    try (Table unsorted = new Table.TestBuilder().column( 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) // GBY Key
-                                                 .column( 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3) // GBY Key
-                                                 .column( 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6) // OBY Key
-                                                 .column( 7, 5, 1, 9, 7, 9, 8, 2, 8, 0, 6, 6) // Agg Column
-                                                 .build()) {
+    try (Table unsorted = new Table.TestBuilder()
+        .column(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) // GBY Key
+        .column(1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3) // GBY Key
+        .column(1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6) // OBY Key
+        .column(7, 5, 1, 9, 7, 9, 8, 2, 8, 0, 6, 6) // Agg Column
+        .decimal32Column(-1, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3) // Decimal GBY Key
+        .decimal64Column(1, 1L, 1L, 2L, 2L, 3L, 3L, 4L, 4L, 5L, 5L, 6L, 6L) // Decimal OBY Key
+        .build()) {
       try (Table sorted = unsorted.orderBy(Table.asc(0), Table.asc(1), Table.asc(2));
+           Table decSorted = unsorted.orderBy(Table.asc(0), Table.asc(4), Table.asc(5));
            ColumnVector expectSortedAggColumn = ColumnVector.fromBoxedInts(7, 5, 1, 9, 7, 9, 8, 2, 8, 0, 6, 6)) {
         ColumnVector sortedAggColumn = sorted.getColumn(3);
         assertColumnsAreEqual(expectSortedAggColumn, sortedAggColumn);
+        ColumnVector decSortedAggColumn = decSorted.getColumn(3);
+        assertColumnsAreEqual(expectSortedAggColumn, decSortedAggColumn);
 
         WindowOptions window = WindowOptions.builder()
-                                            .minPeriods(1)
-                                            .window(2, 1)
-                                            .build();
+            .minPeriods(1)
+            .window(2, 1)
+            .build();
 
         try (Table windowAggResults = sorted.groupBy(0, 1)
-                                            .aggregateWindows(WindowAggregate.count(3, window));
+            .aggregateWindows(Aggregation.count().onColumn(3).overWindow(window));
+             Table decWindowAggResults = decSorted.groupBy(0, 4)
+                 .aggregateWindows(Aggregation.count().onColumn(3).overWindow(window));
              ColumnVector expect = ColumnVector.fromBoxedInts(2, 3, 3, 2, 2, 3, 3, 2, 2, 3, 3, 2)) {
           assertColumnsAreEqual(expect, windowAggResults.getColumn(0));
+          assertColumnsAreEqual(expect, decWindowAggResults.getColumn(0));
         }
       }
     }
@@ -2275,25 +2284,37 @@ void testWindowingCount() {
 
   @Test
   void testWindowingMin() {
-    try (Table unsorted = new Table.TestBuilder().column( 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) // GBY Key
-                                                 .column( 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3) // GBY Key
-                                                 .column( 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6) // OBY Key
-                                                 .column( 7, 5, 1, 9, 7, 9, 8, 2, 8, 0, 6, 6) // Agg Column
-                                                 .build()) {
+    try (Table unsorted = new Table.TestBuilder()
+        .column(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) // GBY Key
+        .column(1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3) // GBY Key
+        .column(1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6) // OBY Key
+        .column(7, 5, 1, 9, 7, 9, 8, 2, 8, 0, 6, 6) // Agg Column
+        .decimal32Column(-1, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3) // Decimal GBY Key
+        .decimal64Column(1, 1L, 1L, 2L, 2L, 3L, 3L, 4L, 4L, 5L, 5L, 6L, 6L) // Decimal OBY Key
+        .decimal64Column(2, 7L, 5L, 1L, 9L, 7L, 9L, 8L, 2L, 8L, 0L, 6L, 6L) // Decimal Agg Column
+        .build()) {
       try (Table sorted = unsorted.orderBy(Table.asc(0), Table.asc(1), Table.asc(2));
-           ColumnVector expectSortedAggCol = ColumnVector.fromBoxedInts(7, 5, 1, 9, 7, 9, 8, 2, 8, 0, 6, 6)) {
+           Table decSorted = unsorted.orderBy(Table.asc(0), Table.asc(4), Table.asc(5));
+           ColumnVector expectSortedAggCol = ColumnVector.fromBoxedInts(7, 5, 1, 9, 7, 9, 8, 2, 8, 0, 6, 6);
+           ColumnVector expectDecSortedAggCol = ColumnVector.decimalFromLongs(2, 7, 5, 1, 9, 7, 9, 8, 2, 8, 0, 6, 6)) {
         ColumnVector sortedAggColumn = sorted.getColumn(3);
         assertColumnsAreEqual(expectSortedAggCol, sortedAggColumn);
+        ColumnVector decSortedAggColumn = decSorted.getColumn(6);
+        assertColumnsAreEqual(expectDecSortedAggCol, decSortedAggColumn);
 
         WindowOptions window = WindowOptions.builder()
-                                            .minPeriods(1)
-                                            .window(2, 1)
-                                            .build();
+            .minPeriods(1)
+            .window(2, 1)
+            .build();
 
         try (Table windowAggResults = sorted.groupBy(0, 1)
-                                            .aggregateWindows(WindowAggregate.min(3, window));
-             ColumnVector expect = ColumnVector.fromBoxedInts(5, 1, 1, 1, 7, 7, 2, 2, 0, 0, 0, 6)) {
+            .aggregateWindows(Aggregation.min().onColumn(3).overWindow(window));
+             Table decWindowAggResults = decSorted.groupBy(0, 4)
+                 .aggregateWindows(Aggregation.min().onColumn(6).overWindow(window));
+             ColumnVector expect = ColumnVector.fromBoxedInts(5, 1, 1, 1, 7, 7, 2, 2, 0, 0, 0, 6);
+             ColumnVector decExpect = ColumnVector.decimalFromLongs(2, 5, 1, 1, 1, 7, 7, 2, 2, 0, 0, 0, 6)) {
           assertColumnsAreEqual(expect, windowAggResults.getColumn(0));
+          assertColumnsAreEqual(decExpect, decWindowAggResults.getColumn(0));
         }
       }
     }
@@ -2301,25 +2322,37 @@ void testWindowingMin() {
 
   @Test
   void testWindowingMax() {
-    try (Table unsorted = new Table.TestBuilder().column( 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) // GBY Key
-                                                 .column( 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3) // GBY Key
-                                                 .column( 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6) // OBY Key
-                                                 .column( 7, 5, 1, 9, 7, 9, 8, 2, 8, 0, 6, 6) // Agg Column
-                                                 .build()) {
+    try (Table unsorted = new Table.TestBuilder()
+        .column(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) // GBY Key
+        .column(1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3) // GBY Key
+        .column(1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6) // OBY Key
+        .column(7, 5, 1, 9, 7, 9, 8, 2, 8, 0, 6, 6) // Agg Column
+        .decimal32Column(-1, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3) // Decimal GBY Key
+        .decimal64Column(1, 1L, 1L, 2L, 2L, 3L, 3L, 4L, 4L, 5L, 5L, 6L, 6L) // Decimal OBY Key
+        .decimal64Column(2, 7L, 5L, 1L, 9L, 7L, 9L, 8L, 2L, 8L, 0L, 6L, 6L) // Decimal Agg Column
+        .build()) {
       try (Table sorted = unsorted.orderBy(Table.asc(0), Table.asc(1), Table.asc(2));
-           ColumnVector expectSortedAggCol = ColumnVector.fromBoxedInts(7, 5, 1, 9, 7, 9, 8, 2, 8, 0, 6, 6)) {
+           Table decSorted = unsorted.orderBy(Table.asc(0), Table.asc(4), Table.asc(5));
+           ColumnVector expectSortedAggCol = ColumnVector.fromBoxedInts(7, 5, 1, 9, 7, 9, 8, 2, 8, 0, 6, 6);
+           ColumnVector expectDecSortedAggCol = ColumnVector.decimalFromLongs(2, 7, 5, 1, 9, 7, 9, 8, 2, 8, 0, 6, 6)) {
         ColumnVector sortedAggColumn = sorted.getColumn(3);
         assertColumnsAreEqual(expectSortedAggCol, sortedAggColumn);
+        ColumnVector decSortedAggColumn = decSorted.getColumn(6);
+        assertColumnsAreEqual(expectDecSortedAggCol, decSortedAggColumn);
 
         WindowOptions window = WindowOptions.builder()
-                                            .minPeriods(1)
-                                            .window(2, 1)
-                                            .build();
+            .minPeriods(1)
+            .window(2, 1)
+            .build();
 
         try (Table windowAggResults = sorted.groupBy(0, 1)
-                                            .aggregateWindows(WindowAggregate.max(3, window));
-             ColumnVector expect = ColumnVector.fromBoxedInts(7, 7, 9, 9, 9, 9, 9, 8, 8, 8, 6, 6)) {
+            .aggregateWindows(Aggregation.max().onColumn(3).overWindow(window));
+             Table decWindowAggResults = decSorted.groupBy(0, 4)
+                 .aggregateWindows(Aggregation.max().onColumn(6).overWindow(window));
+             ColumnVector expect = ColumnVector.fromBoxedInts(7, 7, 9, 9, 9, 9, 9, 8, 8, 8, 6, 6);
+             ColumnVector decExpect = ColumnVector.decimalFromLongs(2, 7, 7, 9, 9, 9, 9, 9, 8, 8, 8, 6, 6)) {
           assertColumnsAreEqual(expect, windowAggResults.getColumn(0));
+          assertColumnsAreEqual(decExpect, decWindowAggResults.getColumn(0));
         }
       }
     }
@@ -2327,26 +2360,25 @@ void testWindowingMax() {
 
   @Test
   void testWindowingSum() {
-    try (Table unsorted = new Table.TestBuilder().column( 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) // GBY Key
-                                                 .column( 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3) // GBY Key
-                                                 .column( 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6) // OBY Key
-                                                 .column( 7, 5, 1, 9, 7, 9, 8, 2, 8, 0, 6, 6) // Agg Column
-                                                 .build();
-         ColumnVector expectSortedAggColumn = ColumnVector.fromBoxedInts(7, 5, 1, 9, 7, 9, 8, 2, 8, 0, 6, 6)) {
-
-      try (Table sorted = unsorted.orderBy(Table.asc(0), Table.asc(1), Table.asc(2))) {
+    try (Table unsorted = new Table.TestBuilder()
+        .column(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) // GBY Key
+        .column(1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3) // GBY Key
+        .column(1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6) // OBY Key
+        .column(7, 5, 1, 9, 7, 9, 8, 2, 8, 0, 6, 6) // Agg Column
+        .build()) {
+      try (Table sorted = unsorted.orderBy(Table.asc(0), Table.asc(1), Table.asc(2));
+           ColumnVector expectSortedAggColumn = ColumnVector.fromBoxedInts(7, 5, 1, 9, 7, 9, 8, 2, 8, 0, 6, 6)) {
         ColumnVector sortedAggColumn = sorted.getColumn(3);
         assertColumnsAreEqual(expectSortedAggColumn, sortedAggColumn);
 
         WindowOptions window = WindowOptions.builder()
-                                            .minPeriods(1)
-                                            .window(2, 1)
-                                            .build();
+            .minPeriods(1)
+            .window(2, 1)
+            .build();
 
         try (Table windowAggResults = sorted.groupBy(0, 1)
-                                            .aggregateWindows(WindowAggregate.sum(3, window));
-             ColumnVector expectAggResult = ColumnVector.fromBoxedLongs(12L, 13L, 15L, 10L, 16L, 24L, 19L, 10L, 8L, 14L, 12L, 12L);
-        ) {
+            .aggregateWindows(Aggregation.sum().onColumn(3).overWindow(window));
+             ColumnVector expectAggResult = ColumnVector.fromBoxedLongs(12L, 13L, 15L, 10L, 16L, 24L, 19L, 10L, 8L, 14L, 12L, 12L)) {
           assertColumnsAreEqual(expectAggResult, windowAggResults.getColumn(0));
         }
       }
@@ -2355,38 +2387,69 @@ void testWindowingSum() {
 
   @Test
   void testWindowingRowNumber() {
-    try (Table unsorted = new Table.TestBuilder().column( 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) // GBY Key
-                                                 .column( 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3) // GBY Key
-                                                 .column( 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6) // OBY Key
-                                                 .column( 7, 5, 1, 9, 7, 9, 8, 2, 8, 0, 6, 6) // Agg Column
-                                                 .build();
-         ColumnVector expectSortedAggColumn = ColumnVector.fromBoxedInts(7, 5, 1, 9, 7, 9, 8, 2, 8, 0, 6, 6)) {
-
-      try (Table sorted = unsorted.orderBy(Table.asc(0), Table.asc(1), Table.asc(2))) {
+    try (Table unsorted = new Table.TestBuilder()
+        .column(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) // GBY Key
+        .column(1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3) // GBY Key
+        .column(1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6) // OBY Key
+        .column(7, 5, 1, 9, 7, 9, 8, 2, 8, 0, 6, 6) // Agg Column
+        .decimal32Column(-1, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3) // Decimal GBY Key
+        .decimal64Column(1, 1L, 1L, 2L, 2L, 3L, 3L, 4L, 4L, 5L, 5L, 6L, 6L) // Decimal OBY Key
+        .decimal64Column(2, 7L, 5L, 1L, 9L, 7L, 9L, 8L, 2L, 8L, 0L, 6L, 6L) // Decimal Agg Column
+        .build()) {
+      try (Table sorted = unsorted.orderBy(Table.asc(0), Table.asc(1), Table.asc(2));
+           Table decSorted = unsorted.orderBy(Table.asc(0), Table.asc(4), Table.asc(5));
+           ColumnVector expectSortedAggColumn = ColumnVector.fromBoxedInts(7, 5, 1, 9, 7, 9, 8, 2, 8, 0, 6, 6);
+           ColumnVector expectDecSortedAggColumn = ColumnVector.decimalFromLongs(2, 7, 5, 1, 9, 7, 9, 8, 2, 8, 0, 6, 6)) {
         ColumnVector sortedAggColumn = sorted.getColumn(3);
         assertColumnsAreEqual(expectSortedAggColumn, sortedAggColumn);
+        ColumnVector decSortedAggColumn = decSorted.getColumn(6);
+        assertColumnsAreEqual(expectDecSortedAggColumn, decSortedAggColumn);
 
         WindowOptions.Builder windowBuilder = WindowOptions.builder().minPeriods(1);
 
         try (Table windowAggResults = sorted.groupBy(0, 1)
-                                            .aggregateWindows(WindowAggregate.row_number(3, windowBuilder.window(2,1).build()));
-             ColumnVector expectAggResult = ColumnVector.fromBoxedInts(1,2,2,2, 1,2,2,2, 1,2,2,2);
-        ) {
+            .aggregateWindows(Aggregation
+                .rowNumber()
+                .onColumn(3)
+                .overWindow(windowBuilder.window(2, 1).build()));
+             Table decWindowAggResults = decSorted.groupBy(0, 4)
+                 .aggregateWindows(Aggregation
+                     .rowNumber()
+                     .onColumn(6)
+                     .overWindow(windowBuilder.window(2, 1).build()));
+             ColumnVector expectAggResult = ColumnVector.fromBoxedInts(1, 2, 2, 2, 1, 2, 2, 2, 1, 2, 2, 2)) {
           assertColumnsAreEqual(expectAggResult, windowAggResults.getColumn(0));
+          assertColumnsAreEqual(expectAggResult, decWindowAggResults.getColumn(0));
         }
 
         try (Table windowAggResults = sorted.groupBy(0, 1)
-                                            .aggregateWindows(WindowAggregate.row_number(3, windowBuilder.window(3,2).build()));
-             ColumnVector expectAggResult = ColumnVector.fromBoxedInts(1,2,3,3, 1,2,3,3, 1,2,3,3);
-        ) {
+            .aggregateWindows(Aggregation
+                .rowNumber()
+                .onColumn(3)
+                .overWindow(windowBuilder.window(3, 2).build()));
+             Table decWindowAggResults = decSorted.groupBy(0, 4)
+                 .aggregateWindows(Aggregation
+                     .rowNumber()
+                     .onColumn(6)
+                     .overWindow(windowBuilder.window(3, 2).build()));
+             ColumnVector expectAggResult = ColumnVector.fromBoxedInts(1, 2, 3, 3, 1, 2, 3, 3, 1, 2, 3, 3)) {
           assertColumnsAreEqual(expectAggResult, windowAggResults.getColumn(0));
+          assertColumnsAreEqual(expectAggResult, decWindowAggResults.getColumn(0));
         }
 
         try (Table windowAggResults = sorted.groupBy(0, 1)
-                                            .aggregateWindows(WindowAggregate.row_number(3, windowBuilder.window(4,3).build()));
-             ColumnVector expectAggResult = ColumnVector.fromBoxedInts(1,2,3,4, 1,2,3,4, 1,2,3,4);
-        ) {
+            .aggregateWindows(Aggregation
+                .rowNumber()
+                .onColumn(3)
+                .overWindow(windowBuilder.window(4, 3).build()));
+             Table decWindowAggResults = decSorted.groupBy(0, 4)
+                 .aggregateWindows(Aggregation
+                     .rowNumber()
+                     .onColumn(6)
+                     .overWindow(windowBuilder.window(4, 3).build()));
+             ColumnVector expectAggResult = ColumnVector.fromBoxedInts(1, 2, 3, 4, 1, 2, 3, 4, 1, 2, 3, 4)) {
           assertColumnsAreEqual(expectAggResult, windowAggResults.getColumn(0));
+          assertColumnsAreEqual(expectAggResult, decWindowAggResults.getColumn(0));
         }
        }
     }
@@ -2395,16 +2458,23 @@ void testWindowingRowNumber() {
   @Test
   void testWindowingLead() {
     try (Table unsorted = new Table.TestBuilder()
-        .column( 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) // GBY Key
-        .column( 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3) // GBY Key
-        .column( 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6) // OBY Key
-        .column( 7, 5, 1, 9, 7, 9, 8, 2, 8, 0, 6, 6) // Agg Column
-        .build();
-         ColumnVector expectSortedAggColumn = ColumnVector.fromBoxedInts(7, 5, 1, 9, 7, 9, 8, 2, 8, 0, 6, 6)) {
+        .column(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) // GBY Key
+        .column(1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3) // GBY Key
+        .column(1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6) // OBY Key
+        .column(7, 5, 1, 9, 7, 9, 8, 2, 8, 0, 6, 6) // Agg Column
+        .decimal32Column(-1, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3) // Decimal GBY Key
+        .decimal64Column(1, 1L, 1L, 2L, 2L, 3L, 3L, 4L, 4L, 5L, 5L, 6L, 6L) // Decimal OBY Key
+        .decimal64Column(-2, 7L, 5L, 1L, 9L, 7L, 9L, 8L, 2L, 8L, 0L, 6L, 6L) // Decimal Agg Column
+        .build()) {
 
-      try (Table sorted = unsorted.orderBy(Table.asc(0), Table.asc(1), Table.asc(2))) {
+      try (Table sorted = unsorted.orderBy(Table.asc(0), Table.asc(1), Table.asc(2));
+           Table decSorted = unsorted.orderBy(Table.asc(0), Table.asc(4), Table.asc(5));
+           ColumnVector expectSortedAggColumn = ColumnVector.fromBoxedInts(7, 5, 1, 9, 7, 9, 8, 2, 8, 0, 6, 6);
+           ColumnVector expectDecSortedAggColumn = ColumnVector.decimalFromLongs(-2, 7, 5, 1, 9, 7, 9, 8, 2, 8, 0, 6, 6)) {
         ColumnVector sortedAggColumn = sorted.getColumn(3);
         assertColumnsAreEqual(expectSortedAggColumn, sortedAggColumn);
+        ColumnVector decSortedAggColumn = decSorted.getColumn(6);
+        assertColumnsAreEqual(expectDecSortedAggColumn, decSortedAggColumn);
 
         WindowOptions.Builder windowBuilder = WindowOptions.builder().minPeriods(1);
 
@@ -2412,9 +2482,16 @@ void testWindowingLead() {
             .aggregateWindows(Aggregation
                 .lead(0)
                 .onColumn(3)
-                .overWindow(windowBuilder.window(2,1).build()));
-             ColumnVector expectAggResult = ColumnVector.fromBoxedInts(7, 5, 1, 9, 7, 9, 8, 2, 8, 0, 6, 6)) {
+                .overWindow(windowBuilder.window(2, 1).build()));
+             Table decWindowAggResults = decSorted.groupBy(0, 4)
+                 .aggregateWindows(Aggregation
+                     .lead(0)
+                     .onColumn(6)
+                     .overWindow(windowBuilder.window(2, 1).build()));
+             ColumnVector expectAggResult = ColumnVector.fromBoxedInts(7, 5, 1, 9, 7, 9, 8, 2, 8, 0, 6, 6);
+             ColumnVector decExpectAggResult = ColumnVector.decimalFromLongs(-2, 7, 5, 1, 9, 7, 9, 8, 2, 8, 0, 6, 6)) {
           assertColumnsAreEqual(expectAggResult, windowAggResults.getColumn(0));
+          assertColumnsAreEqual(decExpectAggResult, decWindowAggResults.getColumn(0));
         }
 
         try (Table windowAggResults = sorted.groupBy(0, 1)
@@ -2422,18 +2499,33 @@ void testWindowingLead() {
                 .lead(1)
                 .onColumn(3)
                 .overWindow(windowBuilder.window(0,1).build()));
-             ColumnVector expectAggResult = ColumnVector.fromBoxedInts(5, 1, 9, null, 9, 8, 2, null, 0, 6, 6, null)) {
+             Table decWindowAggResults = sorted.groupBy(0, 4)
+                 .aggregateWindows(Aggregation
+                     .lead(1)
+                     .onColumn(6)
+                     .overWindow(windowBuilder.window(0,1).build()));
+             ColumnVector expectAggResult = ColumnVector.fromBoxedInts(5, 1, 9, null, 9, 8, 2, null, 0, 6, 6, null);
+             ColumnVector decExpectAggResult = decimalFromBoxedInts(true, -2, 5, 1, 9, null, 9, 8, 2, null, 0, 6, 6, null)) {
           assertColumnsAreEqual(expectAggResult, windowAggResults.getColumn(0));
+          assertColumnsAreEqual(decExpectAggResult, decWindowAggResults.getColumn(0));
         }
 
         try (ColumnVector defaultOutput = ColumnVector.fromBoxedInts(0, -1, -2, -3, -4, -5, -6, -7, -8, -9, -10, -11);
+             ColumnVector decDefaultOutput = ColumnVector.decimalFromLongs(-2, 0, -1, -2, -3, -4, -5, -6, -7, -8, -9, -10, -11);
              Table windowAggResults = sorted.groupBy(0, 1)
                  .aggregateWindows(Aggregation
                      .lead(1, defaultOutput)
                      .onColumn(3)
                      .overWindow(windowBuilder.window(0,1).build()));
-             ColumnVector expectAggResult = ColumnVector.fromBoxedInts(5, 1, 9, -3, 9, 8, 2, -7, 0, 6, 6, -11)) {
+             Table decWindowAggResults = sorted.groupBy(0, 4)
+                 .aggregateWindows(Aggregation
+                     .lead(1, decDefaultOutput)
+                     .onColumn(6)
+                     .overWindow(windowBuilder.window(0,1).build()));
+             ColumnVector expectAggResult = ColumnVector.fromBoxedInts(5, 1, 9, -3, 9, 8, 2, -7, 0, 6, 6, -11);
+             ColumnVector decExpectAggResult = ColumnVector.decimalFromLongs(-2, 5, 1, 9, -3, 9, 8, 2, -7, 0, 6, 6, -11)) {
           assertColumnsAreEqual(expectAggResult, windowAggResults.getColumn(0));
+          assertColumnsAreEqual(decExpectAggResult, decWindowAggResults.getColumn(0));
         }
 
         // Outside bounds
@@ -2442,8 +2534,15 @@ void testWindowingLead() {
                 .lead(3)
                 .onColumn(3)
                 .overWindow(windowBuilder.window(0,1).build()));
-             ColumnVector expectAggResult = ColumnVector.fromBoxedInts(null, null, null, null, null, null, null, null, null, null, null, null)) {
+             Table decWindowAggResults = sorted.groupBy(0, 4)
+                 .aggregateWindows(Aggregation
+                     .lead(3)
+                     .onColumn(6)
+                     .overWindow(windowBuilder.window(0,1).build()));
+             ColumnVector expectAggResult = ColumnVector.fromBoxedInts(null, null, null, null, null, null, null, null, null, null, null, null);
+             ColumnVector decExpectAggResult = decimalFromBoxedInts(true, -2, null, null, null, null, null, null, null, null, null, null, null, null)) {
           assertColumnsAreEqual(expectAggResult, windowAggResults.getColumn(0));
+          assertColumnsAreEqual(decExpectAggResult, decWindowAggResults.getColumn(0));
         }
       }
     }
@@ -2452,16 +2551,23 @@ void testWindowingLead() {
   @Test
   void testWindowingLag() {
     try (Table unsorted = new Table.TestBuilder()
-        .column( 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) // GBY Key
-        .column( 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3) // GBY Key
-        .column( 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6) // OBY Key
-        .column( 7, 5, 1, 9, 7, 9, 8, 2, 8, 0, 6, 6) // Agg Column
-        .build();
-         ColumnVector expectSortedAggColumn = ColumnVector.fromBoxedInts(7, 5, 1, 9, 7, 9, 8, 2, 8, 0, 6, 6)) {
+        .column(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) // GBY Key
+        .column(1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3) // GBY Key
+        .column(1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6) // OBY Key
+        .column(7, 5, 1, 9, 7, 9, 8, 2, 8, 0, 6, 6) // Agg Column
+        .decimal32Column(-1, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3) // Decimal GBY Key
+        .decimal64Column(1, 1L, 1L, 2L, 2L, 3L, 3L, 4L, 4L, 5L, 5L, 6L, 6L) // Decimal OBY Key
+        .decimal64Column(-2, 7L, 5L, 1L, 9L, 7L, 9L, 8L, 2L, 8L, 0L, 6L, 6L) // Decimal Agg Column
+        .build()) {
 
-      try (Table sorted = unsorted.orderBy(Table.asc(0), Table.asc(1), Table.asc(2))) {
+      try (Table sorted = unsorted.orderBy(Table.asc(0), Table.asc(1), Table.asc(2));
+           Table decSorted = unsorted.orderBy(Table.asc(0), Table.asc(4), Table.asc(5));
+           ColumnVector expectSortedAggColumn = ColumnVector.fromBoxedInts(7, 5, 1, 9, 7, 9, 8, 2, 8, 0, 6, 6);
+           ColumnVector decExpectSortedAggColumn = ColumnVector.decimalFromLongs(-2, 7, 5, 1, 9, 7, 9, 8, 2, 8, 0, 6, 6)) {
         ColumnVector sortedAggColumn = sorted.getColumn(3);
         assertColumnsAreEqual(expectSortedAggColumn, sortedAggColumn);
+        ColumnVector decSortedAggColumn = decSorted.getColumn(6);
+        assertColumnsAreEqual(decExpectSortedAggColumn, decSortedAggColumn);
 
         WindowOptions.Builder windowBuilder = WindowOptions.builder().minPeriods(1);
 
@@ -2470,8 +2576,15 @@ void testWindowingLag() {
                 .lag(0)
                 .onColumn(3)
                 .overWindow(windowBuilder.window(2,1).build()));
-             ColumnVector expectAggResult = ColumnVector.fromBoxedInts(7, 5, 1, 9, 7, 9, 8, 2, 8, 0, 6, 6)) {
+             Table decWindowAggResults = sorted.groupBy(0, 4)
+                 .aggregateWindows(Aggregation
+                     .lag(0)
+                     .onColumn(6)
+                     .overWindow(windowBuilder.window(2,1).build()));
+             ColumnVector expectAggResult = ColumnVector.fromBoxedInts(7, 5, 1, 9, 7, 9, 8, 2, 8, 0, 6, 6);
+             ColumnVector decExpectAggResult = ColumnVector.decimalFromLongs(-2, 7, 5, 1, 9, 7, 9, 8, 2, 8, 0, 6, 6)) {
           assertColumnsAreEqual(expectAggResult, windowAggResults.getColumn(0));
+          assertColumnsAreEqual(decExpectAggResult, decWindowAggResults.getColumn(0));
         }
 
         try (Table windowAggResults = sorted.groupBy(0, 1)
@@ -2479,18 +2592,33 @@ void testWindowingLag() {
                 .lag(1)
                 .onColumn(3)
                 .overWindow(windowBuilder.window(2,0).build()));
-             ColumnVector expectAggResult = ColumnVector.fromBoxedInts(null, 7, 5, 1, null, 7, 9, 8, null, 8, 0, 6)) {
+             Table decWindowAggResults = sorted.groupBy(0, 4)
+                 .aggregateWindows(Aggregation
+                     .lag(1)
+                     .onColumn(6)
+                     .overWindow(windowBuilder.window(2,0).build()));
+             ColumnVector expectAggResult = ColumnVector.fromBoxedInts(null, 7, 5, 1, null, 7, 9, 8, null, 8, 0, 6);
+             ColumnVector decExpectAggResult = decimalFromBoxedInts(true, -2, null, 7, 5, 1, null, 7, 9, 8, null, 8, 0, 6)) {
           assertColumnsAreEqual(expectAggResult, windowAggResults.getColumn(0));
+          assertColumnsAreEqual(decExpectAggResult, decWindowAggResults.getColumn(0));
         }
 
         try (ColumnVector defaultOutput = ColumnVector.fromBoxedInts(0, -1, -2, -3, -4, -5, -6, -7, -8, -9, -10, -11);
+             ColumnVector decDefaultOutput = ColumnVector.decimalFromLongs(-2, 0, -1, -2, -3, -4, -5, -6, -7, -8, -9, -10, -11);
              Table windowAggResults = sorted.groupBy(0, 1)
                  .aggregateWindows(Aggregation
                      .lag(1, defaultOutput)
                      .onColumn(3)
-                     .overWindow(windowBuilder.window(2,0).build()));
-             ColumnVector expectAggResult = ColumnVector.fromBoxedInts(0, 7, 5, 1, -4, 7, 9, 8, -8, 8, 0, 6)) {
+                     .overWindow(windowBuilder.window(2, 0).build()));
+             Table decWindowAggResults = sorted.groupBy(0, 4)
+                 .aggregateWindows(Aggregation
+                     .lag(1, decDefaultOutput)
+                     .onColumn(6)
+                     .overWindow(windowBuilder.window(2, 0).build()));
+             ColumnVector expectAggResult = ColumnVector.fromBoxedInts(0, 7, 5, 1, -4, 7, 9, 8, -8, 8, 0, 6);
+             ColumnVector decExpectAggResult = ColumnVector.decimalFromLongs(-2, 0, 7, 5, 1, -4, 7, 9, 8, -8, 8, 0, 6)) {
           assertColumnsAreEqual(expectAggResult, windowAggResults.getColumn(0));
+          assertColumnsAreEqual(decExpectAggResult, decWindowAggResults.getColumn(0));
         }
 
         // Outside bounds
@@ -2498,9 +2626,16 @@ void testWindowingLag() {
             .aggregateWindows(Aggregation
                 .lag(3)
                 .onColumn(3)
-                .overWindow(windowBuilder.window(1,0).build()));
-             ColumnVector expectAggResult = ColumnVector.fromBoxedInts(null, null, null, null, null, null, null, null, null, null, null, null)) {
+                .overWindow(windowBuilder.window(1, 0).build()));
+             Table decWindowAggResults = sorted.groupBy(0, 4)
+                 .aggregateWindows(Aggregation
+                     .lag(3)
+                     .onColumn(6)
+                     .overWindow(windowBuilder.window(1, 0).build()));
+             ColumnVector expectAggResult = ColumnVector.fromBoxedInts(null, null, null, null, null, null, null, null, null, null, null, null);
+             ColumnVector decExpectAggResult = decimalFromBoxedInts(true, -2, null, null, null, null, null, null, null, null, null, null, null, null);) {
           assertColumnsAreEqual(expectAggResult, windowAggResults.getColumn(0));
+          assertColumnsAreEqual(decExpectAggResult, decWindowAggResults.getColumn(0));
         }
       }
     }
@@ -4188,6 +4323,19 @@ private String[] strings(String... values) {
     return values;
   }
 
+  private static ColumnVector decimalFromBoxedInts(boolean isDec64, int scale, Integer... values) {
+    BigDecimal[] decimals = new BigDecimal[values.length];
+    for (int i = 0; i < values.length; i++) {
+      if (values[i] == null) {
+        decimals[i] = null;
+      } else {
+        decimals[i] = BigDecimal.valueOf(values[i], -scale);
+      }
+    }
+    DType type = isDec64 ? DType.create(DType.DTypeEnum.DECIMAL64, scale) : DType.create(DType.DTypeEnum.DECIMAL32, scale);
+    return ColumnVector.build(type, decimals.length, (b) -> b.appendBoxed(decimals));
+  }
+
   private Table buildTestTable() {
     StructType mapStructType = new StructType(true,
         new BasicType(false, DType.STRING),