Encode DELTA_BYTE_ARRAY in Parquet writer

cpp/include/cudf/io/parquet.hpp

@@ -563,6 +563,9 @@ class parquet_writer_options {
   std::shared_ptr<writer_compression_statistics> _compression_stats;
   // write V2 page headers?
   bool _v2_page_headers = false;
+  // Use DELTA_BYTE_ARRAY when dictionary encoding is not available rather than the default
+  // DELTA_LENGTH_BYTE_ARRAY
+  bool _prefer_delta_byte_array = false;
 
   /**
    * @brief Constructor from sink and table.
@@ -761,6 +764,13 @@ class parquet_writer_options {
    */
   [[nodiscard]] auto is_enabled_write_v2_headers() const { return _v2_page_headers; }
 
+  /**
+   * @brief Returns `true` if DELTA_BYTE_ARRAY is the preferred string encoding.
+   *
+   * @return `true` if DELTA_BYTE_ARRAY is the preferred string encoding.
+   */
+  [[nodiscard]] auto is_enabled_prefer_dba() const { return _prefer_delta_byte_array; }
+
   /**
    * @brief Sets partitions.
    *
@@ -892,6 +902,13 @@ class parquet_writer_options {
    * @param val Boolean value to enable/disable writing of V2 page headers.
    */
   void enable_write_v2_headers(bool val) { _v2_page_headers = val; }
+
+  /**
+   * @brief Sets preference for delta encoding.
+   *
+   * @param val Boolean value to enable/disable use of DELTA_BYTE_ARRAY encoding.
+   */
+  void enable_prefer_dba(bool val) { _prefer_delta_byte_array = val; }
 };
 
 /**
@@ -1143,6 +1160,20 @@ class parquet_writer_options_builder {
    */
   parquet_writer_options_builder& write_v2_headers(bool enabled);
 
+  /**
+   * @brief Set to true if DELTA_BYTE_ARRAY encoding should be used.
+   *
+   * The default encoding for all columns is dictionary encoding. When dictionary encoding
+   * cannot be used (it was disabled, or the dictionary is too large), the parquet writer
+   * will usually fall back to PLAIN encoding. If V2 headers are enabled, however, the
+   * choice for fall back is DELTA_LENGTH_BYTE_ARRAY. Setting this to `true` will use
+   * DELTA_BYTE_ARRAY encoding instead. This will apply to all string columns.
+   *
+   * @param enabled Boolean value to enable/disable use of DELTA_BYTE_ARRAY encoding.
+   * @return this for chaining
+   */
+  parquet_writer_options_builder& prefer_dba(bool enabled);
+
   /**
    * @brief move parquet_writer_options member once it's built.
    */
@@ -1230,6 +1261,9 @@ class chunked_parquet_writer_options {
   std::shared_ptr<writer_compression_statistics> _compression_stats;
   // write V2 page headers?
   bool _v2_page_headers = false;
+  // Use DELTA_BYTE_ARRAY when dictionary encoding is not available rather than the default
+  // DELTA_LENGTH_BYTE_ARRAY
+  bool _prefer_delta_byte_array = false;
 
   /**
    * @brief Constructor from sink.
@@ -1384,6 +1418,13 @@ class chunked_parquet_writer_options {
    */
   [[nodiscard]] auto is_enabled_write_v2_headers() const { return _v2_page_headers; }
 
+  /**
+   * @brief Returns `true` if DELTA_BYTE_ARRAY is the preferred string encoding.
+   *
+   * @return `true` if DELTA_BYTE_ARRAY is the preferred string encoding.
+   */
+  [[nodiscard]] auto is_enabled_prefer_dba() const { return _prefer_delta_byte_array; }
+
   /**
    * @brief Sets metadata.
    *
@@ -1501,6 +1542,13 @@ class chunked_parquet_writer_options {
    */
   void enable_write_v2_headers(bool val) { _v2_page_headers = val; }
 
+  /**
+   * @brief Sets preference for delta encoding.
+   *
+   * @param val Boolean value to enable/disable use of DELTA_BYTE_ARRAY encoding.
+   */
+  void enable_prefer_dba(bool val) { _prefer_delta_byte_array = val; }
+
   /**
    * @brief creates builder to build chunked_parquet_writer_options.
    *
@@ -1612,6 +1660,20 @@ class chunked_parquet_writer_options_builder {
    */
   chunked_parquet_writer_options_builder& write_v2_headers(bool enabled);
 
+  /**
+   * @brief Set to true if DELTA_BYTE_ARRAY encoding should be used.
+   *
+   * The default encoding for all columns is dictionary encoding. When dictionary encoding
+   * cannot be used (it was disabled, the dictionary is too large), the parquet writer
+   * will usually fall back to PLAIN encoding. If V2 headers are enabled, however, the
+   * choice for fall back is DELTA_LENGTH_BYTE_ARRAY. Setting this to `true` will use
+   * DELTA_BYTE_ARRAY encoding instead. This will apply to all string columns.
+   *
+   * @param enabled Boolean value to enable/disable use of DELTA_BYTE_ARRAY encoding.
+   * @return this for chaining
+   */
+  chunked_parquet_writer_options_builder& prefer_dba(bool enabled);
+
   /**
    * @brief Sets the maximum row group size, in bytes.
    *

cpp/src/io/functions.cpp

@@ -809,6 +809,12 @@ parquet_writer_options_builder& parquet_writer_options_builder::write_v2_headers
   return *this;
 }
 
+parquet_writer_options_builder& parquet_writer_options_builder::prefer_dba(bool enabled)
+{
+  options.enable_prefer_dba(enabled);
+  return *this;
+}
+
 void chunked_parquet_writer_options::set_key_value_metadata(
   std::vector<std::map<std::string, std::string>> metadata)
 {
@@ -897,6 +903,13 @@ chunked_parquet_writer_options_builder& chunked_parquet_writer_options_builder::
   return *this;
 }
 
+chunked_parquet_writer_options_builder& chunked_parquet_writer_options_builder::prefer_dba(
+  bool enabled)
+{
+  options.enable_prefer_dba(enabled);
+  return *this;
+}
+
 chunked_parquet_writer_options_builder&
 chunked_parquet_writer_options_builder::max_page_fragment_size(size_type val)
 {

cpp/src/io/parquet/delta_enc.cuh

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2023, NVIDIA CORPORATION.
+ * Copyright (c) 2023-2024, NVIDIA CORPORATION.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -201,7 +201,7 @@ class delta_binary_packer {
     if (is_valid) { _buffer[delta::rolling_idx(pos + _current_idx + _values_in_buffer)] = value; }
     __syncthreads();
 
-    if (threadIdx.x == 0) {
+    if (num_valid > 0 && threadIdx.x == 0) {
       _values_in_buffer += num_valid;
       // if first pass write header
       if (_current_idx == 0) {