ARROW-17263 [C++]: Utility functions for working with RLE

cpp/src/arrow/CMakeLists.txt

@@ -140,6 +140,7 @@ set(ARROW_SRCS
     array/array_binary.cc
     array/array_decimal.cc
     array/array_dict.cc
+    array/array_encoded.cc
     array/array_nested.cc
     array/array_primitive.cc
     array/builder_adaptive.cc
@@ -217,6 +218,7 @@ set(ARROW_SRCS
     util/key_value_metadata.cc
     util/memory.cc
     util/mutex.cc
+    util/rle_util.cc
     util/string.cc
     util/string_builder.cc
     util/task_group.cc
@@ -715,6 +717,7 @@ add_arrow_test(array_test
                array/array_test.cc
                array/array_binary_test.cc
                array/array_dict_test.cc
+               array/array_encoded_test.cc
                array/array_list_test.cc
                array/array_struct_test.cc
                array/array_union_test.cc

cpp/src/arrow/array.h

@@ -34,10 +34,15 @@
 /// @{
 /// @}
 
+/// \defgroup encoded-arrays Concrete classes for encoded arrays
+/// @{
+/// @}
+
 #include "arrow/array/array_base.h"       // IWYU pragma: keep
 #include "arrow/array/array_binary.h"     // IWYU pragma: keep
 #include "arrow/array/array_decimal.h"    // IWYU pragma: keep
 #include "arrow/array/array_dict.h"       // IWYU pragma: keep
+#include "arrow/array/array_encoded.h"    // IWYU pragma: keep
 #include "arrow/array/array_nested.h"     // IWYU pragma: keep
 #include "arrow/array/array_primitive.h"  // IWYU pragma: keep
 #include "arrow/array/data.h"             // IWYU pragma: keep

cpp/src/arrow/array/array_base.cc

@@ -143,6 +143,10 @@ struct ScalarFromArraySlotImpl {
     return Status::OK();
   }
 
+  Status Visit(const RunLengthEncodedArray& a) {
+    return Status::NotImplemented("Creating scalar from encoded array");
+  }
+
   Status Visit(const ExtensionArray& a) {
     ARROW_ASSIGN_OR_RAISE(auto storage, a.storage()->GetScalar(index_));
     out_ = std::make_shared<ExtensionScalar>(std::move(storage), a.type());

cpp/src/arrow/array/array_encoded.cc

@@ -0,0 +1,77 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "arrow/array/array_encoded.h"
+#include "arrow/array/util.h"
+#include "arrow/util/logging.h"
+#include "arrow/util/rle_util.h"
+
+namespace arrow {
+
+// ----------------------------------------------------------------------
+// RunLengthEncodedArray
+
+RunLengthEncodedArray::RunLengthEncodedArray(const std::shared_ptr<ArrayData>& data) {
+  ARROW_CHECK_EQ(data->type->id(), Type::RUN_LENGTH_ENCODED);
+  SetData(data);
+}
+
+RunLengthEncodedArray::RunLengthEncodedArray(const std::shared_ptr<DataType>& type,
+                                             int64_t length,
+                                             const std::shared_ptr<Array>& run_ends_array,
+                                             const std::shared_ptr<Array>& values_array,
+                                             int64_t offset) {
+  ARROW_CHECK_EQ(type->id(), Type::RUN_LENGTH_ENCODED);
+  SetData(ArrayData::Make(type, length, {NULLPTR}, 0, offset));
+  data_->child_data.push_back(std::move(run_ends_array->data()));
+  data_->child_data.push_back(std::move(values_array->data()));
+}
+
+Result<std::shared_ptr<RunLengthEncodedArray>> RunLengthEncodedArray::Make(
+    const std::shared_ptr<Array>& run_ends_array,
+    const std::shared_ptr<Array>& values_array, int64_t logical_length, int64_t offset) {
+  if (!RunLengthEncodedType::RunEndsTypeValid(*run_ends_array->type())) {
+    return Status::Invalid("Run ends array must be int16, int32 or int64 type");
+  }
+  if (run_ends_array->null_count() != 0) {
+    return Status::Invalid("Run ends array cannot contain null values");
+  }
+
+  return std::make_shared<RunLengthEncodedArray>(
+      run_length_encoded(run_ends_array->type(), values_array->type()), logical_length,
+      run_ends_array, values_array, offset);
+}
+
+std::shared_ptr<Array> RunLengthEncodedArray::values_array() const {
+  return MakeArray(data()->child_data[1]);
+}
+
+std::shared_ptr<Array> RunLengthEncodedArray::run_ends_array() const {
+  return MakeArray(data()->child_data[0]);
+}
+
+int64_t RunLengthEncodedArray::GetPhysicalOffset() const {
+  const ArraySpan span(*this->data_);
+  return rle_util::GetPhysicalOffset(span);
+}
+
+int64_t RunLengthEncodedArray::GetPhysicalLength() const {
+  const ArraySpan span(*this->data_);
+  return rle_util::GetPhysicalLength(span);
+}
+
+}  // namespace arrow

cpp/src/arrow/array/array_encoded.h

@@ -0,0 +1,87 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// Array accessor classes run-length encoded arrays
+
+#pragma once
+
+#include <cstdint>
+#include <memory>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "arrow/array/array_base.h"
+#include "arrow/array/data.h"
+#include "arrow/result.h"
+#include "arrow/status.h"
+#include "arrow/type.h"
+#include "arrow/type_fwd.h"
+#include "arrow/util/checked_cast.h"
+#include "arrow/util/macros.h"
+#include "arrow/util/visibility.h"
+
+namespace arrow {
+
+/// \addtogroup encoded-arrays
+///
+/// @{
+
+// ----------------------------------------------------------------------
+// RunLengthEncoded
+
+/// Concrete Array class for run-length encoded data
+class ARROW_EXPORT RunLengthEncodedArray : public Array {
+ public:
+  using TypeClass = RunLengthEncodedType;
+
+  explicit RunLengthEncodedArray(const std::shared_ptr<ArrayData>& data);
+
+  RunLengthEncodedArray(const std::shared_ptr<DataType>& type, int64_t length,
+                        const std::shared_ptr<Array>& run_ends_array,
+                        const std::shared_ptr<Array>& values_array, int64_t offset = 0);
+
+  /// \brief Construct a RunLengthEncodedArray from values and run ends arrays
+  ///
+  /// The data type is automatically inferred from the arguments.
+  /// The run_ends_array and values_array must be the same length.
+  static Result<std::shared_ptr<RunLengthEncodedArray>> Make(
+      const std::shared_ptr<Array>& run_ends_array,
+      const std::shared_ptr<Array>& values_array, int64_t logical_length,
+      int64_t offset = 0);
+
+  /// \brief Returns an array holding the values of each run. This function does apply the
+  /// physical offset to the array
+  std::shared_ptr<Array> values_array() const;
+
+  /// \brief Returns an array holding the logical indexes of each run end. This function
+  /// does apply the physical offset to the array
+  std::shared_ptr<Array> run_ends_array() const;
+
+  /// \brief Get the physical offset of the RLE array. Warning: calling this may result in
+  /// in an O(log(N)) binary search on the run ends buffer
+  int64_t GetPhysicalOffset() const;
+
+  /// \brief Get the physical offset of the RLE array. Avoid calling this method in a
+  /// context where you can easily calculate the value yourself. Calling this can result
+  /// in an O(log(N)) binary search on the run ends buffer
+  int64_t GetPhysicalLength() const;
+};
+
+/// @}
+
+}  // namespace arrow

cpp/src/arrow/array/array_encoded_test.cc

@@ -0,0 +1,145 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <gtest/gtest.h>
+
+#include <cstdint>
+#include <cstring>
+#include <memory>
+#include <vector>
+
+#include "arrow/array.h"
+#include "arrow/array/builder_nested.h"
+#include "arrow/chunked_array.h"
+#include "arrow/status.h"
+#include "arrow/testing/builder.h"
+#include "arrow/testing/gtest_util.h"
+#include "arrow/type.h"
+#include "arrow/util/checked_cast.h"
+
+namespace arrow {
+
+using internal::checked_cast;
+
+// ----------------------------------------------------------------------
+// Run-length encoded array tests
+
+namespace {
+
+class TestRunLengthEncodedArray
+    : public ::testing::TestWithParam<std::shared_ptr<DataType>> {
+ protected:
+  std::shared_ptr<DataType> run_ends_type;
+  std::shared_ptr<Array> string_values;
+  std::shared_ptr<Array> int32_values;
+  std::shared_ptr<Array> int16_values;
+  std::shared_ptr<Array> size_values;
+  std::shared_ptr<Array> size_only_null;
+
+  void SetUp() override {
+    run_ends_type = GetParam();
+    std::shared_ptr<DataType> run_ends_type = GetParam();
+
+    string_values = ArrayFromJSON(utf8(), R"(["Hello", "World", null])");
+    int32_values = ArrayFromJSON(int32(), "[10, 20, 30]");
+    int16_values = ArrayFromJSON(int16(), "[10, 20, 30]");
+    size_values = ArrayFromJSON(run_ends_type, "[10, 20, 30]");
+    size_only_null = ArrayFromJSON(run_ends_type, "[null, null, null]");
+  }
+};
+
+TEST_P(TestRunLengthEncodedArray, MakeArray) {
+  ASSERT_OK_AND_ASSIGN(auto rle_array,
+                       RunLengthEncodedArray::Make(int32_values, string_values, 3));
+  auto array_data = rle_array->data();
+  auto new_array = MakeArray(array_data);
+  ASSERT_ARRAYS_EQUAL(*new_array, *rle_array);
+  // should be the exact same ArrayData object
+  ASSERT_EQ(new_array->data(), array_data);
+  ASSERT_NE(std::dynamic_pointer_cast<RunLengthEncodedArray>(new_array), NULLPTR);
+}
+
+TEST_P(TestRunLengthEncodedArray, FromRunEndsAndValues) {
+  std::shared_ptr<RunLengthEncodedArray> rle_array;
+
+  ASSERT_OK_AND_ASSIGN(rle_array,
+                       RunLengthEncodedArray::Make(size_values, int32_values, 3));
+  ASSERT_EQ(rle_array->length(), 3);
+  ASSERT_ARRAYS_EQUAL(*rle_array->values_array(), *int32_values);
+  ASSERT_ARRAYS_EQUAL(*rle_array->run_ends_array(), *size_values);
+  ASSERT_EQ(rle_array->offset(), 0);
+  ASSERT_EQ(rle_array->data()->null_count, 0);
+  // one dummy buffer, since code may assume there is exactly one buffer
+  ASSERT_EQ(rle_array->data()->buffers.size(), 1);
+
+  // explicitly passing offset
+  ASSERT_OK_AND_ASSIGN(rle_array,
+                       RunLengthEncodedArray::Make(size_values, string_values, 2, 1));
+  ASSERT_EQ(rle_array->length(), 2);
+  ASSERT_ARRAYS_EQUAL(*rle_array->values_array(), *string_values);
+  ASSERT_ARRAYS_EQUAL(*rle_array->run_ends_array(), *size_values);
+  ASSERT_EQ(rle_array->offset(), 1);
+  // explicitly access null count variable so it is not calculated automatically
+  ASSERT_EQ(rle_array->data()->null_count, 0);
+
+  ASSERT_RAISES_WITH_MESSAGE(Invalid,
+                             "Invalid: Run ends array must be int16, int32 or int64 type",
+                             RunLengthEncodedArray::Make(string_values, int32_values, 3));
+  ASSERT_RAISES_WITH_MESSAGE(
+      Invalid, "Invalid: Run ends array cannot contain null values",
+      RunLengthEncodedArray::Make(size_only_null, int32_values, 3));
+}
+
+TEST_P(TestRunLengthEncodedArray, OffsetLength) {
+  auto run_ends = ArrayFromJSON(run_ends_type, "[100, 200, 300, 400, 500]");
+  auto values = ArrayFromJSON(utf8(), R"(["Hello", "beautiful", "world", "of", "RLE"])");
+  ASSERT_OK_AND_ASSIGN(auto rle_array,
+                       RunLengthEncodedArray::Make(run_ends, values, 500));
+
+  ASSERT_EQ(rle_array->GetPhysicalLength(), 5);
+  ASSERT_EQ(rle_array->GetPhysicalOffset(), 0);
+
+  auto slice = std::dynamic_pointer_cast<RunLengthEncodedArray>(rle_array->Slice(199, 5));
+  ASSERT_EQ(slice->GetPhysicalLength(), 2);
+  ASSERT_EQ(slice->GetPhysicalOffset(), 1);
+
+  auto slice2 =
+      std::dynamic_pointer_cast<RunLengthEncodedArray>(rle_array->Slice(199, 101));
+  ASSERT_EQ(slice2->GetPhysicalLength(), 2);
+  ASSERT_EQ(slice2->GetPhysicalOffset(), 1);
+
+  auto slice3 =
+      std::dynamic_pointer_cast<RunLengthEncodedArray>(rle_array->Slice(400, 100));
+  ASSERT_EQ(slice3->GetPhysicalLength(), 1);
+  ASSERT_EQ(slice3->GetPhysicalOffset(), 4);
+
+  auto slice4 =
+      std::dynamic_pointer_cast<RunLengthEncodedArray>(rle_array->Slice(0, 150));
+  ASSERT_EQ(slice4->GetPhysicalLength(), 2);
+  ASSERT_EQ(slice4->GetPhysicalOffset(), 0);
+
+  auto zero_length_at_end =
+      std::dynamic_pointer_cast<RunLengthEncodedArray>(rle_array->Slice(500, 0));
+  ASSERT_EQ(zero_length_at_end->GetPhysicalLength(), 0);
+  ASSERT_EQ(zero_length_at_end->GetPhysicalOffset(), 5);
+}
+
+INSTANTIATE_TEST_SUITE_P(EncodedArrayTests, TestRunLengthEncodedArray,
+                         ::testing::Values(int16(), int32(), int64()));
+}  // anonymous namespace
+
+}  // namespace arrow

cpp/src/arrow/array/builder_base.cc

@@ -279,7 +279,7 @@ struct AppendScalarImpl {
     return Status::NotImplemented("AppendScalar for type ", type);
   }
 
-  Status Convert() { return VisitTypeInline(*(*scalars_begin_)->type, this); }
+  Status Convert() { return VisitScalarTypeInline(*(*scalars_begin_)->type, this); }
 
   const std::shared_ptr<Scalar>* scalars_begin_;
   const std::shared_ptr<Scalar>* scalars_end_;

cpp/src/arrow/array/concatenate.cc

@@ -436,6 +436,10 @@ class ConcatenateImpl {
     return Status::OK();
   }
 
+  Status Visit(const RunLengthEncodedType& type) {
+    return Status::NotImplemented("concatenation of ", type);
+  }
+
   Status Visit(const ExtensionType& e) {
     // XXX can we just concatenate their storage?
     return Status::NotImplemented("concatenation of ", e);

cpp/src/arrow/array/data.cc

@@ -195,6 +195,7 @@ int GetNumBuffers(const DataType& type) {
     case Type::NA:
     case Type::STRUCT:
     case Type::FIXED_SIZE_LIST:
+    case Type::RUN_LENGTH_ENCODED:
       return 1;
     case Type::BINARY:
     case Type::LARGE_BINARY: