diff --git a/be/src/vec/exec/jni_connector.cpp b/be/src/vec/exec/jni_connector.cpp index bad7f52cc3b547..88b860f64b117c 100644 --- a/be/src/vec/exec/jni_connector.cpp +++ b/be/src/vec/exec/jni_connector.cpp @@ -18,7 +18,6 @@ #include "jni_connector.h" #include -#include #include #include @@ -27,13 +26,17 @@ #include "runtime/decimalv2_value.h" #include "runtime/runtime_state.h" #include "util/jni-util.h" -#include "vec/columns/column.h" +#include "vec/columns/column_array.h" +#include "vec/columns/column_map.h" #include "vec/columns/column_nullable.h" #include "vec/columns/column_string.h" +#include "vec/columns/column_struct.h" #include "vec/core/block.h" -#include "vec/core/column_with_type_and_name.h" #include "vec/core/types.h" +#include "vec/data_types/data_type_array.h" +#include "vec/data_types/data_type_map.h" #include "vec/data_types/data_type_nullable.h" +#include "vec/data_types/data_type_struct.h" namespace doris { class RuntimeProfile; @@ -301,6 +304,12 @@ Status JniConnector::_fill_column(ColumnPtr& doris_column, DataTypePtr& data_typ [[fallthrough]]; case TypeIndex::FixedString: return _fill_string_column(data_column, num_rows); + case TypeIndex::Array: + return _fill_array_column(data_column, data_type, num_rows); + case TypeIndex::Map: + return _fill_map_column(data_column, data_type, num_rows); + case TypeIndex::Struct: + return _fill_struct_column(data_column, data_type, num_rows); default: return Status::InvalidArgument("Unsupported type {} in jni scanner", getTypeName(logical_type)); @@ -308,17 +317,88 @@ Status JniConnector::_fill_column(ColumnPtr& doris_column, DataTypePtr& data_typ return Status::OK(); } +Status JniConnector::_fill_array_column(MutableColumnPtr& doris_column, DataTypePtr& data_type, + size_t num_rows) { + ColumnPtr& element_column = static_cast(*doris_column).get_data_ptr(); + DataTypePtr& element_type = const_cast( + (reinterpret_cast(remove_nullable(data_type).get())) + ->get_nested_type()); + ColumnArray::Offsets64& offsets_data = static_cast(*doris_column).get_offsets(); + + int64* offsets = reinterpret_cast(_next_meta_as_ptr()); + size_t origin_size = offsets_data.size(); + offsets_data.resize(origin_size + num_rows); + size_t start_offset = offsets_data[origin_size - 1]; + for (size_t i = 0; i < num_rows; ++i) { + offsets_data[origin_size + i] = offsets[i] + start_offset; + } + + // offsets[num_rows - 1] == offsets_data[origin_size + num_rows - 1] - start_offset + // but num_row equals 0 when there are all empty arrays + return _fill_column(element_column, element_type, + offsets_data[origin_size + num_rows - 1] - start_offset); +} + +Status JniConnector::_fill_map_column(MutableColumnPtr& doris_column, DataTypePtr& data_type, + size_t num_rows) { + auto& map = static_cast(*doris_column); + DataTypePtr& key_type = const_cast( + reinterpret_cast(remove_nullable(data_type).get())->get_key_type()); + DataTypePtr& value_type = const_cast( + reinterpret_cast(remove_nullable(data_type).get()) + ->get_value_type()); + ColumnPtr& key_column = map.get_keys_ptr(); + ColumnPtr& value_column = map.get_values_ptr(); + ColumnArray::Offsets64& map_offsets = map.get_offsets(); + + int64* offsets = reinterpret_cast(_next_meta_as_ptr()); + size_t origin_size = map_offsets.size(); + map_offsets.resize(origin_size + num_rows); + size_t start_offset = map_offsets[origin_size - 1]; + for (size_t i = 0; i < num_rows; ++i) { + map_offsets[origin_size + i] = offsets[i] + start_offset; + } + + RETURN_IF_ERROR(_fill_column(key_column, key_type, + map_offsets[origin_size + num_rows - 1] - start_offset)); + return _fill_column(value_column, value_type, + map_offsets[origin_size + num_rows - 1] - start_offset); +} + +Status JniConnector::_fill_struct_column(MutableColumnPtr& doris_column, DataTypePtr& data_type, + size_t num_rows) { + auto& doris_struct = static_cast(*doris_column); + const DataTypeStruct* doris_struct_type = + reinterpret_cast(remove_nullable(data_type).get()); + for (int i = 0; i < doris_struct.tuple_size(); ++i) { + ColumnPtr& struct_field = doris_struct.get_column_ptr(i); + DataTypePtr& field_type = const_cast(doris_struct_type->get_element(i)); + RETURN_IF_ERROR(_fill_column(struct_field, field_type, num_rows)); + } + return Status::OK(); +} + Status JniConnector::_fill_string_column(MutableColumnPtr& doris_column, size_t num_rows) { + if (num_rows == 0) { + return Status::OK(); + } + auto& string_col = static_cast(*doris_column); + ColumnString::Chars& string_chars = const_cast(string_col.get_chars()); + ColumnString::Offsets& string_offsets = + const_cast(string_col.get_offsets()); int* offsets = reinterpret_cast(_next_meta_as_ptr()); - char* data = reinterpret_cast(_next_meta_as_ptr()); - std::vector string_values; - string_values.reserve(num_rows); + char* chars = reinterpret_cast(_next_meta_as_ptr()); + + size_t origin_chars_size = string_chars.size(); + string_chars.resize(origin_chars_size + offsets[num_rows - 1]); + memcpy(string_chars.data() + origin_chars_size, chars, offsets[num_rows - 1]); + + size_t origin_offsets_size = string_offsets.size(); + size_t start_offset = string_offsets[origin_offsets_size - 1]; + string_offsets.resize(origin_offsets_size + num_rows); for (size_t i = 0; i < num_rows; ++i) { - int start_offset = i == 0 ? 0 : offsets[i - 1]; - int end_offset = offsets[i]; - string_values.emplace_back(data + start_offset, end_offset - start_offset); + string_offsets[origin_offsets_size + i] = offsets[i] + start_offset; } - doris_column->insert_many_strings(&string_values[0], num_rows); return Status::OK(); } @@ -418,77 +498,108 @@ std::string JniConnector::get_hive_type(const TypeDescriptor& desc) { } } -Status JniConnector::generate_meta_info(Block* block, std::unique_ptr& meta) { - std::vector meta_data; - // insert number of rows - meta_data.emplace_back(block->rows()); - for (int i = 0; i < block->columns(); ++i) { - auto& column_with_type_and_name = block->get_by_position(i); - auto& column_ptr = column_with_type_and_name.column; - auto& column_type = column_with_type_and_name.type; - TypeIndex logical_type = remove_nullable(column_type)->get_type_id(); - - // insert null map address - MutableColumnPtr data_column; - if (column_ptr->is_nullable()) { - auto* nullable_column = reinterpret_cast( - column_ptr->assume_mutable().get()); - data_column = nullable_column->get_nested_column_ptr(); - NullMap& null_map = nullable_column->get_null_map_data(); - meta_data.emplace_back((long)null_map.data()); - } else { - meta_data.emplace_back(0); - data_column = column_ptr->assume_mutable(); - } - - switch (logical_type) { +void JniConnector::_fill_column_meta(ColumnPtr& doris_column, DataTypePtr& data_type, + std::vector& meta_data) { + TypeIndex logical_type = remove_nullable(data_type)->get_type_id(); + // insert null map address + MutableColumnPtr data_column; + if (doris_column->is_nullable()) { + auto* nullable_column = + reinterpret_cast(doris_column->assume_mutable().get()); + data_column = nullable_column->get_nested_column_ptr(); + NullMap& null_map = nullable_column->get_null_map_data(); + meta_data.emplace_back((long)null_map.data()); + } else { + meta_data.emplace_back(0); + data_column = doris_column->assume_mutable(); + } + switch (logical_type) { #define DISPATCH(NUMERIC_TYPE, CPP_NUMERIC_TYPE) \ case NUMERIC_TYPE: { \ meta_data.emplace_back(_get_numeric_data_address(data_column)); \ break; \ } - FOR_LOGICAL_NUMERIC_TYPES(DISPATCH) + FOR_LOGICAL_NUMERIC_TYPES(DISPATCH) #undef DISPATCH - case TypeIndex::Decimal128: - [[fallthrough]]; - case TypeIndex::Decimal128I: { - meta_data.emplace_back(_get_decimal_data_address(data_column)); - break; - } - case TypeIndex::Decimal32: { - meta_data.emplace_back(_get_decimal_data_address(data_column)); - break; - } - case TypeIndex::Decimal64: { - meta_data.emplace_back(_get_decimal_data_address(data_column)); - break; - } - case TypeIndex::DateV2: { - meta_data.emplace_back(_get_time_data_address(data_column)); - break; - } - case TypeIndex::DateTimeV2: { - meta_data.emplace_back(_get_time_data_address(data_column)); - break; - } - case TypeIndex::String: - [[fallthrough]]; - case TypeIndex::FixedString: { - auto& string_column = static_cast(*data_column); - // inert offsets - meta_data.emplace_back((long)string_column.get_offsets().data()); - meta_data.emplace_back((long)string_column.get_chars().data()); - break; - } - case TypeIndex::Array: - [[fallthrough]]; - case TypeIndex::Struct: - [[fallthrough]]; - case TypeIndex::Map: - return Status::IOError("Unhandled type {}", getTypeName(logical_type)); - default: - return Status::IOError("Unsupported type {}", getTypeName(logical_type)); + case TypeIndex::Decimal128: + [[fallthrough]]; + case TypeIndex::Decimal128I: { + meta_data.emplace_back(_get_decimal_data_address(data_column)); + break; + } + case TypeIndex::Decimal32: { + meta_data.emplace_back(_get_decimal_data_address(data_column)); + break; + } + case TypeIndex::Decimal64: { + meta_data.emplace_back(_get_decimal_data_address(data_column)); + break; + } + case TypeIndex::DateV2: { + meta_data.emplace_back(_get_time_data_address(data_column)); + break; + } + case TypeIndex::DateTimeV2: { + meta_data.emplace_back(_get_time_data_address(data_column)); + break; + } + case TypeIndex::String: + [[fallthrough]]; + case TypeIndex::FixedString: { + auto& string_column = static_cast(*data_column); + // inert offsets + meta_data.emplace_back((long)string_column.get_offsets().data()); + meta_data.emplace_back((long)string_column.get_chars().data()); + break; + } + case TypeIndex::Array: { + ColumnPtr& element_column = static_cast(*data_column).get_data_ptr(); + meta_data.emplace_back((long)static_cast(*data_column).get_offsets().data()); + DataTypePtr& element_type = const_cast( + (reinterpret_cast(remove_nullable(data_type).get())) + ->get_nested_type()); + _fill_column_meta(element_column, element_type, meta_data); + break; + } + case TypeIndex::Struct: { + auto& doris_struct = static_cast(*data_column); + const DataTypeStruct* doris_struct_type = + reinterpret_cast(remove_nullable(data_type).get()); + for (int i = 0; i < doris_struct.tuple_size(); ++i) { + ColumnPtr& struct_field = doris_struct.get_column_ptr(i); + DataTypePtr& field_type = const_cast(doris_struct_type->get_element(i)); + _fill_column_meta(struct_field, field_type, meta_data); } + break; + } + case TypeIndex::Map: { + auto& map = static_cast(*data_column); + DataTypePtr& key_type = const_cast( + reinterpret_cast(remove_nullable(data_type).get()) + ->get_key_type()); + DataTypePtr& value_type = const_cast( + reinterpret_cast(remove_nullable(data_type).get()) + ->get_value_type()); + ColumnPtr& key_column = map.get_keys_ptr(); + ColumnPtr& value_column = map.get_values_ptr(); + meta_data.emplace_back((long)map.get_offsets().data()); + _fill_column_meta(key_column, key_type, meta_data); + _fill_column_meta(value_column, value_type, meta_data); + break; + } + default: + return; + } +} + +Status JniConnector::generate_meta_info(Block* block, std::unique_ptr& meta) { + std::vector meta_data; + // insert number of rows + meta_data.emplace_back(block->rows()); + for (int i = 0; i < block->columns(); ++i) { + auto& column_with_type_and_name = block->get_by_position(i); + _fill_column_meta(column_with_type_and_name.column, column_with_type_and_name.type, + meta_data); } meta.reset(new long[meta_data.size()]); diff --git a/be/src/vec/exec/jni_connector.h b/be/src/vec/exec/jni_connector.h index f7779736c2c784..8ea94242100648 100644 --- a/be/src/vec/exec/jni_connector.h +++ b/be/src/vec/exec/jni_connector.h @@ -298,6 +298,18 @@ class JniConnector { Status _fill_column(ColumnPtr& doris_column, DataTypePtr& data_type, size_t num_rows); + Status _fill_map_column(MutableColumnPtr& doris_column, DataTypePtr& data_type, + size_t num_rows); + + Status _fill_array_column(MutableColumnPtr& doris_column, DataTypePtr& data_type, + size_t num_rows); + + Status _fill_struct_column(MutableColumnPtr& doris_column, DataTypePtr& data_type, + size_t num_rows); + + static void _fill_column_meta(ColumnPtr& doris_column, DataTypePtr& data_type, + std::vector& meta_data); + template Status _fill_numeric_column(MutableColumnPtr& doris_column, CppType* ptr, size_t num_rows) { auto& column_data = static_cast&>(*doris_column).get_data(); diff --git a/fe/be-java-extensions/hudi-scanner/src/main/java/org/apache/doris/hudi/HudiColumnValue.java b/fe/be-java-extensions/hudi-scanner/src/main/java/org/apache/doris/hudi/HudiColumnValue.java index 7d402e84292df7..1c489affe16521 100644 --- a/fe/be-java-extensions/hudi-scanner/src/main/java/org/apache/doris/hudi/HudiColumnValue.java +++ b/fe/be-java-extensions/hudi-scanner/src/main/java/org/apache/doris/hudi/HudiColumnValue.java @@ -17,13 +17,13 @@ package org.apache.doris.hudi; - import org.apache.doris.common.jni.vec.ColumnType; import org.apache.doris.common.jni.vec.ColumnValue; -import org.apache.doris.common.jni.vec.NativeColumnValue; import org.apache.spark.sql.catalyst.InternalRow; -import org.apache.spark.sql.catalyst.expressions.UnsafeRow; +import org.apache.spark.sql.catalyst.expressions.SpecializedGetters; +import org.apache.spark.sql.catalyst.util.ArrayData; +import org.apache.spark.sql.catalyst.util.MapData; import java.math.BigDecimal; import java.math.BigInteger; @@ -33,41 +33,33 @@ import java.time.ZoneId; import java.util.List; -public class HudiColumnValue implements ColumnValue, NativeColumnValue { - private boolean isUnsafe; - private InternalRow internalRow; +public class HudiColumnValue implements ColumnValue { + private SpecializedGetters data; private int ordinal; - private int precision; - private int scale; + private ColumnType columnType; HudiColumnValue() { } - HudiColumnValue(InternalRow internalRow, int ordinal, int precision, int scale) { - this.isUnsafe = internalRow instanceof UnsafeRow; - this.internalRow = internalRow; + HudiColumnValue(SpecializedGetters data, int ordinal, ColumnType columnType) { + this.data = data; this.ordinal = ordinal; - this.precision = precision; - this.scale = scale; + this.columnType = columnType; } - public void reset(InternalRow internalRow, int ordinal, int precision, int scale) { - this.isUnsafe = internalRow instanceof UnsafeRow; - this.internalRow = internalRow; + public void reset(SpecializedGetters data, int ordinal, ColumnType columnType) { + this.data = data; this.ordinal = ordinal; - this.precision = precision; - this.scale = scale; + this.columnType = columnType; } - public void reset(int ordinal, int precision, int scale) { + public void reset(int ordinal, ColumnType columnType) { this.ordinal = ordinal; - this.precision = precision; - this.scale = scale; + this.columnType = columnType; } - public void reset(InternalRow internalRow) { - this.isUnsafe = internalRow instanceof UnsafeRow; - this.internalRow = internalRow; + public void reset(SpecializedGetters data) { + this.data = data; } @Override @@ -77,42 +69,42 @@ public boolean canGetStringAsBytes() { @Override public boolean isNull() { - return internalRow.isNullAt(ordinal); + return data.isNullAt(ordinal); } @Override public boolean getBoolean() { - return internalRow.getBoolean(ordinal); + return data.getBoolean(ordinal); } @Override public byte getByte() { - return internalRow.getByte(ordinal); + return data.getByte(ordinal); } @Override public short getShort() { - return internalRow.getShort(ordinal); + return data.getShort(ordinal); } @Override public int getInt() { - return internalRow.getInt(ordinal); + return data.getInt(ordinal); } @Override public float getFloat() { - return internalRow.getFloat(ordinal); + return data.getFloat(ordinal); } @Override public long getLong() { - return internalRow.getLong(ordinal); + return data.getLong(ordinal); } @Override public double getDouble() { - return internalRow.getDouble(ordinal); + return data.getDouble(ordinal); } @Override @@ -122,78 +114,74 @@ public BigInteger getBigInteger() { @Override public BigDecimal getDecimal() { - return internalRow.getDecimal(ordinal, precision, scale).toJavaBigDecimal(); + return data.getDecimal(ordinal, columnType.getPrecision(), columnType.getScale()).toJavaBigDecimal(); } @Override public String getString() { - return internalRow.getUTF8String(ordinal).toString(); + return data.getUTF8String(ordinal).toString(); } @Override public byte[] getStringAsBytes() { - return internalRow.getUTF8String(ordinal).getBytes(); + return data.getUTF8String(ordinal).getBytes(); } @Override public LocalDate getDate() { - return LocalDate.ofEpochDay(internalRow.getInt(ordinal)); + return LocalDate.ofEpochDay(data.getInt(ordinal)); } @Override public LocalDateTime getDateTime() { - long datetime = internalRow.getLong(ordinal); + long datetime = data.getLong(ordinal); long seconds; long nanoseconds; - if (precision == 3) { + if (columnType.getPrecision() == 3) { seconds = datetime / 1000; nanoseconds = (datetime % 1000) * 1000000; - } else if (precision == 6) { + } else if (columnType.getPrecision() == 6) { seconds = datetime / 1000000; nanoseconds = (datetime % 1000000) * 1000; } else { - throw new RuntimeException("Hoodie timestamp only support milliseconds and microseconds"); + throw new RuntimeException("Hoodie timestamp only support milliseconds and microseconds, wrong precision = " + + columnType.getPrecision()); } return LocalDateTime.ofInstant(Instant.ofEpochSecond(seconds, nanoseconds), ZoneId.systemDefault()); } @Override public byte[] getBytes() { - return internalRow.getBinary(ordinal); + return data.getBinary(ordinal); } @Override public void unpackArray(List values) { - + ArrayData array = data.getArray(ordinal); + for (int i = 0; i < array.numElements(); ++i) { + values.add(new HudiColumnValue(array, i, columnType.getChildTypes().get(0))); + } } @Override public void unpackMap(List keys, List values) { - + MapData map = data.getMap(ordinal); + ArrayData key = map.keyArray(); + for (int i = 0; i < key.numElements(); ++i) { + keys.add(new HudiColumnValue(key, i, columnType.getChildTypes().get(0))); + } + ArrayData value = map.valueArray(); + for (int i = 0; i < value.numElements(); ++i) { + values.add(new HudiColumnValue(value, i, columnType.getChildTypes().get(1))); + } } @Override public void unpackStruct(List structFieldIndex, List values) { - - } - - @Override - public NativeValue getNativeValue(ColumnType.Type type) { - if (isUnsafe) { - UnsafeRow unsafeRow = (UnsafeRow) internalRow; - switch (type) { - case CHAR: - case VARCHAR: - case BINARY: - case STRING: - long offsetAndSize = unsafeRow.getLong(ordinal); - int offset = (int) (offsetAndSize >> 32); - int size = (int) offsetAndSize; - return new NativeValue(unsafeRow.getBaseObject(), offset, size); - default: - return null; - } + // todo: support pruned struct fields + InternalRow struct = data.getStruct(ordinal, structFieldIndex.size()); + for (int i : structFieldIndex) { + values.add(new HudiColumnValue(struct, i, columnType.getChildTypes().get(i))); } - return null; } } diff --git a/fe/be-java-extensions/hudi-scanner/src/main/java/org/apache/doris/hudi/HudiJniScanner.java b/fe/be-java-extensions/hudi-scanner/src/main/java/org/apache/doris/hudi/HudiJniScanner.java index 64c4fd70e7b542..daf8d4a21fa87b 100644 --- a/fe/be-java-extensions/hudi-scanner/src/main/java/org/apache/doris/hudi/HudiJniScanner.java +++ b/fe/be-java-extensions/hudi-scanner/src/main/java/org/apache/doris/hudi/HudiJniScanner.java @@ -224,7 +224,7 @@ public int getNext() throws IOException { while (readRowNumbers < fetchSize && recordIterator.hasNext()) { columnValue.reset(recordIterator.next()); for (int i = 0; i < numFields; i++) { - columnValue.reset(i, columnTypes[i].getPrecision(), columnTypes[i].getScale()); + columnValue.reset(i, columnTypes[i]); appendData(i, columnValue); } readRowNumbers++; diff --git a/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/MockJniScanner.java b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/MockJniScanner.java index 3557b3b9032073..bc7561e2a23fc7 100644 --- a/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/MockJniScanner.java +++ b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/MockJniScanner.java @@ -42,6 +42,10 @@ public static class MockColumnValue implements ColumnValue { private int i; private int j; + public MockColumnValue(int i, int j) { + set(i, j); + } + public MockColumnValue() { } @@ -132,17 +136,40 @@ public byte[] getBytes() { @Override public void unpackArray(List values) { - + for (int m = 1; m < i; ++m) { + if (m % 3 == 0) { + values.add(null); + } else { + values.add(new MockColumnValue(i, j + m)); + } + } } @Override public void unpackMap(List keys, List values) { - + for (int m = 0; m < i; ++m) { + values.add(new MockColumnValue(i + m, j)); + } + for (int m = 0; m < i; ++m) { + if (m % 3 == 0) { + values.add(null); + } else { + values.add(new MockColumnValue(i, j + m)); + } + } } @Override public void unpackStruct(List structFieldIndex, List values) { - + structFieldIndex.clear(); + structFieldIndex.add(0); + structFieldIndex.add(1); + if ((i + j) % 4 == 0) { + values.add(null); + } else { + values.add(new MockColumnValue(i, j)); + } + values.add(new MockColumnValue(i, j + 3)); } } diff --git a/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/VectorColumn.java b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/VectorColumn.java index 3998a1a3270aff..6d0ede0a4f3774 100644 --- a/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/VectorColumn.java +++ b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/VectorColumn.java @@ -28,6 +28,8 @@ import java.nio.charset.StandardCharsets; import java.time.LocalDate; import java.time.LocalDateTime; +import java.util.ArrayList; +import java.util.Collections; import java.util.List; /** @@ -46,6 +48,9 @@ public class VectorColumn { // For String / Array / Map. private long offsets; + // String's offset is int32, while Array&Map's offset is int64 + // todo: how to solve the overflow of offsets when taking Array&Map's offset as int32 + private boolean isComplexType = false; // Number of elements in vector column private int capacity; // Upper limit for the maximum capacity for this column. @@ -59,6 +64,10 @@ public class VectorColumn { // For nested column type: String / Array/ Map / Struct private VectorColumn[] childColumns; + // For struct, only support to read all fields in struct now + // todo: support pruned struct fields + private List structFieldIndex; + public VectorColumn(ColumnType columnType, int capacity) { this.columnType = columnType; this.capacity = 0; @@ -68,11 +77,18 @@ public VectorColumn(ColumnType columnType, int capacity) { this.numNulls = 0; this.appendIndex = 0; if (columnType.isComplexType()) { + isComplexType = true; List children = columnType.getChildTypes(); childColumns = new VectorColumn[children.size()]; for (int i = 0; i < children.size(); ++i) { childColumns[i] = new VectorColumn(children.get(i), capacity); } + if (columnType.isStruct()) { + structFieldIndex = new ArrayList<>(); + for (int i = 0; i < children.size(); ++i) { + structFieldIndex.add(i); + } + } } else if (columnType.isStringType()) { childColumns = new VectorColumn[1]; childColumns[0] = new VectorColumn(new ColumnType("#stringBytes", Type.BYTE), @@ -91,6 +107,13 @@ public VectorColumn(long address, int capacity, ColumnType columnType) { this.offsets = 0; this.numNulls = 0; this.appendIndex = capacity; + if (columnType.isStruct()) { + List children = columnType.getChildTypes(); + structFieldIndex = new ArrayList<>(); + for (int i = 0; i < children.size(); ++i) { + structFieldIndex.add(i); + } + } } // restore block column @@ -114,8 +137,20 @@ public VectorColumn(ColumnType columnType, int numRows, long columnMetaAddress) this.appendIndex = numRows; if (columnType.isComplexType()) { - // todo: support complex type - throw new RuntimeException("Unhandled type: " + columnType); + isComplexType = true; + int childRows = numRows; + if (!columnType.isStruct()) { + this.offsets = OffHeap.getLong(null, address); + address += 8; + childRows = getArrayEndOffset(numRows - 1); + } + this.data = 0; + List children = columnType.getChildTypes(); + childColumns = new VectorColumn[children.size()]; + for (int i = 0; i < children.size(); ++i) { + childColumns[i] = new VectorColumn(children.get(i), childRows, address); + address += children.get(i).metaSize() * 8L; + } } else if (columnType.isStringType()) { this.offsets = OffHeap.getLong(null, address); address += 8; @@ -130,6 +165,19 @@ public VectorColumn(ColumnType columnType, int numRows, long columnMetaAddress) } } + private int getArrayEndOffset(int rowId) { + if (rowId >= 0 && rowId < appendIndex) { + if (isComplexType) { + // maybe overflowed + return (int) OffHeap.getLong(null, offsets + 8L * rowId); + } else { + return OffHeap.getInt(null, offsets + 4L * rowId); + } + } else { + return 0; + } + } + public long nullMapAddress() { return nullMap; } @@ -200,21 +248,21 @@ private void reserve(int requiredCapacity) { } private void reserveCapacity(int newCapacity) { + long offsetLength = isComplexType ? 8L : 4L; long oldCapacity = capacity; - long oldOffsetSize = capacity * 4L; - long newOffsetSize = newCapacity * 4L; + long oldOffsetSize = capacity * offsetLength; + long newOffsetSize = newCapacity * offsetLength; long typeSize = columnType.getTypeSize(); if (columnType.isUnsupported()) { // do nothing return; } else if (typeSize != -1) { this.data = OffHeap.reallocateMemory(data, oldCapacity * typeSize, newCapacity * typeSize); - } else if (columnType.isStringType()) { + } else if (columnType.isStringType() || columnType.isArray() || columnType.isMap()) { this.offsets = OffHeap.reallocateMemory(offsets, oldOffsetSize, newOffsetSize); - } else { + } else if (!columnType.isStruct()) { throw new RuntimeException("Unhandled type: " + columnType); } - // todo: support complex type if (!"#stringBytes".equals(columnType.getName())) { this.nullMap = OffHeap.reallocateMemory(nullMap, oldCapacity, newCapacity); OffHeap.setMemory(nullMap + oldCapacity, (byte) 0, newCapacity - oldCapacity); @@ -292,6 +340,12 @@ public int appendNull(ColumnType.Type typeValue) { case STRING: case BINARY: return appendBytesAndOffset(new byte[0]); + case ARRAY: + return appendArray(Collections.emptyList()); + case MAP: + return appendMap(Collections.emptyList(), Collections.emptyList()); + case STRUCT: + return appendStruct(structFieldIndex, null); default: throw new RuntimeException("Unknown type value: " + typeValue); } @@ -530,6 +584,45 @@ public String getStringWithOffset(int rowId) { return new String(bytes, StandardCharsets.UTF_8); } + public int appendArray(List values) { + int length = values.size(); + int startOffset = childColumns[0].appendIndex; + for (ColumnValue v : values) { + childColumns[0].appendValue(v); + } + reserve(appendIndex + 1); + OffHeap.putLong(null, offsets + 8L * appendIndex, startOffset + length); + return appendIndex++; + } + + public int appendMap(List keys, List values) { + int length = keys.size(); + int startOffset = childColumns[0].appendIndex; + for (ColumnValue k : keys) { + childColumns[0].appendValue(k); + } + for (ColumnValue v : values) { + childColumns[1].appendValue(v); + } + reserve(appendIndex + 1); + OffHeap.putInt(null, offsets + 8L * appendIndex, startOffset + length); + return appendIndex++; + } + + public int appendStruct(List structFieldIndex, List values) { + if (values == null) { + for (int i : structFieldIndex) { + childColumns[i].appendValue(null); + } + } else { + for (int i = 0; i < structFieldIndex.size(); ++i) { + childColumns[structFieldIndex.get(i)].appendValue(values.get(i)); + } + } + reserve(appendIndex + 1); + return appendIndex++; + } + public void updateMeta(VectorColumn meta) { if (columnType.isUnsupported()) { meta.appendLong(0); @@ -558,19 +651,7 @@ public void appendNativeValue(NativeColumnValue o) { return; } NativeValue nativeValue = o.getNativeValue(typeValue); - if (nativeValue == null) { - // can't get native value, fall back to materialized value - appendValue((ColumnValue) o); - return; - } - if (nativeValue.length == -1) { - // java origin types - long typeSize = typeValue.size; - reserve(appendIndex + 1); - OffHeap.copyMemory(nativeValue.baseObject, nativeValue.offset, - null, data + typeSize * appendIndex, typeSize); - appendIndex++; - } else { + if (nativeValue != null && columnType.isStringType()) { int byteLength = nativeValue.length; VectorColumn bytesColumn = childColumns[0]; int startOffset = bytesColumn.appendIndex; @@ -580,6 +661,9 @@ public void appendNativeValue(NativeColumnValue o) { bytesColumn.appendIndex += byteLength; OffHeap.putInt(null, offsets + 4L * appendIndex, startOffset + byteLength); appendIndex++; + } else { + // can't get native value, fall back to materialized value + appendValue((ColumnValue) o); } } @@ -639,6 +723,25 @@ public void appendValue(ColumnValue o) { case BINARY: appendBytesAndOffset(o.getBytes()); break; + case ARRAY: { + List values = new ArrayList<>(); + o.unpackArray(values); + appendArray(values); + break; + } + case MAP: { + List keys = new ArrayList<>(); + List values = new ArrayList<>(); + o.unpackMap(keys, values); + appendMap(keys, values); + break; + } + case STRUCT: { + List values = new ArrayList<>(); + o.unpackStruct(structFieldIndex, values); + appendStruct(structFieldIndex, values); + break; + } default: throw new RuntimeException("Unknown type value: " + typeValue); } @@ -695,6 +798,67 @@ public void dump(StringBuilder sb, int i) { case BINARY: sb.append(getStringWithOffset(i)); break; + case ARRAY: { + int begin = getArrayEndOffset(i - 1); + int end = getArrayEndOffset(i); + sb.append("["); + for (int rowId = begin; rowId < end; rowId++) { + if (rowId != begin) { + sb.append(","); + } + childColumns[0].dump(sb, rowId); + } + sb.append("]"); + break; + } + case MAP: { + int begin = getArrayEndOffset(i - 1); + int end = getArrayEndOffset(i); + sb.append("{"); + VectorColumn key = childColumns[0]; + VectorColumn value = childColumns[1]; + for (int rowId = begin; rowId < end; rowId++) { + if (rowId != begin) { + sb.append(","); + } + if (key.columnType.isStringType()) { + sb.append("\""); + } + key.dump(sb, rowId); + if (key.columnType.isStringType()) { + sb.append("\""); + } + sb.append(":"); + if (value.columnType.isStringType()) { + sb.append("\""); + } + value.dump(sb, rowId); + if (value.columnType.isStringType()) { + sb.append("\""); + } + } + sb.append("}"); + break; + } + case STRUCT: { + sb.append("{"); + for (int fieldIndex = 0; fieldIndex < childColumns.length; ++fieldIndex) { + VectorColumn child = childColumns[fieldIndex]; + if (fieldIndex != 0) { + sb.append(","); + } + sb.append("\"").append(child.columnType.getName()).append("\":"); + if (child.columnType.isStringType()) { + sb.append("\""); + } + child.dump(sb, i); + if (child.columnType.isStringType()) { + sb.append("\""); + } + } + sb.append("}"); + break; + } default: throw new RuntimeException("Unknown type value: " + typeValue); } diff --git a/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/VectorTable.java b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/VectorTable.java index 63b6f1ac2a9038..bb6e455f87a402 100644 --- a/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/VectorTable.java +++ b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/VectorTable.java @@ -58,7 +58,7 @@ public VectorTable(ColumnType[] types, String[] fields, long metaAddress) { this.numRows = (int) OffHeap.getLong(null, address); address += 8; - int metaSize = 1; // number of rows + int metaSize = 1; // stores the number of rows + other columns meta data for (int i = 0; i < types.length; i++) { columns[i] = new VectorColumn(types[i], numRows, address); metaSize += types[i].metaSize(); diff --git a/fe/be-java-extensions/java-common/src/test/java/org/apache/doris/common/jni/JniScannerTest.java b/fe/be-java-extensions/java-common/src/test/java/org/apache/doris/common/jni/JniScannerTest.java index e8a6d49df796ee..6f47b5bbb71a89 100644 --- a/fe/be-java-extensions/java-common/src/test/java/org/apache/doris/common/jni/JniScannerTest.java +++ b/fe/be-java-extensions/java-common/src/test/java/org/apache/doris/common/jni/JniScannerTest.java @@ -35,9 +35,10 @@ public void testMockJniScanner() throws IOException { { put("mock_rows", "128"); put("required_fields", "boolean,tinyint,smallint,int,bigint,largeint,float,double," - + "date,timestamp,char,varchar,string,decimalv2,decimal64"); + + "date,timestamp,char,varchar,string,decimalv2,decimal64,array,map,struct"); put("columns_types", "boolean#tinyint#smallint#int#bigint#largeint#float#double#" - + "date#timestamp#char(10)#varchar(10)#string#decimalv2(12,4)#decimal64(10,3)"); + + "date#timestamp#char(10)#varchar(10)#string#decimalv2(12,4)#decimal64(10,3)#" + + "array>#map>#struct>"); } }); scanner.open(); @@ -50,7 +51,7 @@ public void testMockJniScanner() throws IOException { VectorTable restoreTable = new VectorTable(scanner.getTable().getColumnTypes(), scanner.getTable().getFields(), metaAddress); - System.out.println(restoreTable.dump((int) rows)); + System.out.println(restoreTable.dump((int) rows).substring(0, 128)); // Restored table is release by the origin table. } scanner.resetTable(); diff --git a/regression-test/data/external_table_p2/hive/test_hive_hudi.out b/regression-test/data/external_table_p2/hive/test_hive_hudi.out index a695d3cdb7d1f0..3bee0e318da571 100644 --- a/regression-test/data/external_table_p2/hive/test_hive_hudi.out +++ b/regression-test/data/external_table_p2/hive/test_hive_hudi.out @@ -17,6 +17,108 @@ row_1 2011-11-11 1 v_1 row_2 2021-01-01 0 v_0 row_4 2021-02-01 4 v_4 +-- !complex_types -- +20230922203209630 20230922203209630_0_10000 10002 8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet 10002 a9999 [[9999], [3333], [9999, 8999, NULL]] {"k9999":[1, NULL, 9], "k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-03 00:26:48.000000, [10008, 9999, NULL]} +20230922203209630 20230922203209630_0_9999 10001 8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet 10001 a9998 [[9998], [3332], [9998, 8998, NULL]] {"k9998":[1, NULL, 9], "k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-02 23:25:47.000000, [10007, 9998, NULL]} +20230922203209630 20230922203209630_0_9998 10000 8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet 10000 a9997 [[9997], [3332], [9997, 8997, NULL]] {"k9997":[1, NULL, 9], "k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-02 22:24:46.000000, [10006, 9997, NULL]} +20230922203209630 20230922203209630_0_9997 9999 8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet 9999 a9996 [[9996], [3332], [9996, 8996, NULL]] {"k9996":[1, NULL, 9], "k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-02 21:23:45.000000, [10005, 9996, NULL]} +20230922203209630 20230922203209630_0_9996 9998 8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet 9998 a9995 [[9995], [], [9995, 8995, NULL]] {"k9995":[1, NULL, 9], "k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-02 20:22:44.000000, [10004, 9995, NULL]} +20230922203209630 20230922203209630_0_9995 9997 8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet 9997 a9994 [[9994], [3331], [9994, 8994, NULL]] {"k9994":[1, NULL, 9], "k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-02 19:21:43.000000, [10003, 9994, NULL]} +20230922203209630 20230922203209630_0_9994 9996 8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet 9996 a9993 [[9993], [3331], [9993, 8993, NULL]] {"k9993":[1, NULL, 9], "k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-02 18:20:42.000000, [10002, 9993, NULL]} +20230922204306289 20230922204306289_0_542 9995 8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0 9995 a9992-1 [[9992], [3330], [9992, 8992, NULL]] {"k3":NULL, "k4":[NULL, 9], "k9992":[1, NULL, 9], "k2":[]} {2012-02-02 17:19:41.000000, [10001, 9992, NULL]} +20230922203209630 20230922203209630_0_9992 9994 8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet 9994 a9991 [[9991], [3330], [9991, 8991, NULL]] {"k9991":[1, NULL, 9], "k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-02 16:18:40.000000, [10000, 9991, NULL]} +20230922203209630 20230922203209630_0_9991 9993 8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet 9993 a9990 [NULL, [], [9990, 8990, NULL]] {"k9990":[1, NULL, 9], "k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-02 15:17:39.000000, [9999, NULL, NULL]} +20230922203209630 20230922203209630_0_1000 1002 8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet 1002 a999 [[999], [333], [999, -1, NULL]] {"k999":[1, NULL, 9], "k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-02 14:32:13.000000, [1008, 999, NULL]} +20230922203209630 20230922203209630_0_9990 9992 8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet 9992 a9989 [[9989], [3329], [9989, 8989, NULL]] {"k9989":[1, NULL, 9], "k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-02 14:16:38.000000, [9998, 9989, NULL]} +20230922203209630 20230922203209630_0_9989 9991 8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet 9991 a9988 [[9988], [3329], [9988, 8988, NULL]] {"k9988":[1, NULL, 9], "k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-02 13:15:37.000000, [9997, 9988, NULL]} +20230922203209630 20230922203209630_0_9988 9990 8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet 9990 a9987 [[9987], [3329], [9987, 8987, NULL]] {"k9987":[1, NULL, 9], "k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-02 12:14:36.000000, [9996, 9987, NULL]} +20230922203209630 20230922203209630_0_9987 9989 8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet 9989 a9986 [[9986], [3328], [9986, 8986, NULL]] {"k9986":[1, NULL, 9], "k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-02 11:13:35.000000, [9995, 9986, NULL]} +20230922203209630 20230922203209630_0_9986 9988 8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet 9988 a9985 [[9985], [], [9985, 8985, NULL]] {"k9985":[1, NULL, 9], "k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-02 10:12:34.000000, [9994, 9985, NULL]} +20230922203209630 20230922203209630_0_9985 9987 8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet 9987 a9984 [[9984], [3328], [9984, 8984, NULL]] {"k9984":[1, NULL, 9], "k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-02 09:11:33.000000, [9993, 9984, NULL]} +20230922203209630 20230922203209630_0_9984 9986 8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet 9986 a9983 [[9983], [3327], [9983, 8983, NULL]] {"k9983":[1, NULL, 9], "k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-02 08:10:32.000000, [9992, 9983, NULL]} +20230922204306289 20230922204306289_0_32 9985 8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0 9985 a9982-1 [[9982], [3327], [9982, 8982, NULL]] {"k3":NULL, "k4":[NULL, 9], "k9982":[1, NULL, 9], "k2":[]} {2012-02-02 07:09:31.000000, [9991, 9982, NULL]} +20230922203209630 20230922203209630_0_9982 9984 8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet 9984 a9981 [[9981], [3327], [9981, 8981, NULL]] {"k9981":[1, NULL, 9], "k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-02 06:08:30.000000, [9990, 9981, NULL]} +20230922203209630 20230922203209630_0_9981 9983 8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet 9983 a9980 [NULL, [], [9980, 8980, NULL]] {"k9980":[1, NULL, 9], "k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-02 05:07:29.000000, [9989, NULL, NULL]} +20230922203209630 20230922203209630_0_999 1001 8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet 1001 a998 [[998], [332], [998, -2, NULL]] {"k998":[1, NULL, 9], "k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-02 13:31:12.000000, [1007, 998, NULL]} +20230922203209630 20230922203209630_0_9980 9982 8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet 9982 a9979 [[9979], [3326], [9979, 8979, NULL]] {"k9979":[1, NULL, 9], "k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-02 04:06:28.000000, [9988, 9979, NULL]} +20230922203209630 20230922203209630_0_9979 9981 8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet 9981 a9978 [[9978], [3326], [9978, 8978, NULL]] {"k9978":[1, NULL, 9], "k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-02 03:05:27.000000, [9987, 9978, NULL]} +20230922203209630 20230922203209630_0_9978 9980 8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet 9980 a9977 [[9977], [3325], [9977, 8977, NULL]] {"k9977":[1, NULL, 9], "k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-02 02:04:26.000000, [9986, 9977, NULL]} +20230922203209630 20230922203209630_0_9977 9979 8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet 9979 a9976 [[9976], [3325], [9976, 8976, NULL]] {"k9976":[1, NULL, 9], "k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-02 01:03:25.000000, [9985, 9976, NULL]} +20230922203209630 20230922203209630_0_9976 9978 8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet 9978 a9975 [[9975], [], [9975, 8975, NULL]] {"k9975":[1, NULL, 9], "k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-04 10:02:24.000000, [9984, 9975, NULL]} +20230922203209630 20230922203209630_0_9975 9977 8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet 9977 a9974 [[9974], [3324], [9974, 8974, NULL]] {"k9974":[1, NULL, 9], "k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-04 09:58:23.000000, [9983, 9974, NULL]} +20230922203209630 20230922203209630_0_9974 9976 8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet 9976 a9973 [[9973], [3324], [9973, 8973, NULL]] {"k9973":[1, NULL, 9], "k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-04 08:57:22.000000, [9982, 9973, NULL]} +20230922204306289 20230922204306289_0_939 9975 8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0 9975 a9972-1 [[9972], [3324], [9972, 8972, NULL]] {"k9972":[1, NULL, 9], "k3":NULL, "k4":[NULL, 9], "k2":[]} {2012-02-04 07:56:21.000000, [9981, 9972, NULL]} +20230922203209630 20230922203209630_0_9972 9974 8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet 9974 a9971 [[9971], [3323], [9971, 8971, NULL]] {"k9971":[1, NULL, 9], "k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-04 06:55:20.000000, [9980, 9971, NULL]} +20230922203209630 20230922203209630_0_9971 9973 8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet 9973 a9970 [NULL, [], [9970, 8970, NULL]] {"k9970":[1, NULL, 9], "k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-04 05:54:19.000000, [9979, NULL, NULL]} +20230922203209630 20230922203209630_0_998 1000 8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet 1000 a997 [[997], [332], [997, -3, NULL]] {"k997":[1, NULL, 9], "k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-02 12:30:11.000000, [1006, 997, NULL]} +20230922203209630 20230922203209630_0_9970 9972 8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet 9972 a9969 [[9969], [3323], [9969, 8969, NULL]] {"k9969":[1, NULL, 9], "k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-04 04:53:18.000000, [9978, 9969, NULL]} +20230922203209630 20230922203209630_0_9969 9971 8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet 9971 a9968 [[9968], [3322], [9968, 8968, NULL]] {"k9968":[1, NULL, 9], "k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-04 03:52:17.000000, [9977, 9968, NULL]} +20230922203209630 20230922203209630_0_9968 9970 8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet 9970 a9967 [[9967], [3322], [9967, 8967, NULL]] {"k9967":[1, NULL, 9], "k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-04 02:51:16.000000, [9976, 9967, NULL]} +20230922203209630 20230922203209630_0_9967 9969 8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet 9969 a9966 [[9966], [3322], [9966, 8966, NULL]] {"k9966":[1, NULL, 9], "k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-04 01:50:15.000000, [9975, 9966, NULL]} +20230922203209630 20230922203209630_0_9966 9968 8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet 9968 a9965 [[9965], [], [9965, 8965, NULL]] {"k9965":[1, NULL, 9], "k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-04 00:49:14.000000, [9974, 9965, NULL]} +20230922203209630 20230922203209630_0_9965 9967 8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet 9967 a9964 [[9964], [3321], [9964, 8964, NULL]] {"k9964":[1, NULL, 9], "k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-03 23:48:13.000000, [9973, 9964, NULL]} +20230922203209630 20230922203209630_0_9964 9966 8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet 9966 a9963 [[9963], [3321], [9963, 8963, NULL]] {"k9963":[1, NULL, 9], "k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-03 22:47:12.000000, [9972, 9963, NULL]} +20230922204306289 20230922204306289_0_173 9965 8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0 9965 a9962-1 [[9962], [3320], [9962, 8962, NULL]] {"k3":NULL, "k9962":[1, NULL, 9], "k4":[NULL, 9], "k2":[]} {2012-02-03 21:46:11.000000, [9971, 9962, NULL]} +20230922203209630 20230922203209630_0_9962 9964 8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet 9964 a9961 [[9961], [3320], [9961, 8961, NULL]] {"k9961":[1, NULL, 9], "k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-03 20:45:10.000000, [9970, 9961, NULL]} +20230922203209630 20230922203209630_0_9961 9963 8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet 9963 a9960 [NULL, [], [9960, 8960, NULL]] {"k9960":[1, NULL, 9], "k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-03 19:44:09.000000, [9969, NULL, NULL]} +20230922203209630 20230922203209630_0_997 999 8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet 999 a996 [[996], [332], [996, -4, NULL]] {"k996":[1, NULL, 9], "k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-02 11:29:10.000000, [1005, 996, NULL]} +20230922203209630 20230922203209630_0_9960 9962 8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet 9962 a9959 [[9959], [3319], [9959, 8959, NULL]] {"k9959":[1, NULL, 9], "k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-03 18:43:08.000000, [9968, 9959, NULL]} +20230922203209630 20230922203209630_0_9959 9961 8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet 9961 a9958 [[9958], [3319], [9958, 8958, NULL]] {"k9958":[1, NULL, 9], "k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-03 17:42:07.000000, [9967, 9958, NULL]} +20230922203209630 20230922203209630_0_9958 9960 8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet 9960 a9957 [[9957], [3319], [9957, 8957, NULL]] {"k9957":[1, NULL, 9], "k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-03 16:41:06.000000, [9966, 9957, NULL]} +20230922203209630 20230922203209630_0_9957 9959 8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet 9959 a9956 [[9956], [3318], [9956, 8956, NULL]] {"k9956":[1, NULL, 9], "k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-03 15:40:05.000000, [9965, 9956, NULL]} +20230922203209630 20230922203209630_0_9956 9958 8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet 9958 a9955 [[9955], [], [9955, 8955, NULL]] {"k9955":[1, NULL, 9], "k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-03 14:39:04.000000, [9964, 9955, NULL]} +20230922203209630 20230922203209630_0_9955 9957 8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet 9957 a9954 [[9954], [3318], [9954, 8954, NULL]] {"k9954":[1, NULL, 9], "k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-03 13:38:58.000000, [9963, 9954, NULL]} +20230922203209630 20230922203209630_0_9954 9956 8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet 9956 a9953 [[9953], [3317], [9953, 8953, NULL]] {"k9953":[1, NULL, 9], "k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-03 12:37:57.000000, [9962, 9953, NULL]} +20230922204306289 20230922204306289_0_665 9955 8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0 9955 a9952-1 [[9952], [3317], [9952, 8952, NULL]] {"k3":NULL, "k4":[NULL, 9], "k9952":[1, NULL, 9], "k2":[]} {2012-02-03 11:36:56.000000, [9961, 9952, NULL]} +20230922203209630 20230922203209630_0_9952 9954 8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet 9954 a9951 [[9951], [3317], [9951, 8951, NULL]] {"k9951":[1, NULL, 9], "k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-03 10:35:55.000000, [9960, 9951, NULL]} +20230922203209630 20230922203209630_0_9951 9953 8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet 9953 a9950 [NULL, [], [9950, 8950, NULL]] {"k9950":[1, NULL, 9], "k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-03 09:34:54.000000, [9959, NULL, NULL]} +20230922203209630 20230922203209630_0_996 998 8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet 998 a995 [[995], [], [995, -5, NULL]] {"k995":[1, NULL, 9], "k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-02 10:28:09.000000, [1004, 995, NULL]} +20230922203209630 20230922203209630_0_9950 9952 8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet 9952 a9949 [[9949], [3316], [9949, 8949, NULL]] {"k9949":[1, NULL, 9], "k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-03 08:33:53.000000, [9958, 9949, NULL]} +20230922203209630 20230922203209630_0_9949 9951 8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet 9951 a9948 [[9948], [3316], [9948, 8948, NULL]] {"k9948":[1, NULL, 9], "k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-03 07:32:52.000000, [9957, 9948, NULL]} +20230922203209630 20230922203209630_0_9948 9950 8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet 9950 a9947 [[9947], [3315], [9947, 8947, NULL]] {"k9947":[1, NULL, 9], "k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-03 06:31:51.000000, [9956, 9947, NULL]} +20230922203209630 20230922203209630_0_9947 9949 8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet 9949 a9946 [[9946], [3315], [9946, 8946, NULL]] {"k9946":[1, NULL, 9], "k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-03 05:30:50.000000, [9955, 9946, NULL]} +20230922203209630 20230922203209630_0_9946 9948 8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet 9948 a9945 [[9945], [], [9945, 8945, NULL]] {"k9945":[1, NULL, 9], "k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-03 04:29:49.000000, [9954, 9945, NULL]} +20230922203209630 20230922203209630_0_9945 9947 8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet 9947 a9944 [[9944], [3314], [9944, 8944, NULL]] {"k9944":[1, NULL, 9], "k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-03 03:28:48.000000, [9953, 9944, NULL]} +20230922203209630 20230922203209630_0_9944 9946 8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet 9946 a9943 [[9943], [3314], [9943, 8943, NULL]] {"k9943":[1, NULL, 9], "k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-03 02:27:47.000000, [9952, 9943, NULL]} +20230922204306289 20230922204306289_0_53 9945 8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0 9945 a9942-1 [[9942], [3314], [9942, 8942, NULL]] {"k3":NULL, "k4":[NULL, 9], "k9942":[1, NULL, 9], "k2":[]} {2012-02-03 01:26:46.000000, [9951, 9942, NULL]} +20230922203209630 20230922203209630_0_9942 9944 8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet 9944 a9941 [[9941], [3313], [9941, 8941, NULL]] {"k9941":[1, NULL, 9], "k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-03 00:25:45.000000, [9950, 9941, NULL]} +20230922203209630 20230922203209630_0_9941 9943 8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet 9943 a9940 [NULL, [], [9940, 8940, NULL]] {"k9940":[1, NULL, 9], "k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-02 23:24:44.000000, [9949, NULL, NULL]} +20230922203209630 20230922203209630_0_995 997 8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet 997 a994 [[994], [331], [994, -6, NULL]] {"k994":[1, NULL, 9], "k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-02 09:27:08.000000, [1003, 994, NULL]} +20230922203209630 20230922203209630_0_9940 9942 8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet 9942 a9939 [[9939], [3313], [9939, 8939, NULL]] {"k9939":[1, NULL, 9], "k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-02 22:23:43.000000, [9948, 9939, NULL]} +20230922203209630 20230922203209630_0_9939 9941 8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet 9941 a9938 [[9938], [3312], [9938, 8938, NULL]] {"k9938":[1, NULL, 9], "k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-02 21:22:42.000000, [9947, 9938, NULL]} +20230922203209630 20230922203209630_0_9938 9940 8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet 9940 a9937 [[9937], [3312], [9937, 8937, NULL]] {"k9937":[1, NULL, 9], "k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-02 20:21:41.000000, [9946, 9937, NULL]} +20230922203209630 20230922203209630_0_9937 9939 8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet 9939 a9936 [[9936], [3312], [9936, 8936, NULL]] {"k9936":[1, NULL, 9], "k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-02 19:20:40.000000, [9945, 9936, NULL]} +20230922203209630 20230922203209630_0_9936 9938 8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet 9938 a9935 [[9935], [], [9935, 8935, NULL]] {"k9935":[1, NULL, 9], "k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-02 18:19:39.000000, [9944, 9935, NULL]} +20230922203209630 20230922203209630_0_9935 9937 8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet 9937 a9934 [[9934], [3311], [9934, 8934, NULL]] {"k9934":[1, NULL, 9], "k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-02 17:18:38.000000, [9943, 9934, NULL]} +20230922203209630 20230922203209630_0_9934 9936 8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet 9936 a9933 [[9933], [3311], [9933, 8933, NULL]] {"k9933":[1, NULL, 9], "k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-02 16:17:37.000000, [9942, 9933, NULL]} +20230922204306289 20230922204306289_0_991 9935 8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0 9935 a9932-1 [[9932], [3310], [9932, 8932, NULL]] {"k3":NULL, "k4":[NULL, 9], "k9932":[1, NULL, 9], "k2":[]} {2012-02-02 15:16:36.000000, [9941, 9932, NULL]} +20230922203209630 20230922203209630_0_9932 9934 8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet 9934 a9931 [[9931], [3310], [9931, 8931, NULL]] {"k9931":[1, NULL, 9], "k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-02 14:15:35.000000, [9940, 9931, NULL]} +20230922203209630 20230922203209630_0_9931 9933 8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet 9933 a9930 [NULL, [], [9930, 8930, NULL]] {"k9930":[1, NULL, 9], "k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-02 13:14:34.000000, [9939, NULL, NULL]} +20230922203209630 20230922203209630_0_994 996 8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet 996 a993 [[993], [331], [993, -7, NULL]] {"k993":[1, NULL, 9], "k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-02 08:26:07.000000, [1002, 993, NULL]} +20230922203209630 20230922203209630_0_9930 9932 8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet 9932 a9929 [[9929], [3309], [9929, 8929, NULL]] {"k9929":[1, NULL, 9], "k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-02 12:13:33.000000, [9938, 9929, NULL]} +20230922203209630 20230922203209630_0_9929 9931 8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet 9931 a9928 [[9928], [3309], [9928, 8928, NULL]] {"k9928":[1, NULL, 9], "k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-02 11:12:32.000000, [9937, 9928, NULL]} +20230922203209630 20230922203209630_0_9928 9930 8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet 9930 a9927 [[9927], [3309], [9927, 8927, NULL]] {"k9927":[1, NULL, 9], "k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-02 10:11:31.000000, [9936, 9927, NULL]} +20230922203209630 20230922203209630_0_9927 9929 8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet 9929 a9926 [[9926], [3308], [9926, 8926, NULL]] {"k9926":[1, NULL, 9], "k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-02 09:10:30.000000, [9935, 9926, NULL]} +20230922203209630 20230922203209630_0_9926 9928 8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet 9928 a9925 [[9925], [], [9925, 8925, NULL]] {"k9925":[1, NULL, 9], "k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-02 08:09:29.000000, [9934, 9925, NULL]} +20230922203209630 20230922203209630_0_9925 9927 8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet 9927 a9924 [[9924], [3308], [9924, 8924, NULL]] {"k9924":[1, NULL, 9], "k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-02 07:08:28.000000, [9933, 9924, NULL]} +20230922203209630 20230922203209630_0_9924 9926 8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet 9926 a9923 [[9923], [3307], [9923, 8923, NULL]] {"k9923":[1, NULL, 9], "k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-02 06:07:27.000000, [9932, 9923, NULL]} +20230922204306289 20230922204306289_0_778 9925 8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0 9925 a9922-1 [[9922], [3307], [9922, 8922, NULL]] {"k3":NULL, "k4":[NULL, 9], "k9922":[1, NULL, 9], "k2":[]} {2012-02-02 05:06:26.000000, [9931, 9922, NULL]} +20230922203209630 20230922203209630_0_9922 9924 8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet 9924 a9921 [[9921], [3307], [9921, 8921, NULL]] {"k9921":[1, NULL, 9], "k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-02 04:05:25.000000, [9930, 9921, NULL]} +20230922203209630 20230922203209630_0_9921 9923 8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet 9923 a9920 [NULL, [], [9920, 8920, NULL]] {"k9920":[1, NULL, 9], "k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-02 03:04:24.000000, [9929, NULL, NULL]} +20230922204306289 20230922204306289_0_100 995 8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0 995 a992-1 [[992], [330], [992, -8, NULL]] {"k3":NULL, "k4":[NULL, 9], "k992":[1, NULL, 9], "k2":[]} {2012-02-02 07:25:06.000000, [1001, 992, NULL]} +20230922203209630 20230922203209630_0_9920 9922 8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet 9922 a9919 [[9919], [3306], [9919, 8919, NULL]] {"k9919":[1, NULL, 9], "k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-02 02:03:23.000000, [9928, 9919, NULL]} +20230922203209630 20230922203209630_0_9919 9921 8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet 9921 a9918 [[9918], [3306], [9918, 8918, NULL]] {"k9918":[1, NULL, 9], "k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-02 01:02:22.000000, [9927, 9918, NULL]} +20230922203209630 20230922203209630_0_9918 9920 8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet 9920 a9917 [[9917], [3305], [9917, 8917, NULL]] {"k9917":[1, NULL, 9], "k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-04 10:58:21.000000, [9926, 9917, NULL]} +20230922203209630 20230922203209630_0_9917 9919 8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet 9919 a9916 [[9916], [3305], [9916, 8916, NULL]] {"k9916":[1, NULL, 9], "k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-04 09:57:20.000000, [9925, 9916, NULL]} +20230922203209630 20230922203209630_0_9916 9918 8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet 9918 a9915 [[9915], [], [9915, 8915, NULL]] {"k9915":[1, NULL, 9], "k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-04 08:56:19.000000, [9924, 9915, NULL]} +20230922203209630 20230922203209630_0_9915 9917 8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet 9917 a9914 [[9914], [3304], [9914, 8914, NULL]] {"k9914":[1, NULL, 9], "k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-04 07:55:18.000000, [9923, 9914, NULL]} +20230922203209630 20230922203209630_0_9914 9916 8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet 9916 a9913 [[9913], [3304], [9913, 8913, NULL]] {"k9913":[1, NULL, 9], "k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-04 06:54:17.000000, [9922, 9913, NULL]} +20230922204306289 20230922204306289_0_750 9915 8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0 9915 a9912-1 [[9912], [3304], [9912, 8912, NULL]] {"k3":NULL, "k4":[NULL, 9], "k9912":[1, NULL, 9], "k2":[]} {2012-02-04 05:53:16.000000, [9921, 9912, NULL]} +20230922203209630 20230922203209630_0_9912 9914 8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet 9914 a9911 [[9911], [3303], [9911, 8911, NULL]] {"k9911":[1, NULL, 9], "k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-04 04:52:15.000000, [9920, 9911, NULL]} +20230922203209630 20230922203209630_0_9911 9913 8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet 9913 a9910 [NULL, [], [9910, 8910, NULL]] {"k9910":[1, NULL, 9], "k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-04 03:51:14.000000, [9919, NULL, NULL]} +20230922203209630 20230922203209630_0_992 994 8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet 994 a991 [[991], [330], [991, -9, NULL]] {"k991":[1, NULL, 9], "k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-02 06:24:05.000000, [1000, 991, NULL]} +20230922203209630 20230922203209630_0_9910 9912 8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet 9912 a9909 [[9909], [3303], [9909, 8909, NULL]] {"k9909":[1, NULL, 9], "k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-04 02:50:13.000000, [9918, 9909, NULL]} + -- !skip_merge -- 20230605145009209 20230605145009209_0_0 rowId:row_1 partitionId=2021-01-01/versionId=v_0 65ffc5d9-397a-456e-a735-30f3ad37466f-0_0-33-96_20230605145009209.parquet row_1 2021-01-01 0 bob v_0 toBeDel0 0 1000000 20230605145403388 20230605145403388_2_0 rowId:row_1 partitionId=2011-11-11/versionId=v_1 dbff8acb-42bc-400c-be33-47d9e0bae9b7-0_2-83-222_20230605145403388.parquet row_1 2011-11-11 1 bob v_1 toBeDel1 0 1000001 diff --git a/regression-test/suites/external_table_p2/hive/test_hive_hudi.groovy b/regression-test/suites/external_table_p2/hive/test_hive_hudi.groovy index abdd5b34dcbca5..8648aa4bc98f1b 100644 --- a/regression-test/suites/external_table_p2/hive/test_hive_hudi.groovy +++ b/regression-test/suites/external_table_p2/hive/test_hive_hudi.groovy @@ -39,6 +39,8 @@ suite("test_hive_hudi", "p2,external,hive,hudi") { // match colum name in lower case qt_lowercase_column """select RoWiD, PaRtiTionID, PrEComB, VerSIonID from partitioned_mor_rt order by rowid, versionid""" + // test complex types + qt_complex_types """select * from complex_type_rt order by name desc limit 100""" // skip logs sql """drop catalog if exists ${catalog_name};"""