Skip to content

Commit

Permalink
fixed
Browse files Browse the repository at this point in the history
  • Loading branch information
amorynan committed Dec 2, 2024
1 parent 9bcd1ba commit f599702
Showing 1 changed file with 13 additions and 14 deletions.
27 changes: 13 additions & 14 deletions be/test/vec/data_types/common_data_type_serder_test.h
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#include <arrow/record_batch.h>
#include <gen_cpp/data.pb.h>
#include <gtest/gtest-message.h>
#include <gtest/gtest-test-part.h>
Expand All @@ -27,7 +28,6 @@
#include "runtime/descriptors.cpp"
#include "runtime/descriptors.h"
#include "util/arrow/block_convertor.h"
#include <arrow/record_batch.h>
#include "util/arrow/row_batch.h"
#include "vec/columns/column.h"
#include "vec/columns/column_array.h"
Expand Down Expand Up @@ -176,16 +176,14 @@ class CommonDataTypeSerdeTest : public ::testing::Test {
columns[c_idx]->insert_default();
std::cout << "error in deserialize but continue: " << st.to_string()
<< std::endl;

}
// serialize data
size_t row_num = columns[c_idx]->size() - 1;
assert_str_cols[c_idx]->reserve(columns[c_idx]->size());
VectorBufferWriter bw(assert_cast<ColumnString&>(*assert_str_cols[c_idx]));
if constexpr (is_hive_format) {
st = serders[c_idx]->serialize_one_cell_to_hive_text(*columns[c_idx],
row_num, bw,
options);
row_num, bw, options);
EXPECT_TRUE(st.ok()) << st.to_string();
} else {
st = serders[c_idx]->serialize_one_cell_to_json(*columns[c_idx], row_num,
Expand All @@ -209,12 +207,11 @@ class CommonDataTypeSerdeTest : public ::testing::Test {
}
}


if (generate_res_file) {
// generate res
auto pos = file_path.find_last_of(".");
string hive_format = is_hive_format ? "_hive" : "";
std::string res_file = file_path.substr(0, pos) + hive_format +"_serde_res.csv";
std::string res_file = file_path.substr(0, pos) + hive_format + "_serde_res.csv";
std::ofstream res_f(res_file);
if (!res_f.is_open()) {
throw std::ios_base::failure("Failed to open file." + res_file);
Expand Down Expand Up @@ -313,14 +310,16 @@ class CommonDataTypeSerdeTest : public ::testing::Test {
auto& col = load_cols[i];
for (size_t j = 0; j < col->size(); ++j) {
Status st;
EXPECT_NO_FATAL_FAILURE(st = serders[i]->write_column_to_mysql(*col, row_buffer, j, false, {}));
EXPECT_NO_FATAL_FAILURE(
st = serders[i]->write_column_to_mysql(*col, row_buffer, j, false, {}));
EXPECT_TRUE(st.ok()) << st.to_string();
}
}
}

// assert arrow serialize
static void assert_arrow_format(MutableColumns& load_cols, DataTypeSerDeSPtrs serders, DataTypes types) {
static void assert_arrow_format(MutableColumns& load_cols, DataTypeSerDeSPtrs serders,
DataTypes types) {
// make a block to write to arrow
auto block = std::make_shared<Block>();
for (size_t i = 0; i < load_cols.size(); ++i) {
Expand All @@ -333,8 +332,9 @@ class CommonDataTypeSerdeTest : public ::testing::Test {
EXPECT_EQ(get_arrow_schema_from_block(*block, &block_arrow_schema, "UTC"), Status::OK());
// convert block to arrow
std::shared_ptr<arrow::RecordBatch> result;
cctz::time_zone _timezone_obj;//default UTC
Status stt = convert_to_arrow_batch(*block, block_arrow_schema, arrow::default_memory_pool(), &result, _timezone_obj);
cctz::time_zone _timezone_obj; //default UTC
Status stt = convert_to_arrow_batch(*block, block_arrow_schema,
arrow::default_memory_pool(), &result, _timezone_obj);
EXPECT_EQ(Status::OK(), stt) << "convert block to arrow failed" << stt.to_string();

// deserialize arrow to block
Expand All @@ -343,10 +343,9 @@ class CommonDataTypeSerdeTest : public ::testing::Test {
for (size_t i = 0; i < load_cols.size(); ++i) {
auto array = result->column(i);
auto& column_with_type_and_name = assert_block.get_by_position(i);
auto ret = arrow_column_to_doris_column(array.get(), 0,
column_with_type_and_name.column,
column_with_type_and_name.type,
rows, _timezone_obj);
auto ret = arrow_column_to_doris_column(
array.get(), 0, column_with_type_and_name.column,
column_with_type_and_name.type, rows, _timezone_obj);
// do check data
EXPECT_EQ(Status::OK(), ret) << "convert arrow to block failed" << ret.to_string();
auto& col = block->get_by_position(i).column;
Expand Down

0 comments on commit f599702

Please sign in to comment.