diff --git a/.licenserc.yaml b/.licenserc.yaml index 066f8a908137426..739c14d4c1b9b94 100644 --- a/.licenserc.yaml +++ b/.licenserc.yaml @@ -40,6 +40,7 @@ header: - "**/*.sql" - "**/*.lock" - "**/*.out" + - "**/*.parquet" - "docs/.markdownlintignore" - "fe/fe-core/src/test/resources/data/net_snmp_normal" - "fe/fe-core/src/main/antlr4/org/apache/doris/nereids/JavaLexer.g4" diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp index 17bca94a459cefa..523df70a2ad9f3b 100644 --- a/be/src/common/config.cpp +++ b/be/src/common/config.cpp @@ -1253,6 +1253,8 @@ DEFINE_mInt64(fetch_remote_schema_rpc_timeout_ms, "60000"); // filter wrong data. DEFINE_mBool(enable_parquet_page_index, "true"); +DEFINE_mBool(ignore_not_found_file_in_external_table, "true"); + // clang-format off #ifdef BE_TEST // test s3 diff --git a/be/src/common/config.h b/be/src/common/config.h index 5fa5b23e977a8b3..8de25205b625d4e 100644 --- a/be/src/common/config.h +++ b/be/src/common/config.h @@ -1339,6 +1339,10 @@ DECLARE_Int64(min_row_group_size); DECLARE_mBool(enable_parquet_page_index); +// Wheather to ignore not found file in external teble(eg, hive) +// Default is true, if set to false, the not found file will result in query failure. +DECLARE_mBool(ignore_not_found_file_in_external_table); + #ifdef BE_TEST // test s3 DECLARE_String(test_s3_resource); diff --git a/be/src/exec/es/es_scan_reader.cpp b/be/src/exec/es/es_scan_reader.cpp index 183719fc7c5d64a..7aeed98cd365ac7 100644 --- a/be/src/exec/es/es_scan_reader.cpp +++ b/be/src/exec/es/es_scan_reader.cpp @@ -74,19 +74,32 @@ ESScanReader::ESScanReader(const std::string& target, std::string filter_path = _doc_value_mode ? DOCVALUE_SCROLL_SEARCH_FILTER_PATH : SOURCE_SCROLL_SEARCH_FILTER_PATH; + // When shard_id is negative(-1), the request will be sent to ES without shard preference. + int32 shard_id = std::stoi(_shards); if (props.find(KEY_TERMINATE_AFTER) != props.end()) { _exactly_once = true; std::stringstream scratch; // just send a normal search against the elasticsearch with additional terminate_after param to achieve terminate early effect when limit take effect if (_type.empty()) { - // `terminate_after` and `size` can not be used together in scroll request of ES 8.x - scratch << _target << REQUEST_SEPARATOR << _index << "/_search?" - << REQUEST_PREFERENCE_PREFIX << _shards << "&" << filter_path; + if (shard_id < 0) { + scratch << _target << REQUEST_SEPARATOR << _index << "/_search?" << filter_path; + } else { + // `terminate_after` and `size` can not be used together in scroll request of ES 8.x + scratch << _target << REQUEST_SEPARATOR << _index << "/_search?" + << REQUEST_PREFERENCE_PREFIX << _shards << "&" << filter_path; + } } else { - scratch << _target << REQUEST_SEPARATOR << _index << REQUEST_SEPARATOR << _type - << "/_search?" - << "terminate_after=" << props.at(KEY_TERMINATE_AFTER) - << REQUEST_PREFERENCE_PREFIX << _shards << "&" << filter_path; + if (shard_id < 0) { + scratch << _target << REQUEST_SEPARATOR << _index << REQUEST_SEPARATOR << _type + << "/_search?" + << "terminate_after=" << props.at(KEY_TERMINATE_AFTER) << "&" + << filter_path; + } else { + scratch << _target << REQUEST_SEPARATOR << _index << REQUEST_SEPARATOR << _type + << "/_search?" + << "terminate_after=" << props.at(KEY_TERMINATE_AFTER) + << REQUEST_PREFERENCE_PREFIX << _shards << "&" << filter_path; + } } _search_url = scratch.str(); } else { @@ -95,15 +108,27 @@ ESScanReader::ESScanReader(const std::string& target, // scroll request for scanning // add terminate_after for the first scroll to avoid decompress all postings list if (_type.empty()) { - // `terminate_after` and `size` can not be used together in scroll request of ES 8.x - scratch << _target << REQUEST_SEPARATOR << _index << "/_search?" - << "scroll=" << _scroll_keep_alive << REQUEST_PREFERENCE_PREFIX << _shards - << "&" << filter_path; + if (shard_id < 0) { + scratch << _target << REQUEST_SEPARATOR << _index << "/_search?" + << "scroll=" << _scroll_keep_alive << "&" << filter_path; + } else { + // `terminate_after` and `size` can not be used together in scroll request of ES 8.x + scratch << _target << REQUEST_SEPARATOR << _index << "/_search?" + << "scroll=" << _scroll_keep_alive << REQUEST_PREFERENCE_PREFIX << _shards + << "&" << filter_path; + } } else { - scratch << _target << REQUEST_SEPARATOR << _index << REQUEST_SEPARATOR << _type - << "/_search?" - << "scroll=" << _scroll_keep_alive << REQUEST_PREFERENCE_PREFIX << _shards - << "&" << filter_path << "&terminate_after=" << batch_size_str; + if (shard_id < 0) { + scratch << _target << REQUEST_SEPARATOR << _index << REQUEST_SEPARATOR << _type + << "/_search?" + << "scroll=" << _scroll_keep_alive << "&" << filter_path + << "&terminate_after=" << batch_size_str; + } else { + scratch << _target << REQUEST_SEPARATOR << _index << REQUEST_SEPARATOR << _type + << "/_search?" + << "scroll=" << _scroll_keep_alive << REQUEST_PREFERENCE_PREFIX << _shards + << "&" << filter_path << "&terminate_after=" << batch_size_str; + } } _init_scroll_url = scratch.str(); _next_scroll_url = _target + REQUEST_SEARCH_SCROLL_PATH + "?" + filter_path; @@ -115,11 +140,13 @@ ESScanReader::~ESScanReader() {} Status ESScanReader::open() { _is_first = true; + // we do not enable set_fail_on_error for ES http request to get more detail error messages + bool set_fail_on_error = false; if (_exactly_once) { - RETURN_IF_ERROR(_network_client.init(_search_url)); + RETURN_IF_ERROR(_network_client.init(_search_url, set_fail_on_error)); LOG(INFO) << "search request URL: " << _search_url; } else { - RETURN_IF_ERROR(_network_client.init(_init_scroll_url)); + RETURN_IF_ERROR(_network_client.init(_init_scroll_url, set_fail_on_error)); LOG(INFO) << "First scroll request URL: " << _init_scroll_url; } _network_client.set_basic_auth(_user_name, _passwd); @@ -132,7 +159,8 @@ Status ESScanReader::open() { Status status = _network_client.execute_post_request(_query, &_cached_response); if (!status.ok() || _network_client.get_http_status() != 200) { std::stringstream ss; - ss << "Failed to connect to ES server, errmsg is: " << status; + ss << "Failed to connect to ES server, errmsg is: " << status + << ", response: " << _cached_response; LOG(WARNING) << ss.str(); return Status::InternalError(ss.str()); } @@ -155,7 +183,9 @@ Status ESScanReader::get_next(bool* scan_eos, std::unique_ptr& scr if (_exactly_once) { return Status::OK(); } - RETURN_IF_ERROR(_network_client.init(_next_scroll_url)); + // we do not enable set_fail_on_error for ES http request to get more detail error messages + bool set_fail_on_error = false; + RETURN_IF_ERROR(_network_client.init(_next_scroll_url, set_fail_on_error)); _network_client.set_basic_auth(_user_name, _passwd); _network_client.set_content_type("application/json"); _network_client.set_timeout_ms(_http_timeout_ms); @@ -168,13 +198,15 @@ Status ESScanReader::get_next(bool* scan_eos, std::unique_ptr& scr long status = _network_client.get_http_status(); if (status == 404) { LOG(WARNING) << "request scroll search failure 404[" - << ", response: " << (response.empty() ? "empty response" : response); + << ", response: " << (response.empty() ? "empty response" : response) + << "]"; return Status::InternalError("No search context found for {}", _scroll_id); } if (status != 200) { LOG(WARNING) << "request scroll search failure[" - << "http status" << status - << ", response: " << (response.empty() ? "empty response" : response); + << "http status: " << status + << ", response: " << (response.empty() ? "empty response" : response) + << "]"; return Status::InternalError("request scroll search failure: {}", (response.empty() ? "empty response" : response)); } @@ -211,7 +243,9 @@ Status ESScanReader::close() { } std::string scratch_target = _target + REQUEST_SEARCH_SCROLL_PATH; - RETURN_IF_ERROR(_network_client.init(scratch_target)); + // we do not enable set_fail_on_error for ES http request to get more detail error messages + bool set_fail_on_error = false; + RETURN_IF_ERROR(_network_client.init(scratch_target, set_fail_on_error)); _network_client.set_basic_auth(_user_name, _passwd); _network_client.set_method(DELETE); _network_client.set_content_type("application/json"); @@ -222,9 +256,13 @@ Status ESScanReader::close() { std::string response; RETURN_IF_ERROR(_network_client.execute_delete_request( ESScrollQueryBuilder::build_clear_scroll_body(_scroll_id), &response)); - if (_network_client.get_http_status() == 200) { + long status = _network_client.get_http_status(); + if (status == 200) { return Status::OK(); } else { + LOG(WARNING) << "es_scan_reader delete scroll context failure[" + << "http status: " << status + << ", response: " << (response.empty() ? "empty response" : response) << "]"; return Status::InternalError("es_scan_reader delete scroll context failure"); } } diff --git a/be/src/exec/es/es_scroll_query.cpp b/be/src/exec/es/es_scroll_query.cpp index e8d214fd9bf2197..fc89341239f9938 100644 --- a/be/src/exec/es/es_scroll_query.cpp +++ b/be/src/exec/es/es_scroll_query.cpp @@ -125,11 +125,19 @@ std::string ESScrollQueryBuilder::build(const std::map } else { size = atoi(properties.at(ESScanReader::KEY_BATCH_SIZE).c_str()); } - rapidjson::Value sort_node(rapidjson::kArrayType); - // use the scroll-scan mode for scan index documents - rapidjson::Value field("_doc", allocator); - sort_node.PushBack(field, allocator); - es_query_dsl.AddMember("sort", sort_node, allocator); + + std::string shard_id; + if (properties.find(ESScanReader::KEY_SHARD) != properties.end()) { + shard_id = properties.at(ESScanReader::KEY_SHARD); + } + // To maintain consistency with the query, when shard_id is negative, do not add sort node in scroll request body. + if (!shard_id.empty() && std::stoi(shard_id) >= 0) { + rapidjson::Value sort_node(rapidjson::kArrayType); + // use the scroll-scan mode for scan index documents + rapidjson::Value field("_doc", allocator); + sort_node.PushBack(field, allocator); + es_query_dsl.AddMember("sort", sort_node, allocator); + } // number of documents returned es_query_dsl.AddMember("size", size, allocator); rapidjson::StringBuffer buffer; diff --git a/be/src/http/http_client.cpp b/be/src/http/http_client.cpp index b8ef983434127eb..218802878bd2c04 100644 --- a/be/src/http/http_client.cpp +++ b/be/src/http/http_client.cpp @@ -66,7 +66,7 @@ HttpClient::~HttpClient() { } } -Status HttpClient::init(const std::string& url) { +Status HttpClient::init(const std::string& url, bool set_fail_on_error) { if (_curl == nullptr) { _curl = curl_easy_init(); if (_curl == nullptr) { @@ -94,10 +94,14 @@ Status HttpClient::init(const std::string& url) { return Status::InternalError("fail to set CURLOPT_NOSIGNAL"); } // set fail on error - code = curl_easy_setopt(_curl, CURLOPT_FAILONERROR, 1L); - if (code != CURLE_OK) { - LOG(WARNING) << "fail to set CURLOPT_FAILONERROR, msg=" << _to_errmsg(code); - return Status::InternalError("fail to set CURLOPT_FAILONERROR"); + // When this option is set to `1L` (enabled), libcurl will return an error directly + // when encountering HTTP error codes (>= 400), without reading the body of the error response. + if (set_fail_on_error) { + code = curl_easy_setopt(_curl, CURLOPT_FAILONERROR, 1L); + if (code != CURLE_OK) { + LOG(WARNING) << "fail to set CURLOPT_FAILONERROR, msg=" << _to_errmsg(code); + return Status::InternalError("fail to set CURLOPT_FAILONERROR"); + } } // set redirect code = curl_easy_setopt(_curl, CURLOPT_FOLLOWLOCATION, 1L); diff --git a/be/src/http/http_client.h b/be/src/http/http_client.h index e379895a73ef257..9659de13cfcac5a 100644 --- a/be/src/http/http_client.h +++ b/be/src/http/http_client.h @@ -45,7 +45,7 @@ class HttpClient { // this function must call before other function, // you can call this multiple times to reuse this object - Status init(const std::string& url); + Status init(const std::string& url, bool set_fail_on_error = true); void set_method(HttpMethod method); diff --git a/be/src/pipeline/exec/table_function_operator.cpp b/be/src/pipeline/exec/table_function_operator.cpp index b4d993ef035acb8..54b5646d1ab82c3 100644 --- a/be/src/pipeline/exec/table_function_operator.cpp +++ b/be/src/pipeline/exec/table_function_operator.cpp @@ -187,16 +187,14 @@ Status TableFunctionLocalState::get_expanded_block(RuntimeState* state, if (skip_child_row = _is_inner_and_empty(); skip_child_row) { continue; } - if (p._fn_num == 1) { - _current_row_insert_times += _fns[0]->get_value( - columns[p._child_slots.size()], - state->batch_size() - columns[p._child_slots.size()]->size()); - } else { - for (int i = 0; i < p._fn_num; i++) { - _fns[i]->get_value(columns[i + p._child_slots.size()]); - } - _current_row_insert_times++; - _fns[p._fn_num - 1]->forward(); + + DCHECK_LE(1, p._fn_num); + auto repeat_times = _fns[p._fn_num - 1]->get_value( + columns[p._child_slots.size() + p._fn_num - 1], + state->batch_size() - columns[p._child_slots.size()]->size()); + _current_row_insert_times += repeat_times; + for (int i = 0; i < p._fn_num - 1; i++) { + _fns[i]->get_same_many_values(columns[i + p._child_slots.size()], repeat_times); } } } diff --git a/be/src/vec/exec/scan/vfile_scanner.cpp b/be/src/vec/exec/scan/vfile_scanner.cpp index dcfb404ae5a5b36..944884434f4637f 100644 --- a/be/src/vec/exec/scan/vfile_scanner.cpp +++ b/be/src/vec/exec/scan/vfile_scanner.cpp @@ -162,6 +162,8 @@ Status VFileScanner::prepare( _convert_to_output_block_timer = ADD_TIMER(_parent->_scanner_profile, "FileScannerConvertOuputBlockTime"); _empty_file_counter = ADD_COUNTER(_parent->_scanner_profile, "EmptyFileNum", TUnit::UNIT); + _not_found_file_counter = + ADD_COUNTER(_parent->_scanner_profile, "NotFoundFileNum", TUnit::UNIT); _file_counter = ADD_COUNTER(_parent->_scanner_profile, "FileNumber", TUnit::UNIT); _has_fully_rf_file_counter = ADD_COUNTER(_parent->_scanner_profile, "HasFullyRfFileNumber", TUnit::UNIT); @@ -180,6 +182,8 @@ Status VFileScanner::prepare( ADD_TIMER(_local_state->scanner_profile(), "FileScannerConvertOuputBlockTime"); _empty_file_counter = ADD_COUNTER(_local_state->scanner_profile(), "EmptyFileNum", TUnit::UNIT); + _not_found_file_counter = + ADD_COUNTER(_local_state->scanner_profile(), "NotFoundFileNum", TUnit::UNIT); _file_counter = ADD_COUNTER(_local_state->scanner_profile(), "FileNumber", TUnit::UNIT); _has_fully_rf_file_counter = ADD_COUNTER(_local_state->scanner_profile(), "HasFullyRfFileNumber", TUnit::UNIT); @@ -329,9 +333,9 @@ Status VFileScanner::_get_block_wrapped(RuntimeState* state, Block* block, bool* // And the file may already be removed from storage. // Just ignore not found files. Status st = _get_next_reader(); - if (st.is()) { + if (st.is() && config::ignore_not_found_file_in_external_table) { _cur_reader_eof = true; - COUNTER_UPDATE(_empty_file_counter, 1); + COUNTER_UPDATE(_not_found_file_counter, 1); continue; } else if (!st) { return st; diff --git a/be/src/vec/exec/scan/vfile_scanner.h b/be/src/vec/exec/scan/vfile_scanner.h index 43c1a8b13da51a5..2c2116172c9816e 100644 --- a/be/src/vec/exec/scan/vfile_scanner.h +++ b/be/src/vec/exec/scan/vfile_scanner.h @@ -182,6 +182,7 @@ class VFileScanner : public VScanner { RuntimeProfile::Counter* _pre_filter_timer = nullptr; RuntimeProfile::Counter* _convert_to_output_block_timer = nullptr; RuntimeProfile::Counter* _empty_file_counter = nullptr; + RuntimeProfile::Counter* _not_found_file_counter = nullptr; RuntimeProfile::Counter* _file_counter = nullptr; RuntimeProfile::Counter* _has_fully_rf_file_counter = nullptr; diff --git a/be/src/vec/exec/vtable_function_node.cpp b/be/src/vec/exec/vtable_function_node.cpp index 0c35fae806ea934..e4476cd25564ab5 100644 --- a/be/src/vec/exec/vtable_function_node.cpp +++ b/be/src/vec/exec/vtable_function_node.cpp @@ -197,16 +197,14 @@ Status VTableFunctionNode::_get_expanded_block(RuntimeState* state, Block* outpu if (skip_child_row = _is_inner_and_empty(); skip_child_row) { continue; } - if (_fn_num == 1) { - _current_row_insert_times += _fns[0]->get_value( - columns[_child_slots.size()], - state->batch_size() - columns[_child_slots.size()]->size()); - } else { - for (int i = 0; i < _fn_num; i++) { - _fns[i]->get_value(columns[i + _child_slots.size()]); - } - _current_row_insert_times++; - _fns[_fn_num - 1]->forward(); + + DCHECK_LE(1, _fn_num); + auto repeat_times = _fns[_fn_num - 1]->get_value( + columns[_child_slots.size()], + state->batch_size() - columns[_child_slots.size()]->size()); + _current_row_insert_times += repeat_times; + for (int i = 0; i < _fn_num - 1; i++) { + _fns[i]->get_same_many_values(columns[i + _child_slots.size()], repeat_times); } } } diff --git a/be/src/vec/exprs/table_function/table_function.h b/be/src/vec/exprs/table_function/table_function.h index 98d811364399f58..bd926c365704fdc 100644 --- a/be/src/vec/exprs/table_function/table_function.h +++ b/be/src/vec/exprs/table_function/table_function.h @@ -53,17 +53,8 @@ class TableFunction { _cur_offset = 0; } - virtual void get_value(MutableColumnPtr& column) = 0; - - virtual int get_value(MutableColumnPtr& column, int max_step) { - max_step = std::max(1, std::min(max_step, (int)(_cur_size - _cur_offset))); - int i = 0; - for (; i < max_step && !eos(); i++) { - get_value(column); - forward(); - } - return i; - } + virtual void get_same_many_values(MutableColumnPtr& column, int length = 0) = 0; + virtual int get_value(MutableColumnPtr& column, int max_step) = 0; virtual Status close() { return Status::OK(); } diff --git a/be/src/vec/exprs/table_function/table_function_factory.cpp b/be/src/vec/exprs/table_function/table_function_factory.cpp index e42c0a27fd11115..9e3a532b528baf3 100644 --- a/be/src/vec/exprs/table_function/table_function_factory.cpp +++ b/be/src/vec/exprs/table_function/table_function_factory.cpp @@ -17,6 +17,10 @@ #include "vec/exprs/table_function/table_function_factory.h" +#include + +#include +#include #include #include "common/object_pool.h" @@ -36,31 +40,21 @@ struct TableFunctionCreator { std::unique_ptr operator()() { return TableFunctionType::create_unique(); } }; -template <> -struct TableFunctionCreator { - ExplodeJsonArrayType type; - std::unique_ptr operator()() const { - return VExplodeJsonArrayTableFunction::create_unique(type); +template +struct VExplodeJsonArrayCreator { + std::unique_ptr operator()() { + return VExplodeJsonArrayTableFunction::create_unique(); } }; -inline auto VExplodeJsonArrayIntCreator = - TableFunctionCreator {ExplodeJsonArrayType::INT}; -inline auto VExplodeJsonArrayDoubleCreator = - TableFunctionCreator {ExplodeJsonArrayType::DOUBLE}; -inline auto VExplodeJsonArrayStringCreator = - TableFunctionCreator {ExplodeJsonArrayType::STRING}; -inline auto VExplodeJsonArrayJsonCreator = - TableFunctionCreator {ExplodeJsonArrayType::JSON}; - const std::unordered_map()>> TableFunctionFactory::_function_map { {"explode_split", TableFunctionCreator()}, {"explode_numbers", TableFunctionCreator()}, - {"explode_json_array_int", VExplodeJsonArrayIntCreator}, - {"explode_json_array_double", VExplodeJsonArrayDoubleCreator}, - {"explode_json_array_string", VExplodeJsonArrayStringCreator}, - {"explode_json_array_json", VExplodeJsonArrayJsonCreator}, + {"explode_json_array_int", VExplodeJsonArrayCreator()}, + {"explode_json_array_double", VExplodeJsonArrayCreator()}, + {"explode_json_array_string", VExplodeJsonArrayCreator()}, + {"explode_json_array_json", VExplodeJsonArrayCreator()}, {"explode_bitmap", TableFunctionCreator()}, {"explode_map", TableFunctionCreator {}}, {"explode", TableFunctionCreator {}}}; diff --git a/be/src/vec/exprs/table_function/vexplode.cpp b/be/src/vec/exprs/table_function/vexplode.cpp index b505339ea7c9151..cfd1260c29cd9bd 100644 --- a/be/src/vec/exprs/table_function/vexplode.cpp +++ b/be/src/vec/exprs/table_function/vexplode.cpp @@ -47,7 +47,6 @@ Status VExplodeTableFunction::process_init(Block* block, RuntimeState* state) { _array_column = block->get_by_position(value_column_idx).column->convert_to_full_column_if_const(); - if (!extract_column_array_info(*_array_column, _detail)) { return Status::NotSupported("column type {} not supported now", block->get_by_position(value_column_idx).column->get_name()); @@ -72,22 +71,46 @@ void VExplodeTableFunction::process_close() { _array_offset = 0; } -void VExplodeTableFunction::get_value(MutableColumnPtr& column) { +void VExplodeTableFunction::get_same_many_values(MutableColumnPtr& column, int length) { size_t pos = _array_offset + _cur_offset; if (current_empty() || (_detail.nested_nullmap_data && _detail.nested_nullmap_data[pos])) { - column->insert_default(); + column->insert_many_defaults(length); } else { if (_is_nullable) { assert_cast(column.get()) ->get_nested_column_ptr() - ->insert_from(*_detail.nested_col, pos); + ->insert_many_from(*_detail.nested_col, pos, length); assert_cast( assert_cast(column.get())->get_null_map_column_ptr().get()) - ->insert_default(); + ->insert_many_defaults(length); } else { - column->insert_from(*_detail.nested_col, pos); + column->insert_many_from(*_detail.nested_col, pos, length); } } } +int VExplodeTableFunction::get_value(MutableColumnPtr& column, int max_step) { + max_step = std::min(max_step, (int)(_cur_size - _cur_offset)); + size_t pos = _array_offset + _cur_offset; + if (current_empty()) { + column->insert_default(); + max_step = 1; + } else { + if (_is_nullable) { + auto* nullable_column = assert_cast(column.get()); + auto nested_column = nullable_column->get_nested_column_ptr(); + auto* nullmap_column = + assert_cast(nullable_column->get_null_map_column_ptr().get()); + nested_column->insert_range_from(*_detail.nested_col, pos, max_step); + size_t old_size = nullmap_column->size(); + nullmap_column->resize(old_size + max_step); + memcpy(nullmap_column->get_data().data() + old_size, + _detail.nested_nullmap_data + pos * sizeof(UInt8), max_step * sizeof(UInt8)); + } else { + column->insert_range_from(*_detail.nested_col, pos, max_step); + } + } + forward(max_step); + return max_step; +} } // namespace doris::vectorized diff --git a/be/src/vec/exprs/table_function/vexplode.h b/be/src/vec/exprs/table_function/vexplode.h index d5d84a70e2edc42..b59b9718ad54d11 100644 --- a/be/src/vec/exprs/table_function/vexplode.h +++ b/be/src/vec/exprs/table_function/vexplode.h @@ -43,7 +43,8 @@ class VExplodeTableFunction : public TableFunction { Status process_init(Block* block, RuntimeState* state) override; void process_row(size_t row_idx) override; void process_close() override; - void get_value(MutableColumnPtr& column) override; + void get_same_many_values(MutableColumnPtr& column, int length) override; + int get_value(MutableColumnPtr& column, int max_step) override; private: ColumnPtr _array_column; diff --git a/be/src/vec/exprs/table_function/vexplode_bitmap.cpp b/be/src/vec/exprs/table_function/vexplode_bitmap.cpp index 0bd9e43c32ebd0c..adede6cd6b0d7e9 100644 --- a/be/src/vec/exprs/table_function/vexplode_bitmap.cpp +++ b/be/src/vec/exprs/table_function/vexplode_bitmap.cpp @@ -71,19 +71,19 @@ void VExplodeBitmapTableFunction::forward(int step) { TableFunction::forward(step); } -void VExplodeBitmapTableFunction::get_value(MutableColumnPtr& column) { +void VExplodeBitmapTableFunction::get_same_many_values(MutableColumnPtr& column, int length) { if (current_empty()) { - column->insert_default(); + column->insert_many_defaults(length); } else { if (_is_nullable) { assert_cast( assert_cast(column.get())->get_nested_column_ptr().get()) - ->insert_value(**_cur_iter); + ->fill(**_cur_iter, length); assert_cast( assert_cast(column.get())->get_null_map_column_ptr().get()) - ->insert_default(); + ->insert_many_defaults(length); } else { - assert_cast(column.get())->insert_value(**_cur_iter); + assert_cast(column.get())->fill(**_cur_iter, length); } } } diff --git a/be/src/vec/exprs/table_function/vexplode_bitmap.h b/be/src/vec/exprs/table_function/vexplode_bitmap.h index 695b60d6093f0a2..17c1070646bb87c 100644 --- a/be/src/vec/exprs/table_function/vexplode_bitmap.h +++ b/be/src/vec/exprs/table_function/vexplode_bitmap.h @@ -42,7 +42,7 @@ class VExplodeBitmapTableFunction final : public TableFunction { ~VExplodeBitmapTableFunction() override = default; void reset() override; - void get_value(MutableColumnPtr& column) override; + void get_same_many_values(MutableColumnPtr& column, int length) override; int get_value(MutableColumnPtr& column, int max_step) override; void forward(int step = 1) override; diff --git a/be/src/vec/exprs/table_function/vexplode_json_array.cpp b/be/src/vec/exprs/table_function/vexplode_json_array.cpp index fbff29390e942d9..00c4d92a359e4f6 100644 --- a/be/src/vec/exprs/table_function/vexplode_json_array.cpp +++ b/be/src/vec/exprs/table_function/vexplode_json_array.cpp @@ -17,6 +17,7 @@ #include "vec/exprs/table_function/vexplode_json_array.h" +#include #include #include #include @@ -25,165 +26,23 @@ #include #include "common/status.h" -#include "rapidjson/stringbuffer.h" -#include "rapidjson/writer.h" #include "vec/columns/column.h" +#include "vec/columns/column_nullable.h" +#include "vec/columns/columns_number.h" #include "vec/core/block.h" #include "vec/core/column_with_type_and_name.h" +#include "vec/core/types.h" #include "vec/exprs/vexpr.h" #include "vec/exprs/vexpr_context.h" namespace doris::vectorized { - -std::string ParsedData::true_value = "true"; -std::string ParsedData::false_value = "false"; -auto max_value = std::numeric_limits::max(); //9223372036854775807 -auto min_value = std::numeric_limits::min(); //-9223372036854775808 - -int ParsedData::set_output(ExplodeJsonArrayType type, rapidjson::Document& document) { - int size = document.GetArray().Size(); - switch (type) { - case ExplodeJsonArrayType::INT: { - _data.resize(size); - _backup_int.resize(size); - int i = 0; - for (auto& v : document.GetArray()) { - if (v.IsInt64()) { - _backup_int[i] = v.GetInt64(); - _data[i] = &_backup_int[i]; - } else if (v.IsUint64()) { - auto value = v.GetUint64(); - if (value > max_value) { - _backup_int[i] = max_value; - } else { - _backup_int[i] = value; - } - _data[i] = &_backup_int[i]; - } else if (v.IsDouble()) { - auto value = v.GetDouble(); - if (value > max_value) { - _backup_int[i] = max_value; - } else if (value < min_value) { - _backup_int[i] = min_value; - } else { - _backup_int[i] = long(value); - } - _data[i] = &_backup_int[i]; - } else { - _data[i] = nullptr; - } - ++i; - } - break; - } - case ExplodeJsonArrayType::DOUBLE: { - _data.resize(size); - _backup_double.resize(size); - int i = 0; - for (auto& v : document.GetArray()) { - if (v.IsDouble()) { - _backup_double[i] = v.GetDouble(); - _data[i] = &_backup_double[i]; - } else { - _data[i] = nullptr; - } - ++i; - } - break; - } - case ExplodeJsonArrayType::STRING: { - _data_string.clear(); - _backup_string.clear(); - _string_nulls.clear(); - int32_t wbytes = 0; - for (auto& v : document.GetArray()) { - switch (v.GetType()) { - case rapidjson::Type::kStringType: - _backup_string.emplace_back(v.GetString(), v.GetStringLength()); - _string_nulls.push_back(false); - // do not set _data_string here. - // Because the address of the string stored in `_backup_string` may - // change each time `emplace_back()` is called. - break; - case rapidjson::Type::kNumberType: - if (v.IsUint()) { - wbytes = snprintf(tmp_buf, sizeof(tmp_buf), "%u", v.GetUint()); - } else if (v.IsInt()) { - wbytes = snprintf(tmp_buf, sizeof(tmp_buf), "%d", v.GetInt()); - } else if (v.IsUint64()) { - wbytes = snprintf(tmp_buf, sizeof(tmp_buf), "%" PRIu64, v.GetUint64()); - } else if (v.IsInt64()) { - wbytes = snprintf(tmp_buf, sizeof(tmp_buf), "%" PRId64, v.GetInt64()); - } else { - wbytes = snprintf(tmp_buf, sizeof(tmp_buf), "%f", v.GetDouble()); - } - _backup_string.emplace_back(tmp_buf, wbytes); - _string_nulls.push_back(false); - // do not set _data_string here. - // Because the address of the string stored in `_backup_string` may - // change each time `emplace_back()` is called. - break; - case rapidjson::Type::kFalseType: - _backup_string.emplace_back(true_value); - _string_nulls.push_back(false); - break; - case rapidjson::Type::kTrueType: - _backup_string.emplace_back(false_value); - _string_nulls.push_back(false); - break; - case rapidjson::Type::kNullType: - _backup_string.emplace_back(); - _string_nulls.push_back(true); - break; - default: - _backup_string.emplace_back(); - _string_nulls.push_back(true); - break; - } - } - // Must set _data_string at the end, so that we can - // save the real addr of string in `_backup_string` to `_data_string`. - for (auto& str : _backup_string) { - _data_string.emplace_back(str); - } - break; - } - case ExplodeJsonArrayType::JSON: { - _data_string.clear(); - _backup_string.clear(); - _string_nulls.clear(); - for (auto& v : document.GetArray()) { - if (v.IsObject()) { - rapidjson::StringBuffer buffer; - rapidjson::Writer writer(buffer); - v.Accept(writer); - _backup_string.emplace_back(buffer.GetString(), buffer.GetSize()); - _string_nulls.push_back(false); - } else { - _data_string.push_back({}); - _string_nulls.push_back(true); - } - } - // Must set _data_string at the end, so that we can - // save the real addr of string in `_backup_string` to `_data_string`. - for (auto& str : _backup_string) { - _data_string.emplace_back(str); - } - break; - } - default: - CHECK(false) << type; - break; - } - return size; -} - -VExplodeJsonArrayTableFunction::VExplodeJsonArrayTableFunction(ExplodeJsonArrayType type) - : _type(type) { +template +VExplodeJsonArrayTableFunction::VExplodeJsonArrayTableFunction() : TableFunction() { _fn_name = "vexplode_json_array"; } -Status VExplodeJsonArrayTableFunction::process_init(Block* block, RuntimeState* state) { +template +Status VExplodeJsonArrayTableFunction::process_init(Block* block, RuntimeState* state) { CHECK(_expr_context->root()->children().size() == 1) << _expr_context->root()->children().size(); @@ -191,11 +50,11 @@ Status VExplodeJsonArrayTableFunction::process_init(Block* block, RuntimeState* RETURN_IF_ERROR(_expr_context->root()->children()[0]->execute(_expr_context.get(), block, &text_column_idx)); _text_column = block->get_by_position(text_column_idx).column; - return Status::OK(); } -void VExplodeJsonArrayTableFunction::process_row(size_t row_idx) { +template +void VExplodeJsonArrayTableFunction::process_row(size_t row_idx) { TableFunction::process_row(row_idx); StringRef text = _text_column->get_data_at(row_idx); @@ -203,22 +62,83 @@ void VExplodeJsonArrayTableFunction::process_row(size_t row_idx) { rapidjson::Document document; document.Parse(text.data, text.size); if (!document.HasParseError() && document.IsArray() && document.GetArray().Size()) { - _cur_size = _parsed_data.set_output(_type, document); + _cur_size = _parsed_data.set_output(document, document.GetArray().Size()); } } } -void VExplodeJsonArrayTableFunction::process_close() { +template +void VExplodeJsonArrayTableFunction::process_close() { _text_column = nullptr; + _parsed_data.reset(); } -void VExplodeJsonArrayTableFunction::get_value(MutableColumnPtr& column) { - if (current_empty() || _parsed_data.get_value(_type, _cur_offset, true) == nullptr) { +template +void VExplodeJsonArrayTableFunction::get_same_many_values(MutableColumnPtr& column, + int length) { + if (current_empty()) { + column->insert_many_defaults(length); + } else { + _insert_same_many_values_into_column(column, length); + } +} + +template +int VExplodeJsonArrayTableFunction::get_value(MutableColumnPtr& column, int max_step) { + max_step = std::min(max_step, (int)(_cur_size - _cur_offset)); + if (current_empty()) { column->insert_default(); + max_step = 1; + } else { + _insert_values_into_column(column, max_step); + } + forward(max_step); + return max_step; +} + +template +void VExplodeJsonArrayTableFunction::_insert_same_many_values_into_column( + MutableColumnPtr& column, int length) { + if (_is_nullable) { + auto* nullable_column = assert_cast(column.get()); + auto nested_column = nullable_column->get_nested_column_ptr(); + + _parsed_data.insert_many_same_value_from_parsed_data(nested_column, _cur_offset, length); + + auto* nullmap_column = + assert_cast(nullable_column->get_null_map_column_ptr().get()); + size_t old_size = nullmap_column->size(); + nullmap_column->resize(old_size + length); + memset(nullmap_column->get_data().data() + old_size, + *(_parsed_data.get_null_flag_address(_cur_offset)), length * sizeof(UInt8)); } else { - column->insert_data((char*)_parsed_data.get_value(_type, _cur_offset, true), - _parsed_data.get_value_length(_type, _cur_offset)); + _parsed_data.insert_many_same_value_from_parsed_data(column, _cur_offset, length); } } +template +void VExplodeJsonArrayTableFunction::_insert_values_into_column(MutableColumnPtr& column, + int max_step) { + if (_is_nullable) { + auto* nullable_column = assert_cast(column.get()); + auto nested_column = nullable_column->get_nested_column_ptr(); + + _parsed_data.insert_result_from_parsed_data(nested_column, _cur_offset, max_step); + + auto* nullmap_column = + assert_cast(nullable_column->get_null_map_column_ptr().get()); + size_t old_size = nullmap_column->size(); + nullmap_column->resize(old_size + max_step); + memcpy(nullmap_column->get_data().data() + old_size, + _parsed_data.get_null_flag_address(_cur_offset), max_step * sizeof(UInt8)); + } else { + _parsed_data.insert_result_from_parsed_data(column, _cur_offset, max_step); + } +} + +template class VExplodeJsonArrayTableFunction; +template class VExplodeJsonArrayTableFunction; +template class VExplodeJsonArrayTableFunction; +template class VExplodeJsonArrayTableFunction; + } // namespace doris::vectorized \ No newline at end of file diff --git a/be/src/vec/exprs/table_function/vexplode_json_array.h b/be/src/vec/exprs/table_function/vexplode_json_array.h index 42b3cba299437a9..a47956ff1775b26 100644 --- a/be/src/vec/exprs/table_function/vexplode_json_array.h +++ b/be/src/vec/exprs/table_function/vexplode_json_array.h @@ -19,8 +19,6 @@ #include #include -#include -#include #include #include @@ -28,76 +26,247 @@ #include "common/status.h" #include "gutil/integral_types.h" +#include "rapidjson/stringbuffer.h" +#include "rapidjson/writer.h" #include "vec/common/string_ref.h" +#include "vec/core/types.h" #include "vec/data_types/data_type.h" #include "vec/exprs/table_function/table_function.h" -namespace doris { -namespace vectorized { -class Block; -} // namespace vectorized -} // namespace doris - namespace doris::vectorized { -enum ExplodeJsonArrayType { INT = 0, DOUBLE, STRING, JSON }; - +template struct ParsedData { - static std::string true_value; - static std::string false_value; - - // The number parsed from json array - // the `_backup` saved the real number entity. - std::vector _data; - std::vector _data_string; - std::vector _backup_int; - std::vector _backup_double; - std::vector _backup_string; - std::vector _string_nulls; - char tmp_buf[128] = {0}; + ParsedData() = default; + virtual ~ParsedData() = default; + virtual void reset() { + _backup_data.clear(); + _values_null_flag.clear(); + } + virtual int set_output(rapidjson::Document& document, int value_size) = 0; + virtual void insert_result_from_parsed_data(MutableColumnPtr& column, int64_t cur_offset, + int max_step) = 0; + virtual void insert_many_same_value_from_parsed_data(MutableColumnPtr& column, + int64_t cur_offset, int length) = 0; + const char* get_null_flag_address(int cur_offset) { + return reinterpret_cast(_values_null_flag.data() + cur_offset); + } + std::vector _values_null_flag; + std::vector _backup_data; +}; + +struct ParsedDataInt : public ParsedData { + static constexpr auto MAX_VALUE = std::numeric_limits::max(); //9223372036854775807 + static constexpr auto MIN_VALUE = std::numeric_limits::min(); //-9223372036854775808 - void* get_value(ExplodeJsonArrayType type, int64_t offset, bool real = false) { - switch (type) { - case ExplodeJsonArrayType::INT: - case ExplodeJsonArrayType::DOUBLE: - return _data[offset]; - case ExplodeJsonArrayType::JSON: - case ExplodeJsonArrayType::STRING: - return _string_nulls[offset] ? nullptr - : real ? reinterpret_cast(_backup_string[offset].data()) - : &_data_string[offset]; - default: - return nullptr; + int set_output(rapidjson::Document& document, int value_size) override { + _values_null_flag.resize(value_size, 0); + _backup_data.resize(value_size); + int i = 0; + for (auto& v : document.GetArray()) { + if (v.IsInt64()) { + _backup_data[i] = v.GetInt64(); + } else if (v.IsUint64()) { + auto value = v.GetUint64(); + if (value > MAX_VALUE) { + _backup_data[i] = MAX_VALUE; + } else { + _backup_data[i] = value; + } + } else if (v.IsDouble()) { + auto value = v.GetDouble(); + if (value > MAX_VALUE) { + _backup_data[i] = MAX_VALUE; + } else if (value < MIN_VALUE) { + _backup_data[i] = MIN_VALUE; + } else { + _backup_data[i] = long(value); + } + } else { + _values_null_flag[i] = 1; + _backup_data[i] = 0; + } + ++i; } + return value_size; + } + + void insert_result_from_parsed_data(MutableColumnPtr& column, int64_t cur_offset, + int max_step) override { + assert_cast(column.get()) + ->insert_many_raw_data( + reinterpret_cast(_backup_data.data() + cur_offset), max_step); } - int64 get_value_length(ExplodeJsonArrayType type, int64_t offset) { - if ((type == ExplodeJsonArrayType::STRING || type == ExplodeJsonArrayType::JSON) && - !_string_nulls[offset]) { - return _backup_string[offset].size(); + void insert_many_same_value_from_parsed_data(MutableColumnPtr& column, int64_t cur_offset, + int length) override { + assert_cast(column.get()) + ->insert_raw_integers(_backup_data[cur_offset], length); + } +}; + +struct ParsedDataDouble : public ParsedData { + int set_output(rapidjson::Document& document, int value_size) override { + _values_null_flag.resize(value_size, 0); + _backup_data.resize(value_size); + int i = 0; + for (auto& v : document.GetArray()) { + if (v.IsDouble()) { + _backup_data[i] = v.GetDouble(); + } else { + _backup_data[i] = 0; + _values_null_flag[i] = 1; + } + ++i; } - return 0; + return value_size; + } + + void insert_result_from_parsed_data(MutableColumnPtr& column, int64_t cur_offset, + int max_step) override { + assert_cast(column.get()) + ->insert_many_raw_data( + reinterpret_cast(_backup_data.data() + cur_offset), max_step); } - int set_output(ExplodeJsonArrayType type, rapidjson::Document& document); + void insert_many_same_value_from_parsed_data(MutableColumnPtr& column, int64_t cur_offset, + int length) override { + assert_cast(column.get()) + ->insert_raw_integers(_backup_data[cur_offset], length); + } }; +struct ParsedDataStringBase : public ParsedData { + void insert_result_from_parsed_data(MutableColumnPtr& column, int64_t cur_offset, + int max_step) override { + assert_cast(column.get()) + ->insert_many_strings(_data_string_ref.data() + cur_offset, max_step); + } + + void insert_many_same_value_from_parsed_data(MutableColumnPtr& column, int64_t cur_offset, + int length) override { + assert_cast(column.get()) + ->insert_many_data(_data_string_ref[cur_offset].data, + _data_string_ref[cur_offset].size, length); + } + + void reset() override { + ParsedData::reset(); + _data_string_ref.clear(); + } + + static constexpr const char* TRUE_VALUE = "true"; + static constexpr const char* FALSE_VALUE = "false"; + std::vector _data_string_ref; + char tmp_buf[128] = {0}; +}; + +struct ParsedDataString : public ParsedDataStringBase { + int set_output(rapidjson::Document& document, int value_size) override { + _data_string_ref.clear(); + _backup_data.clear(); + _values_null_flag.clear(); + int32_t wbytes = 0; + for (auto& v : document.GetArray()) { + switch (v.GetType()) { + case rapidjson::Type::kStringType: { + _backup_data.emplace_back(v.GetString(), v.GetStringLength()); + _values_null_flag.emplace_back(false); + break; + // do not set _data_string here. + // Because the address of the string stored in `_backup_data` may + // change each time `emplace_back()` is called. + } + case rapidjson::Type::kNumberType: { + if (v.IsUint()) { + wbytes = snprintf(tmp_buf, sizeof(tmp_buf), "%u", v.GetUint()); + } else if (v.IsInt()) { + wbytes = snprintf(tmp_buf, sizeof(tmp_buf), "%d", v.GetInt()); + } else if (v.IsUint64()) { + wbytes = snprintf(tmp_buf, sizeof(tmp_buf), "%" PRIu64, v.GetUint64()); + } else if (v.IsInt64()) { + wbytes = snprintf(tmp_buf, sizeof(tmp_buf), "%" PRId64, v.GetInt64()); + } else { + wbytes = snprintf(tmp_buf, sizeof(tmp_buf), "%f", v.GetDouble()); + } + _backup_data.emplace_back(tmp_buf, wbytes); + _values_null_flag.emplace_back(false); + // do not set _data_string here. + // Because the address of the string stored in `_backup_data` may + // change each time `emplace_back()` is called. + break; + } + case rapidjson::Type::kFalseType: + _backup_data.emplace_back(TRUE_VALUE); + _values_null_flag.emplace_back(false); + break; + case rapidjson::Type::kTrueType: + _backup_data.emplace_back(FALSE_VALUE); + _values_null_flag.emplace_back(false); + break; + case rapidjson::Type::kNullType: + _backup_data.emplace_back(); + _values_null_flag.emplace_back(true); + break; + default: + _backup_data.emplace_back(); + _values_null_flag.emplace_back(true); + break; + } + } + // Must set _data_string at the end, so that we can + // save the real addr of string in `_backup_data` to `_data_string`. + for (auto& str : _backup_data) { + _data_string_ref.emplace_back(str.data(), str.length()); + } + return value_size; + } +}; + +struct ParsedDataJSON : public ParsedDataStringBase { + int set_output(rapidjson::Document& document, int value_size) override { + _data_string_ref.clear(); + _backup_data.clear(); + _values_null_flag.clear(); + for (auto& v : document.GetArray()) { + if (v.IsObject()) { + rapidjson::StringBuffer buffer; + rapidjson::Writer writer(buffer); + v.Accept(writer); + _backup_data.emplace_back(buffer.GetString(), buffer.GetSize()); + _values_null_flag.emplace_back(false); + } else { + _backup_data.emplace_back(); + _values_null_flag.emplace_back(true); + } + } + // Must set _data_string at the end, so that we can + // save the real addr of string in `_backup_data` to `_data_string`. + for (auto& str : _backup_data) { + _data_string_ref.emplace_back(str); + } + return value_size; + } +}; + +template class VExplodeJsonArrayTableFunction final : public TableFunction { - ENABLE_FACTORY_CREATOR(VExplodeJsonArrayTableFunction); + ENABLE_FACTORY_CREATOR(VExplodeJsonArrayTableFunction); public: - VExplodeJsonArrayTableFunction(ExplodeJsonArrayType type); + VExplodeJsonArrayTableFunction(); ~VExplodeJsonArrayTableFunction() override = default; Status process_init(Block* block, RuntimeState* state) override; void process_row(size_t row_idx) override; void process_close() override; - void get_value(MutableColumnPtr& column) override; + void get_same_many_values(MutableColumnPtr& column, int length) override; + int get_value(MutableColumnPtr& column, int max_step) override; private: - ParsedData _parsed_data; - ExplodeJsonArrayType _type; - + void _insert_same_many_values_into_column(MutableColumnPtr& column, int max_step); + void _insert_values_into_column(MutableColumnPtr& column, int max_step); + DataImpl _parsed_data; ColumnPtr _text_column; }; diff --git a/be/src/vec/exprs/table_function/vexplode_map.cpp b/be/src/vec/exprs/table_function/vexplode_map.cpp index 923316fd282309f..ee88d9f07dfd49a 100644 --- a/be/src/vec/exprs/table_function/vexplode_map.cpp +++ b/be/src/vec/exprs/table_function/vexplode_map.cpp @@ -90,12 +90,12 @@ void VExplodeMapTableFunction::process_close() { _collection_offset = 0; } -void VExplodeMapTableFunction::get_value(MutableColumnPtr& column) { +void VExplodeMapTableFunction::get_same_many_values(MutableColumnPtr& column, int length) { // now we only support map column explode to struct column size_t pos = _collection_offset + _cur_offset; // if current is empty map row, also append a default value if (current_empty()) { - column->insert_default(); + column->insert_many_defaults(length); return; } ColumnStruct* ret = nullptr; @@ -106,7 +106,7 @@ void VExplodeMapTableFunction::get_value(MutableColumnPtr& column) { assert_cast(column.get())->get_nested_column_ptr().get()); assert_cast( assert_cast(column.get())->get_null_map_column_ptr().get()) - ->insert_default(); + ->insert_many_defaults(length); } else if (column->is_column_struct()) { ret = assert_cast(column.get()); } else { @@ -118,8 +118,42 @@ void VExplodeMapTableFunction::get_value(MutableColumnPtr& column) { ErrorCode::INTERNAL_ERROR, "only support map column explode to two column, but given: ", ret->tuple_size()); } - ret->get_column(0).insert_from(_map_detail.map_col->get_keys(), pos); - ret->get_column(1).insert_from(_map_detail.map_col->get_values(), pos); + ret->get_column(0).insert_many_from(_map_detail.map_col->get_keys(), pos, length); + ret->get_column(1).insert_many_from(_map_detail.map_col->get_values(), pos, length); } +int VExplodeMapTableFunction::get_value(MutableColumnPtr& column, int max_step) { + max_step = std::min(max_step, (int)(_cur_size - _cur_offset)); + size_t pos = _collection_offset + _cur_offset; + if (current_empty()) { + column->insert_default(); + max_step = 1; + } else { + ColumnStruct* struct_column = nullptr; + if (_is_nullable) { + auto* nullable_column = assert_cast(column.get()); + struct_column = + assert_cast(nullable_column->get_nested_column_ptr().get()); + auto* nullmap_column = + assert_cast(nullable_column->get_null_map_column_ptr().get()); + // here nullmap_column insert max_step many defaults as if MAP[row_idx] is NULL + // will be not update value, _cur_size = 0, means current_empty; + // so here could insert directly + nullmap_column->insert_many_defaults(max_step); + } else { + struct_column = assert_cast(column.get()); + } + if (!struct_column || struct_column->tuple_size() != 2) { + throw Exception(ErrorCode::INTERNAL_ERROR, + "only support map column explode to two column, but given: ", + struct_column->tuple_size()); + } + struct_column->get_column(0).insert_range_from(_map_detail.map_col->get_keys(), pos, + max_step); + struct_column->get_column(1).insert_range_from(_map_detail.map_col->get_values(), pos, + max_step); + } + forward(max_step); + return max_step; +} } // namespace doris::vectorized diff --git a/be/src/vec/exprs/table_function/vexplode_map.h b/be/src/vec/exprs/table_function/vexplode_map.h index 770c06e3bb9c682..c9756c9499bd8f1 100644 --- a/be/src/vec/exprs/table_function/vexplode_map.h +++ b/be/src/vec/exprs/table_function/vexplode_map.h @@ -57,7 +57,8 @@ class VExplodeMapTableFunction : public TableFunction { Status process_init(Block* block, RuntimeState* state) override; void process_row(size_t row_idx) override; void process_close() override; - void get_value(MutableColumnPtr& column) override; + void get_same_many_values(MutableColumnPtr& column, int length) override; + int get_value(MutableColumnPtr& column, int max_step) override; private: ColumnPtr _collection_column; diff --git a/be/src/vec/exprs/table_function/vexplode_numbers.cpp b/be/src/vec/exprs/table_function/vexplode_numbers.cpp index 9b5f7ffcbbcd6d2..42f8aa9360e84e8 100644 --- a/be/src/vec/exprs/table_function/vexplode_numbers.cpp +++ b/be/src/vec/exprs/table_function/vexplode_numbers.cpp @@ -92,19 +92,19 @@ void VExplodeNumbersTableFunction::process_close() { _value_column = nullptr; } -void VExplodeNumbersTableFunction::get_value(MutableColumnPtr& column) { +void VExplodeNumbersTableFunction::get_same_many_values(MutableColumnPtr& column, int length) { if (current_empty()) { - column->insert_default(); + column->insert_many_defaults(length); } else { if (_is_nullable) { assert_cast( assert_cast(column.get())->get_nested_column_ptr().get()) - ->insert_value(_cur_offset); + ->insert_raw_integers(_cur_offset, length); assert_cast( assert_cast(column.get())->get_null_map_column_ptr().get()) - ->insert_default(); + ->insert_many_defaults(length); } else { - assert_cast(column.get())->insert_value(_cur_offset); + assert_cast(column.get())->insert_raw_integers(_cur_offset, length); } } } diff --git a/be/src/vec/exprs/table_function/vexplode_numbers.h b/be/src/vec/exprs/table_function/vexplode_numbers.h index 1890719c689aa60..dda4a047a2088c9 100644 --- a/be/src/vec/exprs/table_function/vexplode_numbers.h +++ b/be/src/vec/exprs/table_function/vexplode_numbers.h @@ -44,7 +44,7 @@ class VExplodeNumbersTableFunction : public TableFunction { Status process_init(Block* block, RuntimeState* state) override; void process_row(size_t row_idx) override; void process_close() override; - void get_value(MutableColumnPtr& column) override; + void get_same_many_values(MutableColumnPtr& column, int length) override; int get_value(MutableColumnPtr& column, int max_step) override { max_step = std::min(max_step, (int)(_cur_size - _cur_offset)); if (_is_const) { diff --git a/be/src/vec/exprs/table_function/vexplode_split.cpp b/be/src/vec/exprs/table_function/vexplode_split.cpp index 13ffb602ac22683..51a95dbd5c6f21e 100644 --- a/be/src/vec/exprs/table_function/vexplode_split.cpp +++ b/be/src/vec/exprs/table_function/vexplode_split.cpp @@ -125,11 +125,11 @@ void VExplodeSplitTableFunction::process_close() { _delimiter = {}; } -void VExplodeSplitTableFunction::get_value(MutableColumnPtr& column) { +void VExplodeSplitTableFunction::get_same_many_values(MutableColumnPtr& column, int length) { if (current_empty()) { - column->insert_default(); + column->insert_many_defaults(length); } else { - column->insert_data(_backup[_cur_offset].data, _backup[_cur_offset].size); + column->insert_many_data(_backup[_cur_offset].data, _backup[_cur_offset].size, length); } } diff --git a/be/src/vec/exprs/table_function/vexplode_split.h b/be/src/vec/exprs/table_function/vexplode_split.h index 6d00cc48f781267..089abe23dda140f 100644 --- a/be/src/vec/exprs/table_function/vexplode_split.h +++ b/be/src/vec/exprs/table_function/vexplode_split.h @@ -45,7 +45,7 @@ class VExplodeSplitTableFunction final : public TableFunction { Status process_init(Block* block, RuntimeState* state) override; void process_row(size_t row_idx) override; void process_close() override; - void get_value(MutableColumnPtr& column) override; + void get_same_many_values(MutableColumnPtr& column, int length) override; int get_value(MutableColumnPtr& column, int max_step) override; private: diff --git a/be/test/vec/function/function_test_util.cpp b/be/test/vec/function/function_test_util.cpp index 50d1d3ab090e9d1..0025945ef19adda 100644 --- a/be/test/vec/function/function_test_util.cpp +++ b/be/test/vec/function/function_test_util.cpp @@ -364,7 +364,7 @@ Block* process_table_function(TableFunction* fn, Block* input_block, } do { - fn->get_value(column); + fn->get_same_many_values(column, 1); fn->forward(); } while (!fn->eos()); } diff --git a/docker/thirdparties/docker-compose/elasticsearch/scripts/es_init.sh b/docker/thirdparties/docker-compose/elasticsearch/scripts/es_init.sh index 5c865e660ada4e4..21f1dc55a334cb4 100755 --- a/docker/thirdparties/docker-compose/elasticsearch/scripts/es_init.sh +++ b/docker/thirdparties/docker-compose/elasticsearch/scripts/es_init.sh @@ -22,6 +22,8 @@ curl "http://${ES_5_HOST}:9200/test1" -H "Content-Type:application/json" -X PUT -d "@/mnt/scripts/index/es6_test1.json" # create index test2_20220808 curl "http://${ES_5_HOST}:9200/test2_20220808" -H "Content-Type:application/json" -X PUT -d '@/mnt/scripts/index/es6_test2.json' +# create index test2_20220809 +curl "http://${ES_5_HOST}:9200/test2_20220809" -H "Content-Type:application/json" -X PUT -d '@/mnt/scripts/index/es6_test2.json' # put data for test1 curl "http://${ES_5_HOST}:9200/test1/doc/1" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data1_es6.json' curl "http://${ES_5_HOST}:9200/test1/doc/2" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data2_es6.json' @@ -31,9 +33,14 @@ curl "http://${ES_5_HOST}:9200/test1/doc/3" -H "Content-Type:application/json" - curl "http://${ES_5_HOST}:9200/test2_20220808/doc/1" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data1_es6.json' curl "http://${ES_5_HOST}:9200/test2_20220808/doc/2" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data2_es6.json' curl "http://${ES_5_HOST}:9200/test2_20220808/doc/3" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data3_es5.json' +# put data for test2_20220809 +curl "http://${ES_5_HOST}:9200/test2_20220809/doc/1" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data1_es6.json' +curl "http://${ES_5_HOST}:9200/test2_20220809/doc/2" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data2_es6.json' +curl "http://${ES_5_HOST}:9200/test2_20220809/doc/3" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data3_es5.json' # put _meta for array curl "http://${ES_5_HOST}:9200/test1/doc/_mapping" -H "Content-Type:application/json" -X PUT -d "@/mnt/scripts/index/array_meta.json" curl "http://${ES_5_HOST}:9200/test2_20220808/doc/_mapping" -H "Content-Type:application/json" -X PUT -d "@/mnt/scripts/index/array_meta.json" +curl "http://${ES_5_HOST}:9200/test2_20220809/doc/_mapping" -H "Content-Type:application/json" -X PUT -d "@/mnt/scripts/index/array_meta.json" # create index .hide curl "http://${ES_5_HOST}:9200/.hide" -H "Content-Type:application/json" -X PUT -d "@/mnt/scripts/index/es6_hide.json" @@ -43,6 +50,8 @@ curl "http://${ES_5_HOST}:9200/.hide" -H "Content-Type:application/json" -X PUT curl "http://${ES_6_HOST}:9200/test1" -H "Content-Type:application/json" -X PUT -d "@/mnt/scripts/index/es6_test1.json" # create index test2_20220808 curl "http://${ES_6_HOST}:9200/test2_20220808" -H "Content-Type:application/json" -X PUT -d '@/mnt/scripts/index/es6_test2.json' +# create index test2_20220809 +curl "http://${ES_6_HOST}:9200/test2_20220809" -H "Content-Type:application/json" -X PUT -d '@/mnt/scripts/index/es6_test2.json' # put data for test1 curl "http://${ES_6_HOST}:9200/test1/doc/1" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data1_es6.json' curl "http://${ES_6_HOST}:9200/test1/doc/2" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data2_es6.json' @@ -51,9 +60,14 @@ curl "http://${ES_6_HOST}:9200/test1/doc/3" -H "Content-Type:application/json" - curl "http://${ES_6_HOST}:9200/test2_20220808/doc/1" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data1_es6.json' curl "http://${ES_6_HOST}:9200/test2_20220808/doc/2" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data2_es6.json' curl "http://${ES_6_HOST}:9200/test2_20220808/doc/3" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data3_es6.json' +# put data for test2_20220809 +curl "http://${ES_6_HOST}:9200/test2_20220809/doc/1" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data1_es6.json' +curl "http://${ES_6_HOST}:9200/test2_20220809/doc/2" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data2_es6.json' +curl "http://${ES_6_HOST}:9200/test2_20220809/doc/3" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data3_es6.json' # put _meta for array curl "http://${ES_6_HOST}:9200/test1/doc/_mapping" -H "Content-Type:application/json" -X PUT -d "@/mnt/scripts/index/array_meta.json" curl "http://${ES_6_HOST}:9200/test2_20220808/doc/_mapping" -H "Content-Type:application/json" -X PUT -d "@/mnt/scripts/index/array_meta.json" +curl "http://${ES_6_HOST}:9200/test2_20220809/doc/_mapping" -H "Content-Type:application/json" -X PUT -d "@/mnt/scripts/index/array_meta.json" # create index .hide curl "http://${ES_6_HOST}:9200/.hide" -H "Content-Type:application/json" -X PUT -d "@/mnt/scripts/index/es6_hide.json" @@ -62,6 +76,8 @@ curl "http://${ES_6_HOST}:9200/.hide" -H "Content-Type:application/json" -X PUT curl "http://${ES_7_HOST}:9200/test1" -H "Content-Type:application/json" -X PUT -d "@/mnt/scripts/index/es7_test1.json" # create index test2_20220808 curl "http://${ES_7_HOST}:9200/test2_20220808" -H "Content-Type:application/json" -X PUT -d '@/mnt/scripts/index/es7_test2.json' +# create index test2_20220808 +curl "http://${ES_7_HOST}:9200/test2_20220809" -H "Content-Type:application/json" -X PUT -d '@/mnt/scripts/index/es7_test2.json' # create index test3_20231005 curl "http://${ES_7_HOST}:9200/test3_20231005" -H "Content-Type:application/json" -X PUT -d '@/mnt/scripts/index/es7_test3.json' # put data for tese1 @@ -74,12 +90,18 @@ curl "http://${ES_7_HOST}:9200/test2_20220808/_doc/1" -H "Content-Type:applicati curl "http://${ES_7_HOST}:9200/test2_20220808/_doc/2" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data2.json' curl "http://${ES_7_HOST}:9200/test2_20220808/_doc/3" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data3.json' curl "http://${ES_7_HOST}:9200/test2_20220808/_doc/4" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data4.json' +# put data for test2_20220809 +curl "http://${ES_7_HOST}:9200/test2_20220809/_doc/1" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data1.json' +curl "http://${ES_7_HOST}:9200/test2_20220809/_doc/2" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data2.json' +curl "http://${ES_7_HOST}:9200/test2_20220809/_doc/3" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data3.json' +curl "http://${ES_7_HOST}:9200/test2_20220809/_doc/4" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data4.json' # put data for test3_20231005 curl "http://${ES_7_HOST}:9200/test3_20231005/_doc/1" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data5.json' # put _meta for array curl "http://${ES_7_HOST}:9200/test1/_mapping" -H "Content-Type:application/json" -X PUT -d "@/mnt/scripts/index/array_meta.json" curl "http://${ES_7_HOST}:9200/test2_20220808/_mapping" -H "Content-Type:application/json" -X PUT -d "@/mnt/scripts/index/array_meta.json" +curl "http://${ES_7_HOST}:9200/test2_20220809/_mapping" -H "Content-Type:application/json" -X PUT -d "@/mnt/scripts/index/array_meta.json" # create index .hide curl "http://${ES_7_HOST}:9200/.hide" -H "Content-Type:application/json" -X PUT -d "@/mnt/scripts/index/es7_hide.json" @@ -89,6 +111,8 @@ curl "http://${ES_7_HOST}:9200/.hide" -H "Content-Type:application/json" -X PUT curl "http://${ES_8_HOST}:9200/test1" -H "Content-Type:application/json" -X PUT -d "@/mnt/scripts/index/es7_test1.json" # create index test2_20220808 curl "http://${ES_8_HOST}:9200/test2_20220808" -H "Content-Type:application/json" -X PUT -d '@/mnt/scripts/index/es7_test2.json' +# create index test2_20220809 +curl "http://${ES_8_HOST}:9200/test2_20220809" -H "Content-Type:application/json" -X PUT -d '@/mnt/scripts/index/es7_test2.json' # create index test3_20231005 curl "http://${ES_8_HOST}:9200/test3_20231005" -H "Content-Type:application/json" -X PUT -d '@/mnt/scripts/index/es7_test3.json' @@ -102,6 +126,11 @@ curl "http://${ES_8_HOST}:9200/test2_20220808/_doc/1" -H "Content-Type:applicati curl "http://${ES_8_HOST}:9200/test2_20220808/_doc/2" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data2.json' curl "http://${ES_8_HOST}:9200/test2_20220808/_doc/3" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data3.json' curl "http://${ES_8_HOST}:9200/test2_20220808/_doc/4" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data4.json' +# put data for test2_20220809 +curl "http://${ES_8_HOST}:9200/test2_20220809/_doc/1" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data1.json' +curl "http://${ES_8_HOST}:9200/test2_20220809/_doc/2" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data2.json' +curl "http://${ES_8_HOST}:9200/test2_20220809/_doc/3" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data3.json' +curl "http://${ES_8_HOST}:9200/test2_20220809/_doc/4" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data4.json' # put data for test3_20231005 curl "http://${ES_8_HOST}:9200/test3_20231005/_doc/1" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data5.json' diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/es/source/EsScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/es/source/EsScanNode.java index 7bc4f9ea1c024a1..3b0d2f08b282d10 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/es/source/EsScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/es/source/EsScanNode.java @@ -42,6 +42,7 @@ import org.apache.doris.planner.PartitionPruner; import org.apache.doris.planner.PlanNodeId; import org.apache.doris.planner.RangePartitionPrunerV2; +import org.apache.doris.qe.ConnectContext; import org.apache.doris.statistics.StatisticalType; import org.apache.doris.statistics.query.StatsDelta; import org.apache.doris.system.Backend; @@ -63,6 +64,7 @@ import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -216,8 +218,15 @@ private List getShardLocations() throws UserException { String.join(",", unPartitionedIndices), String.join(",", partitionedIndices)); } List result = Lists.newArrayList(); + boolean enableESParallelScroll = isEnableESParallelScroll(); for (EsShardPartitions indexState : selectedIndex) { - for (List shardRouting : indexState.getShardRoutings().values()) { + // When disabling parallel scroll, only use the first shard routing. + // Because we only need plan a single scan range. + List> shardRoutings = enableESParallelScroll + ? new ArrayList<>(indexState.getShardRoutings().values()) : + Collections.singletonList(indexState.getShardRoutings().get(0)); + + for (List shardRouting : shardRoutings) { // get backends List shardAllocations = new ArrayList<>(); List preLocations = new ArrayList<>(); @@ -229,7 +238,10 @@ private List getShardLocations() throws UserException { FederationBackendPolicy backendPolicy = new FederationBackendPolicy(); backendPolicy.init(preLocations); TScanRangeLocations locations = new TScanRangeLocations(); - for (int i = 0; i < backendPolicy.numBackends(); ++i) { + // When disabling parallel scroll, only use the first backend. + // Because we only need plan a single query to one backend. + int numBackends = enableESParallelScroll ? backendPolicy.numBackends() : 1; + for (int i = 0; i < numBackends; ++i) { TScanRangeLocation location = new TScanRangeLocation(); Backend be = backendPolicy.getNextBe(); location.setBackendId(be.getId()); @@ -240,11 +252,18 @@ private List getShardLocations() throws UserException { // Generate on es scan range TEsScanRange esScanRange = new TEsScanRange(); esScanRange.setEsHosts(shardAllocations); - esScanRange.setIndex(shardRouting.get(0).getIndexName()); + // When disabling parallel scroll, use the index state's index name to prevent the index aliases from + // being expanded. + // eg: index alias `log-20240501` may point to multiple indices, + // such as `log-20240501-1`/`log-20240501-2`. + // When we plan a single query, we should use the index alias instead of the real indices names. + esScanRange.setIndex( + enableESParallelScroll ? shardRouting.get(0).getIndexName() : indexState.getIndexName()); if (table.getType() != null) { esScanRange.setType(table.getMappingType()); } - esScanRange.setShardId(shardRouting.get(0).getShardId()); + // When disabling parallel scroll, set shard id to -1 to disable shard preference in query option. + esScanRange.setShardId(enableESParallelScroll ? shardRouting.get(0).getShardId() : -1); // Scan range TScanRange scanRange = new TScanRange(); scanRange.setEsScanRange(esScanRange); @@ -267,6 +286,11 @@ private List getShardLocations() throws UserException { return result; } + private boolean isEnableESParallelScroll() { + ConnectContext connectContext = ConnectContext.get(); + return connectContext != null && connectContext.getSessionVariable().isEnableESParallelScroll(); + } + /** * if the index name is an alias or index pattern, then the es table is related * with one or more indices some indices could be pruned by using partition info diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/ChildOutputPropertyDeriver.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/ChildOutputPropertyDeriver.java index 3756c1bcfe35187..bb23f46cdc43157 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/ChildOutputPropertyDeriver.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/ChildOutputPropertyDeriver.java @@ -279,7 +279,7 @@ public PhysicalProperties visitPhysicalHashJoin( case RIGHT_SEMI_JOIN: case RIGHT_ANTI_JOIN: case RIGHT_OUTER_JOIN: - if (JoinUtils.couldColocateJoin(leftHashSpec, rightHashSpec)) { + if (JoinUtils.couldColocateJoin(leftHashSpec, rightHashSpec, hashJoin.getHashJoinConjuncts())) { return new PhysicalProperties(rightHashSpec); } else { // retain left shuffle type, since coordinator use left most node to schedule fragment diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/ChildrenPropertiesRegulator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/ChildrenPropertiesRegulator.java index 3beed014aac9109..02cb56ef8eab1e8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/ChildrenPropertiesRegulator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/ChildrenPropertiesRegulator.java @@ -245,7 +245,7 @@ public Boolean visitPhysicalHashJoin(PhysicalHashJoin updatedForLeft = Optional.empty(); Optional updatedForRight = Optional.empty(); - if (JoinUtils.couldColocateJoin(leftHashSpec, rightHashSpec)) { + if (JoinUtils.couldColocateJoin(leftHashSpec, rightHashSpec, hashJoin.getHashJoinConjuncts())) { // check colocate join with scan return true; } else if (couldNotRightBucketShuffleJoin(hashJoin.getJoinType(), leftHashSpec, rightHashSpec)) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalOlapScanToPhysicalOlapScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalOlapScanToPhysicalOlapScan.java index 43436355ae17aa5..472d2e169dba8ba 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalOlapScanToPhysicalOlapScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalOlapScanToPhysicalOlapScan.java @@ -83,8 +83,8 @@ private DistributionSpec convertDistribution(LogicalOlapScan olapScan) { List output = olapScan.getOutput(); List baseOutput = olapScan.getOutputByIndex(olapScan.getTable().getBaseIndexId()); List hashColumns = Lists.newArrayList(); - for (Slot slot : output) { - for (Column column : hashDistributionInfo.getDistributionColumns()) { + for (Column column : hashDistributionInfo.getDistributionColumns()) { + for (Slot slot : output) { if (((SlotReference) slot).getColumn().get().getNameWithoutMvPrefix() .equals(column.getName())) { hashColumns.add(slot.getExprId()); @@ -92,8 +92,8 @@ private DistributionSpec convertDistribution(LogicalOlapScan olapScan) { } } if (hashColumns.size() != hashDistributionInfo.getDistributionColumns().size()) { - for (Slot slot : baseOutput) { - for (Column column : hashDistributionInfo.getDistributionColumns()) { + for (Column column : hashDistributionInfo.getDistributionColumns()) { + for (Slot slot : baseOutput) { // If the length of the column in the bucket key changes after DDL, the length cannot be // determined. As a result, some bucket fields are lost in the query execution plan. // So here we use the column name to avoid this problem @@ -109,8 +109,8 @@ private DistributionSpec convertDistribution(LogicalOlapScan olapScan) { HashDistributionInfo hashDistributionInfo = (HashDistributionInfo) distributionInfo; List output = olapScan.getOutput(); List hashColumns = Lists.newArrayList(); - for (Slot slot : output) { - for (Column column : hashDistributionInfo.getDistributionColumns()) { + for (Column column : hashDistributionInfo.getDistributionColumns()) { + for (Slot slot : output) { // If the length of the column in the bucket key changes after DDL, the length cannot be // determined. As a result, some bucket fields are lost in the query execution plan. // So here we use the column name to avoid this problem diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/JoinUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/JoinUtils.java index 79462a9d8e7ebfe..df0665be3da6faa 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/JoinUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/JoinUtils.java @@ -32,6 +32,7 @@ import org.apache.doris.nereids.trees.expressions.MarkJoinSlotReference; import org.apache.doris.nereids.trees.expressions.Not; import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.expressions.functions.scalar.BitmapContains; import org.apache.doris.nereids.trees.plans.JoinType; import org.apache.doris.nereids.trees.plans.Plan; @@ -54,6 +55,7 @@ import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Set; import java.util.stream.Collectors; @@ -257,13 +259,14 @@ public static boolean shouldColocateJoin(AbstractPhysicalJoin conjuncts) { if (ConnectContext.get() == null || ConnectContext.get().getSessionVariable().isDisableColocatePlan()) { return false; @@ -285,12 +288,50 @@ public static boolean couldColocateJoin(DistributionSpecHash leftHashSpec, Distr boolean noNeedCheckColocateGroup = hitSameIndex && (leftTablePartitions.equals(rightTablePartitions)) && (leftTablePartitions.size() <= 1); ColocateTableIndex colocateIndex = Env.getCurrentColocateIndex(); - if (noNeedCheckColocateGroup - || (colocateIndex.isSameGroup(leftTableId, rightTableId) - && !colocateIndex.isGroupUnstable(colocateIndex.getGroup(leftTableId)))) { + if (noNeedCheckColocateGroup) { return true; } - return false; + if (!colocateIndex.isSameGroup(leftTableId, rightTableId) + || colocateIndex.isGroupUnstable(colocateIndex.getGroup(leftTableId))) { + return false; + } + + Set equalIndices = new HashSet<>(); + for (Expression expr : conjuncts) { + // only simple equal predicate can use colocate join + if (!(expr instanceof EqualPredicate)) { + return false; + } + Expression leftChild = ((EqualPredicate) expr).left(); + Expression rightChild = ((EqualPredicate) expr).right(); + if (!(leftChild instanceof SlotReference) || !(rightChild instanceof SlotReference)) { + return false; + } + + SlotReference leftSlot = (SlotReference) leftChild; + SlotReference rightSlot = (SlotReference) rightChild; + Integer leftIndex = null; + Integer rightIndex = null; + if (leftSlot.getTable().isPresent() && leftSlot.getTable().get().getId() == leftHashSpec.getTableId()) { + leftIndex = leftHashSpec.getExprIdToEquivalenceSet().get(leftSlot.getExprId()); + rightIndex = rightHashSpec.getExprIdToEquivalenceSet().get(rightSlot.getExprId()); + } else { + leftIndex = rightHashSpec.getExprIdToEquivalenceSet().get(leftSlot.getExprId()); + rightIndex = leftHashSpec.getExprIdToEquivalenceSet().get(rightSlot.getExprId()); + } + if (!Objects.equals(leftIndex, rightIndex)) { + return false; + } + if (leftIndex != null) { + equalIndices.add(leftIndex); + } + } + // on conditions must contain all distributed columns + if (equalIndices.containsAll(leftHashSpec.getExprIdToEquivalenceSet().values())) { + return true; + } else { + return false; + } } public static Set getJoinOutputExprIdSet(Plan left, Plan right) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java index 429cf1f03b58810..5046acfd5a4d03c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -574,6 +574,8 @@ public class SessionVariable implements Serializable, Writable { public static final String USE_MAX_LENGTH_OF_VARCHAR_IN_CTAS = "use_max_length_of_varchar_in_ctas"; + public static final String ENABLE_ES_PARALLEL_SCROLL = "enable_es_parallel_scroll"; + public static final List DEBUG_VARIABLES = ImmutableList.of( SKIP_DELETE_PREDICATE, SKIP_DELETE_BITMAP, @@ -1916,6 +1918,24 @@ public void setIgnoreShapePlanNodes(String ignoreShapePlanNodes) { }) public boolean useMaxLengthOfVarcharInCtas = true; + /** + * When enabling shard scroll, FE will plan scan ranges by shards of ES indices. + * Otherwise, FE will plan a single query to ES. + */ + @VariableMgr.VarAttr(name = ENABLE_ES_PARALLEL_SCROLL, description = { + "ES catalog 是否开启 shard 级别并发的 scroll 请求，默认开启。", + "Whether to enable shard-level parallel scroll requests for ES catalog, enabled by default." + }) + public boolean enableESParallelScroll = true; + + public void setEnableEsParallelScroll(boolean enableESParallelScroll) { + this.enableESParallelScroll = enableESParallelScroll; + } + + public boolean isEnableESParallelScroll() { + return enableESParallelScroll; + } + public boolean isEnableJoinSpill() { return enableJoinSpill; } diff --git a/regression-test/data/external_table_p0/es/test_es_query.out b/regression-test/data/external_table_p0/es/test_es_query.out index b84387fb542caf0..a2289bec1b49275 100644 --- a/regression-test/data/external_table_p0/es/test_es_query.out +++ b/regression-test/data/external_table_p0/es/test_es_query.out @@ -60,6 +60,482 @@ I'm not null or empty [{"name":"Andy","age":18},{"name":"Tim","age":28}] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] "Andy" "White" [{"name":"Andy","age":18},{"name":"Tim","age":28}] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] "Andy" "White" +-- !sql01 -- +["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [-1, 0, 1, 2] [0, 1, 2, 3] ["d", "e", "f"] [128, 129, -129, -130] ["192.168.0.1", "127.0.0.1"] string1 [1, 2, 3, 4] 2022-08-08 2022-08-08T12:10:10 text#1 ["2020-01-01", "2020-01-02"] 3.14 [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] ["a", "b", "c"] [{"name":"Andy","age":18},{"name":"Tim","age":28}] 2022-08-08T12:10:10 2022-08-08T12:10:10 2022-08-08T20:10:10 [1, -2, -3, 4] [1, 0, 1, 1] [32768, 32769, -32769, -32770] \N [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] + +-- !sql02 -- +["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [-1, 0, 1, 2] [0, 1, 2, 3] ["d", "e", "f"] [128, 129, -129, -130] ["192.168.0.1", "127.0.0.1"] string1 [1, 2, 3, 4] 2022-08-08 2022-08-08T12:10:10 text#1 ["2020-01-01", "2020-01-02"] 3.14 [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] ["a", "b", "c"] [{"name":"Andy","age":18},{"name":"Tim","age":28}] 2022-08-08T12:10:10 2022-08-08T12:10:10 2022-08-08T20:10:10 [1, -2, -3, 4] [1, 0, 1, 1] [32768, 32769, -32769, -32770] \N [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] + +-- !sql03 -- +2022-08-08 2022-08-08T12:10:10 2022-08-08T12:10:10 2022-08-08T04:10:10 2022-08-08T20:10:10 +2022-08-08 2022-08-09T12:10:10 2022-08-09T12:10:10 2022-08-09T12:10:10 2022-08-09T12:10:10 +2022-08-08 2022-08-10T12:10:10 2022-08-10T12:10:10 2022-08-10T04:10:10 2022-08-10T20:10:10 +2022-08-08 2022-08-11T12:10:10 2022-08-11T12:10:10 2022-08-11T12:10:10 2022-08-11T11:10:10 + +-- !sql04 -- +I'm not null or empty + +-- !sql05 -- + +I'm not null or empty + +-- !sql06 -- +\N +\N + +I'm not null or empty + +-- !sql07 -- +["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [-1, 0, 1, 2] [0, 1, 2, 3] ["d", "e", "f"] [128, 129, -129, -130] ["192.168.0.1", "127.0.0.1"] string1 [1, 2, 3, 4] 2022-08-08 2022-08-08T12:10:10 text#1 ["2020-01-01", "2020-01-02"] 3.14 [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] ["a", "b", "c"] [{"name":"Andy","age":18},{"name":"Tim","age":28}] 2022-08-08T12:10:10 2022-08-08T12:10:10 2022-08-08T20:10:10 [1, -2, -3, 4] [1, 0, 1, 1] [32768, 32769, -32769, -32770] \N [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] +["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [-1, 0, 1, 2] [0, 1, 2, 3] ["d", "e", "f"] [128, 129, -129, -130] ["192.168.0.1", "127.0.0.1"] string2 [1, 2, 3, 4] 2022-08-08 2022-08-09T12:10:10 text2 ["2020-01-01", "2020-01-02"] 4.0 [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] ["a", "b", "c"] [{"name":"Andy","age":18},{"name":"Tim","age":28}] 2022-08-09T12:10:10 2022-08-09T12:10:10 2022-08-09T12:10:10 [1, -2, -3, 4] [1, 0, 1, 1] [32768, 32769, -32769, -32770] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] +["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [-1, 0, 1, 2] [0, 1, 2, 3] ["d", "e", "f"] [128, 129, -129, -130] ["192.168.0.1", "127.0.0.1"] string3 [1, 2, 3, 4] 2022-08-08 2022-08-10T12:10:10 text3_4*5 ["2020-01-01", "2020-01-02"] 5.0 [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] ["a", "b", "c"] [{"name":"Andy","age":18},{"name":"Tim","age":28}] 2022-08-10T12:10:10 2022-08-10T12:10:10 2022-08-10T20:10:10 [1, -2, -3, 4] [1, 0, 1, 1] [32768, 32769, -32769, -32770] I'm not null or empty [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] +["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [-1, 0, 1, 2] [0, 1, 2, 3] ["d", "e", "f"] [128, 129, -129, -130] ["192.168.0.1", "127.0.0.1"] string4 [1, 2, 3, 4] 2022-08-08 2022-08-11T12:10:10 text3_4*5 ["2020-01-01", "2020-01-02"] 6.0 [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] ["a", "b", "c"] [{"name":"Andy","age":18},{"name":"Tim","age":28}] 2022-08-11T12:10:10 2022-08-11T12:10:10 2022-08-11T11:10:10 [1, -2, -3, 4] [1, 0, 1, 1] [32768, 32769, -32769, -32770] \N [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] + +-- !sql08 -- +[{"name":"Andy","age":18},{"name":"Tim","age":28}] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] "Andy" "White" +[{"name":"Andy","age":18},{"name":"Tim","age":28}] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] "Andy" "White" +[{"name":"Andy","age":18},{"name":"Tim","age":28}] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] "Andy" "White" +[{"name":"Andy","age":18},{"name":"Tim","age":28}] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] "Andy" "White" + +-- !sql20 -- +["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [-1, 0, 1, 2] [0, 1, 2, 3] ["d", "e", "f"] [128, 129, -129, -130] ["192.168.0.1", "127.0.0.1"] string1 [1, 2, 3, 4] 2022-08-08 2022-08-08T12:10:10 text#1 ["2020-01-01", "2020-01-02"] 3.14 [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] ["a", "b", "c"] [{"name":"Andy","age":18},{"name":"Tim","age":28}] 2022-08-08T12:10:10 2022-08-08T12:10:10 2022-08-08T20:10:10 [1, -2, -3, 4] [1, 0, 1, 1] [32768, 32769, -32769, -32770] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] + +-- !sql21 -- +["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [-1, 0, 1, 2] [0, 1, 2, 3] ["d", "e", "f"] [128, 129, -129, -130] ["192.168.0.1", "127.0.0.1"] string1 [1, 2, 3, 4] 2022-08-08 2022-08-08T12:10:10 text#1 ["2020-01-01", "2020-01-02"] 3.14 [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] ["a", "b", "c"] [{"name":"Andy","age":18},{"name":"Tim","age":28}] 2022-08-08T12:10:10 2022-08-08T12:10:10 2022-08-08T20:10:10 [1, -2, -3, 4] [1, 0, 1, 1] [32768, 32769, -32769, -32770] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] + +-- !sql22 -- +2022-08-08 2022-08-08T12:10:10 2022-08-08T12:10:10 2022-08-08T04:10:10 2022-08-08T20:10:10 +2022-08-08 2022-08-09T12:10:10 2022-08-09T12:10:10 2022-08-09T12:10:10 2022-08-09T12:10:10 +2022-08-08 2022-08-10T12:10:10 2022-08-10T12:10:10 2022-08-10T04:10:10 2022-08-10T20:10:10 +2022-08-08 2022-08-11T12:10:10 2022-08-11T12:10:10 2022-08-11T12:10:10 2022-08-11T11:10:10 + +-- !sql23 -- +["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [-1, 0, 1, 2] [0, 1, 2, 3] ["d", "e", "f"] [128, 129, -129, -130] ["192.168.0.1", "127.0.0.1"] string1 [1, 2, 3, 4] 2022-08-08 2022-08-08T12:10:10 text#1 ["2020-01-01", "2020-01-02"] 3.14 [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] ["a", "b", "c"] [{"name":"Andy","age":18},{"name":"Tim","age":28}] 2022-08-08T12:10:10 2022-08-08T12:10:10 2022-08-08T20:10:10 [1, -2, -3, 4] [1, 0, 1, 1] [32768, 32769, -32769, -32770] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] +["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [-1, 0, 1, 2] [0, 1, 2, 3] ["d", "e", "f"] [128, 129, -129, -130] ["192.168.0.1", "127.0.0.1"] string2 [1, 2, 3, 4] 2022-08-08 2022-08-09T12:10:10 text2 ["2020-01-01", "2020-01-02"] 4.0 [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] ["a", "b", "c"] [{"name":"Andy","age":18},{"name":"Tim","age":28}] 2022-08-09T12:10:10 2022-08-09T12:10:10 2022-08-09T12:10:10 [1, -2, -3, 4] [1, 0, 1, 1] [32768, 32769, -32769, -32770] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] +["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [-1, 0, 1, 2] [0, 1, 2, 3] ["d", "e", "f"] [128, 129, -129, -130] ["192.168.0.1", "127.0.0.1"] string3 [1, 2, 3, 4] 2022-08-08 2022-08-10T12:10:10 text3_4*5 ["2020-01-01", "2020-01-02"] 5.0 [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] ["a", "b", "c"] [{"name":"Andy","age":18},{"name":"Tim","age":28}] 2022-08-10T12:10:10 2022-08-10T12:10:10 2022-08-10T20:10:10 [1, -2, -3, 4] [1, 0, 1, 1] [32768, 32769, -32769, -32770] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] +["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [-1, 0, 1, 2] [0, 1, 2, 3] ["d", "e", "f"] [128, 129, -129, -130] ["192.168.0.1", "127.0.0.1"] string4 [1, 2, 3, 4] 2022-08-08 2022-08-11T12:10:10 text3_4*5 ["2020-01-01", "2020-01-02"] 6.0 [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] ["a", "b", "c"] [{"name":"Andy","age":18},{"name":"Tim","age":28}] 2022-08-11T12:10:10 2022-08-11T12:10:10 2022-08-11T11:10:10 [1, -2, -3, 4] [1, 0, 1, 1] [32768, 32769, -32769, -32770] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] + +-- !sql24 -- +[{"name":"Andy","age":18},{"name":"Tim","age":28}] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] "Andy" "White" +[{"name":"Andy","age":18},{"name":"Tim","age":28}] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] "Andy" "White" +[{"name":"Andy","age":18},{"name":"Tim","age":28}] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] "Andy" "White" +[{"name":"Andy","age":18},{"name":"Tim","age":28}] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] "Andy" "White" + +-- !sql_5_02 -- +[1, 0, 1, 1] [1, -2, -3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ["192.168.0.1", "127.0.0.1"] ["a", "b", "c"] [-1, 0, 1, 2] [{"name":"Andy","age":18},{"name":"Tim","age":28}] [1, 2, 3, 4] [128, 129, -129, -130] ["d", "e", "f"] [0, 1, 2, 3] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] \N string1 text#1 3.14 2022-08-08T00:00 12345 2022-08-08T20:10:10 + +-- !sql_5_03 -- +[1, 0, 1, 1] [1, -2, -3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ["192.168.0.1", "127.0.0.1"] ["a", "b", "c"] [-1, 0, 1, 2] [{"name":"Andy","age":18},{"name":"Tim","age":28}] [1, 2, 3, 4] [128, 129, -129, -130] ["d", "e", "f"] [0, 1, 2, 3] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] \N string1 text#1 3.14 2022-08-08T00:00 12345 2022-08-08T20:10:10 +[1, 0, 1, 1] [1, -2, -3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ["192.168.0.1", "127.0.0.1"] ["a", "b", "c"] [-1, 0, 1, 2] [{"name":"Andy","age":18},{"name":"Tim","age":28}] [1, 2, 3, 4] [128, 129, -129, -130] ["d", "e", "f"] [0, 1, 2, 3] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] string2 text2 4.0 2022-08-08T00:00 2222 2022-08-08T12:10:10 +[1, 0, 1, 1] [1, -2, -3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ["192.168.0.1", "127.0.0.1"] ["a", "b", "c"] [-1, 0, 1, 2] [{"name":"Andy","age":18},{"name":"Tim","age":28}] [1, 2, 3, 4] [128, 129, -129, -130] ["d", "e", "f"] [0, 1, 2, 3] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] I'm not null or empty string3 text3_4*5 5.0 2022-08-08T00:00 3333 2022-08-08T20:10:10 + +-- !sql_5_04 -- +[1, 0, 1, 1] [1, -2, -3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ["192.168.0.1", "127.0.0.1"] ["a", "b", "c"] [-1, 0, 1, 2] [{"name":"Andy","age":18},{"name":"Tim","age":28}] [1, 2, 3, 4] [128, 129, -129, -130] ["d", "e", "f"] [0, 1, 2, 3] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] string2 text2 4.0 2022-08-08T00:00 2222 2022-08-08T12:10:10 + +-- !sql_5_05 -- +true 1 128 32768 -1 0 1.0 1.0 1.0 1.0 2020-01-01 2020-01-01T12:00 a d 192.168.0.1 {"name":"Andy","age":18} +true 1 128 32768 -1 0 1.0 1.0 1.0 1.0 2020-01-01 2020-01-01T12:00 a d 192.168.0.1 {"name":"Andy","age":18} +true 1 128 32768 -1 0 1.0 1.0 1.0 1.0 2020-01-01 2020-01-01T12:00 a d 192.168.0.1 {"name":"Andy","age":18} + +-- !sql_5_06 -- +true 1 128 32768 -1 0 1.0 1.0 1.0 1.0 2020-01-01 2020-01-01T12:00 a d 192.168.0.1 {"name":"Andy","age":18} +true 1 128 32768 -1 0 1.0 1.0 1.0 1.0 2020-01-01 2020-01-01T12:00 a d 192.168.0.1 {"name":"Andy","age":18} +true 1 128 32768 -1 0 1.0 1.0 1.0 1.0 2020-01-01 2020-01-01T12:00 a d 192.168.0.1 {"name":"Andy","age":18} + +-- !sql_5_07 -- +[1, 0, 1, 1] [1, -2, -3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ["192.168.0.1", "127.0.0.1"] ["a", "b", "c"] [-1, 0, 1, 2] [{"name":"Andy","age":18},{"name":"Tim","age":28}] [1, 2, 3, 4] [128, 129, -129, -130] ["d", "e", "f"] [0, 1, 2, 3] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] \N string1 text#1 3.14 2022-08-08T00:00 12345 2022-08-08T20:10:10 + +-- !sql_5_08 -- +[1, 0, 1, 1] [1, -2, -3, 4] [128, 129, -129, -130] [32768, 32769, -32769, -32770] [-1, 0, 1, 2] [0, 1, 2, 3] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [1, 2, 3, 4] [1, 2, 3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] ["a", "b", "c"] ["d", "e", "f"] ["192.168.0.1", "127.0.0.1"] [{"name":"Andy","age":18},{"name":"Tim","age":28}] +[1, 0, 1, 1] [1, -2, -3, 4] [128, 129, -129, -130] [32768, 32769, -32769, -32770] [-1, 0, 1, 2] [0, 1, 2, 3] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [1, 2, 3, 4] [1, 2, 3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] ["a", "b", "c"] ["d", "e", "f"] ["192.168.0.1", "127.0.0.1"] [{"name":"Andy","age":18},{"name":"Tim","age":28}] +[1, 0, 1, 1] [1, -2, -3, 4] [128, 129, -129, -130] [32768, 32769, -32769, -32770] [-1, 0, 1, 2] [0, 1, 2, 3] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [1, 2, 3, 4] [1, 2, 3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] ["a", "b", "c"] ["d", "e", "f"] ["192.168.0.1", "127.0.0.1"] [{"name":"Andy","age":18},{"name":"Tim","age":28}] + +-- !sql_5_09 -- +[1, 0, 1, 1] [1, -2, -3, 4] [128, 129, -129, -130] [32768, 32769, -32769, -32770] [-1, 0, 1, 2] [0, 1, 2, 3] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [1, 2, 3, 4] [1, 2, 3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] ["a", "b", "c"] ["d", "e", "f"] ["192.168.0.1", "127.0.0.1"] [{"name":"Andy","age":18},{"name":"Tim","age":28}] +[1, 0, 1, 1] [1, -2, -3, 4] [128, 129, -129, -130] [32768, 32769, -32769, -32770] [-1, 0, 1, 2] [0, 1, 2, 3] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [1, 2, 3, 4] [1, 2, 3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] ["a", "b", "c"] ["d", "e", "f"] ["192.168.0.1", "127.0.0.1"] [{"name":"Andy","age":18},{"name":"Tim","age":28}] +[1, 0, 1, 1] [1, -2, -3, 4] [128, 129, -129, -130] [32768, 32769, -32769, -32770] [-1, 0, 1, 2] [0, 1, 2, 3] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [1, 2, 3, 4] [1, 2, 3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] ["a", "b", "c"] ["d", "e", "f"] ["192.168.0.1", "127.0.0.1"] [{"name":"Andy","age":18},{"name":"Tim","age":28}] + +-- !sql_5_10 -- +[1, 0, 1, 1] [1, -2, -3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ["192.168.0.1", "127.0.0.1"] ["a", "b", "c"] [-1, 0, 1, 2] [{"name":"Andy","age":18},{"name":"Tim","age":28}] [1, 2, 3, 4] [128, 129, -129, -130] ["d", "e", "f"] [0, 1, 2, 3] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] \N string1 text#1 3.14 2022-08-08T00:00 12345 2022-08-08T20:10:10 + +-- !sql_5_11 -- +[1, 0, 1, 1] [1, -2, -3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ["192.168.0.1", "127.0.0.1"] ["a", "b", "c"] [-1, 0, 1, 2] [{"name":"Andy","age":18},{"name":"Tim","age":28}] [1, 2, 3, 4] [128, 129, -129, -130] ["d", "e", "f"] [0, 1, 2, 3] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] string2 text2 4.0 2022-08-08T00:00 2222 2022-08-08T12:10:10 + +-- !sql_5_12 -- +[1, 0, 1, 1] [1, -2, -3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ["192.168.0.1", "127.0.0.1"] ["a", "b", "c"] [-1, 0, 1, 2] [{"name":"Andy","age":18},{"name":"Tim","age":28}] [1, 2, 3, 4] [128, 129, -129, -130] ["d", "e", "f"] [0, 1, 2, 3] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] I'm not null or empty string3 text3_4*5 5.0 2022-08-08T00:00 3333 2022-08-08T20:10:10 + +-- !sql_5_13 -- +2022-08-08T20:10:10 + +-- !sql_5_14 -- +2022-08-08T12:10:10 + +-- !sql_5_15 -- +2022-08-08T20:10:10 + +-- !sql_5_16 -- +I'm not null or empty + +-- !sql_5_17 -- + +I'm not null or empty + +-- !sql_5_18 -- +I'm not null or empty + +-- !sql_5_19 -- +[1, 0, 1, 1] [1, -2, -3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ["192.168.0.1", "127.0.0.1"] ["a", "b", "c"] [-1, 0, 1, 2] [{"name":"Andy","age":18},{"name":"Tim","age":28}] [1, 2, 3, 4] [128, 129, -129, -130] ["d", "e", "f"] [0, 1, 2, 3] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] \N string1 text#1 3.14 2022-08-08T00:00 12345 2022-08-08T20:10:10 +[1, 0, 1, 1] [1, -2, -3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ["192.168.0.1", "127.0.0.1"] ["a", "b", "c"] [-1, 0, 1, 2] [{"name":"Andy","age":18},{"name":"Tim","age":28}] [1, 2, 3, 4] [128, 129, -129, -130] ["d", "e", "f"] [0, 1, 2, 3] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] string2 text2 4.0 2022-08-08T00:00 2222 2022-08-08T12:10:10 +[1, 0, 1, 1] [1, -2, -3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ["192.168.0.1", "127.0.0.1"] ["a", "b", "c"] [-1, 0, 1, 2] [{"name":"Andy","age":18},{"name":"Tim","age":28}] [1, 2, 3, 4] [128, 129, -129, -130] ["d", "e", "f"] [0, 1, 2, 3] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] I'm not null or empty string3 text3_4*5 5.0 2022-08-08T00:00 3333 2022-08-08T20:10:10 + +-- !sql_5_20 -- +[{"name":"Andy","age":18},{"name":"Tim","age":28}] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] "Andy" "White" +[{"name":"Andy","age":18},{"name":"Tim","age":28}] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] "Andy" "White" +[{"name":"Andy","age":18},{"name":"Tim","age":28}] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] "Andy" "White" + +-- !sql_6_02 -- +[1, 0, 1, 1] [1, -2, -3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ["192.168.0.1", "127.0.0.1"] ["a", "b", "c"] [-1, 0, 1, 2] [{"name":"Andy","age":18},{"name":"Tim","age":28}] [1, 2, 3, 4] [128, 129, -129, -130] ["d", "e", "f"] [0, 1, 2, 3] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] \N string1 text#1 3.14 2022-08-08T00:00 12345 2022-08-08T20:10:10 + +-- !sql_6_03 -- +[1, 0, 1, 1] [1, -2, -3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ["192.168.0.1", "127.0.0.1"] ["a", "b", "c"] [-1, 0, 1, 2] [{"name":"Andy","age":18},{"name":"Tim","age":28}] [1, 2, 3, 4] [128, 129, -129, -130] ["d", "e", "f"] [0, 1, 2, 3] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] \N string1 text#1 3.14 2022-08-08T00:00 12345 2022-08-08T20:10:10 +[1, 0, 1, 1] [1, -2, -3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ["192.168.0.1", "127.0.0.1"] ["a", "b", "c"] [-1, 0, 1, 2] [{"name":"Andy","age":18},{"name":"Tim","age":28}] [1, 2, 3, 4] [128, 129, -129, -130] ["d", "e", "f"] [0, 1, 2, 3] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] string2 text2 4.0 2022-08-08T00:00 2222 2022-08-08T12:10:10 +[1, 0, 1, 1] [1, -2, -3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ["192.168.0.1", "127.0.0.1"] ["a", "b", "c"] [-1, 0, 1, 2] [{"name":"Andy","age":18},{"name":"Tim","age":28}] [1, 2, 3, 4] [128, 129, -129, -130] ["d", "e", "f"] [0, 1, 2, 3] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] I'm not null or empty string3 text3_4*5 5.0 2022-08-08T00:00 3333 2022-08-08T20:10:10 + +-- !sql_6_04 -- +[1, 0, 1, 1] [1, -2, -3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ["192.168.0.1", "127.0.0.1"] ["a", "b", "c"] [-1, 0, 1, 2] [{"name":"Andy","age":18},{"name":"Tim","age":28}] [1, 2, 3, 4] [128, 129, -129, -130] ["d", "e", "f"] [0, 1, 2, 3] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] string2 text2 4.0 2022-08-08T00:00 2222 2022-08-08T12:10:10 + +-- !sql_6_05 -- +true 1 128 32768 -1 0 1.0 1.0 1.0 1.0 2020-01-01 2020-01-01T12:00 a d 192.168.0.1 {"name":"Andy","age":18} +true 1 128 32768 -1 0 1.0 1.0 1.0 1.0 2020-01-01 2020-01-01T12:00 a d 192.168.0.1 {"name":"Andy","age":18} +true 1 128 32768 -1 0 1.0 1.0 1.0 1.0 2020-01-01 2020-01-01T12:00 a d 192.168.0.1 {"name":"Andy","age":18} + +-- !sql_6_06 -- +true 1 128 32768 -1 0 1.0 1.0 1.0 1.0 2020-01-01 2020-01-01T12:00 a d 192.168.0.1 {"name":"Andy","age":18} +true 1 128 32768 -1 0 1.0 1.0 1.0 1.0 2020-01-01 2020-01-01T12:00 a d 192.168.0.1 {"name":"Andy","age":18} +true 1 128 32768 -1 0 1.0 1.0 1.0 1.0 2020-01-01 2020-01-01T12:00 a d 192.168.0.1 {"name":"Andy","age":18} + +-- !sql_6_07 -- +[1, 0, 1, 1] [1, -2, -3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ["192.168.0.1", "127.0.0.1"] ["a", "b", "c"] [-1, 0, 1, 2] [{"name":"Andy","age":18},{"name":"Tim","age":28}] [1, 2, 3, 4] [128, 129, -129, -130] ["d", "e", "f"] [0, 1, 2, 3] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] \N string1 text#1 3.14 2022-08-08T00:00 12345 2022-08-08T20:10:10 + +-- !sql_6_08 -- +[1, 0, 1, 1] [1, -2, -3, 4] [128, 129, -129, -130] [32768, 32769, -32769, -32770] [-1, 0, 1, 2] [0, 1, 2, 3] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [1, 2, 3, 4] [1, 2, 3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] ["a", "b", "c"] ["d", "e", "f"] ["192.168.0.1", "127.0.0.1"] [{"name":"Andy","age":18},{"name":"Tim","age":28}] +[1, 0, 1, 1] [1, -2, -3, 4] [128, 129, -129, -130] [32768, 32769, -32769, -32770] [-1, 0, 1, 2] [0, 1, 2, 3] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [1, 2, 3, 4] [1, 2, 3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] ["a", "b", "c"] ["d", "e", "f"] ["192.168.0.1", "127.0.0.1"] [{"name":"Andy","age":18},{"name":"Tim","age":28}] +[1, 0, 1, 1] [1, -2, -3, 4] [128, 129, -129, -130] [32768, 32769, -32769, -32770] [-1, 0, 1, 2] [0, 1, 2, 3] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [1, 2, 3, 4] [1, 2, 3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] ["a", "b", "c"] ["d", "e", "f"] ["192.168.0.1", "127.0.0.1"] [{"name":"Andy","age":18},{"name":"Tim","age":28}] + +-- !sql_6_09 -- +[1, 0, 1, 1] [1, -2, -3, 4] [128, 129, -129, -130] [32768, 32769, -32769, -32770] [-1, 0, 1, 2] [0, 1, 2, 3] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [1, 2, 3, 4] [1, 2, 3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] ["a", "b", "c"] ["d", "e", "f"] ["192.168.0.1", "127.0.0.1"] [{"name":"Andy","age":18},{"name":"Tim","age":28}] +[1, 0, 1, 1] [1, -2, -3, 4] [128, 129, -129, -130] [32768, 32769, -32769, -32770] [-1, 0, 1, 2] [0, 1, 2, 3] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [1, 2, 3, 4] [1, 2, 3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] ["a", "b", "c"] ["d", "e", "f"] ["192.168.0.1", "127.0.0.1"] [{"name":"Andy","age":18},{"name":"Tim","age":28}] +[1, 0, 1, 1] [1, -2, -3, 4] [128, 129, -129, -130] [32768, 32769, -32769, -32770] [-1, 0, 1, 2] [0, 1, 2, 3] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [1, 2, 3, 4] [1, 2, 3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] ["a", "b", "c"] ["d", "e", "f"] ["192.168.0.1", "127.0.0.1"] [{"name":"Andy","age":18},{"name":"Tim","age":28}] + +-- !sql_6_10 -- +[1, 0, 1, 1] [1, -2, -3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ["192.168.0.1", "127.0.0.1"] ["a", "b", "c"] [-1, 0, 1, 2] [{"name":"Andy","age":18},{"name":"Tim","age":28}] [1, 2, 3, 4] [128, 129, -129, -130] ["d", "e", "f"] [0, 1, 2, 3] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] \N string1 text#1 3.14 2022-08-08T00:00 12345 2022-08-08T20:10:10 + +-- !sql_6_11 -- +[1, 0, 1, 1] [1, -2, -3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ["192.168.0.1", "127.0.0.1"] ["a", "b", "c"] [-1, 0, 1, 2] [{"name":"Andy","age":18},{"name":"Tim","age":28}] [1, 2, 3, 4] [128, 129, -129, -130] ["d", "e", "f"] [0, 1, 2, 3] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] string2 text2 4.0 2022-08-08T00:00 2222 2022-08-08T12:10:10 + +-- !sql_6_12 -- +[1, 0, 1, 1] [1, -2, -3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ["192.168.0.1", "127.0.0.1"] ["a", "b", "c"] [-1, 0, 1, 2] [{"name":"Andy","age":18},{"name":"Tim","age":28}] [1, 2, 3, 4] [128, 129, -129, -130] ["d", "e", "f"] [0, 1, 2, 3] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] I'm not null or empty string3 text3_4*5 5.0 2022-08-08T00:00 3333 2022-08-08T20:10:10 + +-- !sql_6_13 -- +2022-08-08T20:10:10 + +-- !sql_6_14 -- +2022-08-08T12:10:10 + +-- !sql_6_15 -- +2022-08-08T20:10:10 + +-- !sql_6_16 -- +I'm not null or empty + +-- !sql_6_17 -- + +I'm not null or empty + +-- !sql_6_18 -- +I'm not null or empty + +-- !sql_6_19 -- +[1, 0, 1, 1] [1, -2, -3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ["192.168.0.1", "127.0.0.1"] ["a", "b", "c"] [-1, 0, 1, 2] [{"name":"Andy","age":18},{"name":"Tim","age":28}] [1, 2, 3, 4] [128, 129, -129, -130] ["d", "e", "f"] [0, 1, 2, 3] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] \N string1 text#1 3.14 2022-08-08T00:00 12345 2022-08-08T20:10:10 +[1, 0, 1, 1] [1, -2, -3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ["192.168.0.1", "127.0.0.1"] ["a", "b", "c"] [-1, 0, 1, 2] [{"name":"Andy","age":18},{"name":"Tim","age":28}] [1, 2, 3, 4] [128, 129, -129, -130] ["d", "e", "f"] [0, 1, 2, 3] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] string2 text2 4.0 2022-08-08T00:00 2222 2022-08-08T12:10:10 +[1, 0, 1, 1] [1, -2, -3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ["192.168.0.1", "127.0.0.1"] ["a", "b", "c"] [-1, 0, 1, 2] [{"name":"Andy","age":18},{"name":"Tim","age":28}] [1, 2, 3, 4] [128, 129, -129, -130] ["d", "e", "f"] [0, 1, 2, 3] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] I'm not null or empty string3 text3_4*5 5.0 2022-08-08T00:00 3333 2022-08-08T20:10:10 + +-- !sql_6_20 -- +[{"name":"Andy","age":18},{"name":"Tim","age":28}] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] "Andy" "White" +[{"name":"Andy","age":18},{"name":"Tim","age":28}] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] "Andy" "White" +[{"name":"Andy","age":18},{"name":"Tim","age":28}] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] "Andy" "White" + +-- !sql_7_02 -- +[1, 0, 1, 1] [1, -2, -3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ["192.168.0.1", "127.0.0.1"] ["a", "b", "c"] [-1, 0, 1, 2] [{"name":"Andy","age":18},{"name":"Tim","age":28}] [1, 2, 3, 4] [128, 129, -129, -130] ["d", "e", "f"] [0, 1, 2, 3] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] debug \N This string can be quite lengthy string1 2022-08-08T20:10:10 text#1 3.14 2022-08-08T00:00 2022-08-08T12:10:10 1659931810000 2022-08-08T12:10:10 2022-08-08T20:10:10 12345 + +-- !sql_7_03 -- +[1, 0, 1, 1] [1, -2, -3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ["192.168.0.1", "127.0.0.1"] ["a", "b", "c"] [-1, 0, 1, 2] [{"name":"Andy","age":18},{"name":"Tim","age":28}] [1, 2, 3, 4] [128, 129, -129, -130] ["d", "e", "f"] [0, 1, 2, 3] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] \N \N \N string4 2022-08-08T20:10:10 text3_4*5 6.0 2022-08-08T00:00 2022-08-11T12:10:10 1660191010000 2022-08-11T12:10:10 2022-08-11T11:10:10 4444 +[1, 0, 1, 1] [1, -2, -3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ["192.168.0.1", "127.0.0.1"] ["a", "b", "c"] [-1, 0, 1, 2] [{"name":"Andy","age":18},{"name":"Tim","age":28}] [1, 2, 3, 4] [128, 129, -129, -130] ["d", "e", "f"] [0, 1, 2, 3] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] \N \N string2 2022-08-08T12:10:10 text2 4.0 2022-08-08T00:00 2022-08-09T12:10:10 1660018210000 2022-08-09T12:10:10 2022-08-09T12:10:10 2222 +[1, 0, 1, 1] [1, -2, -3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ["192.168.0.1", "127.0.0.1"] ["a", "b", "c"] [-1, 0, 1, 2] [{"name":"Andy","age":18},{"name":"Tim","age":28}] [1, 2, 3, 4] [128, 129, -129, -130] ["d", "e", "f"] [0, 1, 2, 3] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] \N I'm not null or empty \N string3 2022-08-09T00:40:10 text3_4*5 5.0 2022-08-08T00:00 2022-08-10T12:10:10 1660104610000 2022-08-10T12:10:10 2022-08-10T20:10:10 3333 +[1, 0, 1, 1] [1, -2, -3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ["192.168.0.1", "127.0.0.1"] ["a", "b", "c"] [-1, 0, 1, 2] [{"name":"Andy","age":18},{"name":"Tim","age":28}] [1, 2, 3, 4] [128, 129, -129, -130] ["d", "e", "f"] [0, 1, 2, 3] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] debug \N This string can be quite lengthy string1 2022-08-08T20:10:10 text#1 3.14 2022-08-08T00:00 2022-08-08T12:10:10 1659931810000 2022-08-08T12:10:10 2022-08-08T20:10:10 12345 + +-- !sql_7_04 -- +[1, 0, 1, 1] [1, -2, -3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ["192.168.0.1", "127.0.0.1"] ["a", "b", "c"] [-1, 0, 1, 2] [{"name":"Andy","age":18},{"name":"Tim","age":28}] [1, 2, 3, 4] [128, 129, -129, -130] ["d", "e", "f"] [0, 1, 2, 3] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] \N \N string2 2022-08-08T12:10:10 text2 4.0 2022-08-08T00:00 2022-08-09T12:10:10 1660018210000 2022-08-09T12:10:10 2022-08-09T12:10:10 2222 + +-- !sql_7_05 -- +true 1 128 32768 -1 0 1.0 1.0 1.0 1.0 2020-01-01 2020-01-01T12:00 a d 192.168.0.1 {"name":"Andy","age":18} +true 1 128 32768 -1 0 1.0 1.0 1.0 1.0 2020-01-01 2020-01-01T12:00 a d 192.168.0.1 {"name":"Andy","age":18} +true 1 128 32768 -1 0 1.0 1.0 1.0 1.0 2020-01-01 2020-01-01T12:00 a d 192.168.0.1 {"name":"Andy","age":18} +true 1 128 32768 -1 0 1.0 1.0 1.0 1.0 2020-01-01 2020-01-01T12:00 a d 192.168.0.1 {"name":"Andy","age":18} + +-- !sql_7_06 -- +true 1 128 32768 -1 0 1.0 1.0 1.0 1.0 2020-01-01 2020-01-01T12:00 a d 192.168.0.1 {"name":"Andy","age":18} +true 1 128 32768 -1 0 1.0 1.0 1.0 1.0 2020-01-01 2020-01-01T12:00 a d 192.168.0.1 {"name":"Andy","age":18} +true 1 128 32768 -1 0 1.0 1.0 1.0 1.0 2020-01-01 2020-01-01T12:00 a d 192.168.0.1 {"name":"Andy","age":18} +true 1 128 32768 -1 0 1.0 1.0 1.0 1.0 2020-01-01 2020-01-01T12:00 a d 192.168.0.1 {"name":"Andy","age":18} +true 1 128 32768 -1 0 1.0 1.0 1.0 1.0 2020-01-01 2020-01-01T12:00 a d 192.168.0.1 {"name":"Andy","age":18} +true 1 128 32768 -1 0 1.0 1.0 1.0 1.0 2020-01-01 2020-01-01T12:00 a d 192.168.0.1 {"name":"Andy","age":18} +true 1 128 32768 -1 0 1.0 1.0 1.0 1.0 2020-01-01 2020-01-01T12:00 a d 192.168.0.1 {"name":"Andy","age":18} +true 1 128 32768 -1 0 1.0 1.0 1.0 1.0 2020-01-01 2020-01-01T12:00 a d 192.168.0.1 {"name":"Andy","age":18} + +-- !sql_7_07 -- +[1, 0, 1, 1] [1, -2, -3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ["192.168.0.1", "127.0.0.1"] ["a", "b", "c"] [-1, 0, 1, 2] [{"name":"Andy","age":18},{"name":"Tim","age":28}] [1, 2, 3, 4] [128, 129, -129, -130] ["d", "e", "f"] [0, 1, 2, 3] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] debug \N This string can be quite lengthy string1 2022-08-08T20:10:10 text#1 3.14 2022-08-08T00:00 2022-08-08T12:10:10 1659931810000 2022-08-08T12:10:10 2022-08-08T20:10:10 12345 + +-- !sql_7_08 -- +[1, 0, 1, 1] [1, -2, -3, 4] [128, 129, -129, -130] [32768, 32769, -32769, -32770] [-1, 0, 1, 2] [0, 1, 2, 3] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [1, 2, 3, 4] [1, 2, 3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] ["a", "b", "c"] ["d", "e", "f"] ["192.168.0.1", "127.0.0.1"] [{"name":"Andy","age":18},{"name":"Tim","age":28}] +[1, 0, 1, 1] [1, -2, -3, 4] [128, 129, -129, -130] [32768, 32769, -32769, -32770] [-1, 0, 1, 2] [0, 1, 2, 3] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [1, 2, 3, 4] [1, 2, 3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] ["a", "b", "c"] ["d", "e", "f"] ["192.168.0.1", "127.0.0.1"] [{"name":"Andy","age":18},{"name":"Tim","age":28}] +[1, 0, 1, 1] [1, -2, -3, 4] [128, 129, -129, -130] [32768, 32769, -32769, -32770] [-1, 0, 1, 2] [0, 1, 2, 3] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [1, 2, 3, 4] [1, 2, 3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] ["a", "b", "c"] ["d", "e", "f"] ["192.168.0.1", "127.0.0.1"] [{"name":"Andy","age":18},{"name":"Tim","age":28}] +[1, 0, 1, 1] [1, -2, -3, 4] [128, 129, -129, -130] [32768, 32769, -32769, -32770] [-1, 0, 1, 2] [0, 1, 2, 3] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [1, 2, 3, 4] [1, 2, 3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] ["a", "b", "c"] ["d", "e", "f"] ["192.168.0.1", "127.0.0.1"] [{"name":"Andy","age":18},{"name":"Tim","age":28}] + +-- !sql_7_09 -- +[1, 0, 1, 1] [1, -2, -3, 4] [128, 129, -129, -130] [32768, 32769, -32769, -32770] [-1, 0, 1, 2] [0, 1, 2, 3] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [1, 2, 3, 4] [1, 2, 3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] ["a", "b", "c"] ["d", "e", "f"] ["192.168.0.1", "127.0.0.1"] [{"name":"Andy","age":18},{"name":"Tim","age":28}] +[1, 0, 1, 1] [1, -2, -3, 4] [128, 129, -129, -130] [32768, 32769, -32769, -32770] [-1, 0, 1, 2] [0, 1, 2, 3] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [1, 2, 3, 4] [1, 2, 3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] ["a", "b", "c"] ["d", "e", "f"] ["192.168.0.1", "127.0.0.1"] [{"name":"Andy","age":18},{"name":"Tim","age":28}] +[1, 0, 1, 1] [1, -2, -3, 4] [128, 129, -129, -130] [32768, 32769, -32769, -32770] [-1, 0, 1, 2] [0, 1, 2, 3] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [1, 2, 3, 4] [1, 2, 3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] ["a", "b", "c"] ["d", "e", "f"] ["192.168.0.1", "127.0.0.1"] [{"name":"Andy","age":18},{"name":"Tim","age":28}] +[1, 0, 1, 1] [1, -2, -3, 4] [128, 129, -129, -130] [32768, 32769, -32769, -32770] [-1, 0, 1, 2] [0, 1, 2, 3] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [1, 2, 3, 4] [1, 2, 3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] ["a", "b", "c"] ["d", "e", "f"] ["192.168.0.1", "127.0.0.1"] [{"name":"Andy","age":18},{"name":"Tim","age":28}] +[1, 0, 1, 1] [1, -2, -3, 4] [128, 129, -129, -130] [32768, 32769, -32769, -32770] [-1, 0, 1, 2] [0, 1, 2, 3] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [1, 2, 3, 4] [1, 2, 3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] ["a", "b", "c"] ["d", "e", "f"] ["192.168.0.1", "127.0.0.1"] [{"name":"Andy","age":18},{"name":"Tim","age":28}] +[1, 0, 1, 1] [1, -2, -3, 4] [128, 129, -129, -130] [32768, 32769, -32769, -32770] [-1, 0, 1, 2] [0, 1, 2, 3] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [1, 2, 3, 4] [1, 2, 3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] ["a", "b", "c"] ["d", "e", "f"] ["192.168.0.1", "127.0.0.1"] [{"name":"Andy","age":18},{"name":"Tim","age":28}] +[1, 0, 1, 1] [1, -2, -3, 4] [128, 129, -129, -130] [32768, 32769, -32769, -32770] [-1, 0, 1, 2] [0, 1, 2, 3] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [1, 2, 3, 4] [1, 2, 3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] ["a", "b", "c"] ["d", "e", "f"] ["192.168.0.1", "127.0.0.1"] [{"name":"Andy","age":18},{"name":"Tim","age":28}] +[1, 0, 1, 1] [1, -2, -3, 4] [128, 129, -129, -130] [32768, 32769, -32769, -32770] [-1, 0, 1, 2] [0, 1, 2, 3] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [1, 2, 3, 4] [1, 2, 3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] ["a", "b", "c"] ["d", "e", "f"] ["192.168.0.1", "127.0.0.1"] [{"name":"Andy","age":18},{"name":"Tim","age":28}] + +-- !sql_7_10 -- +value1 value2 + +-- !sql_7_11 -- +[1, 0, 1, 1] [1, -2, -3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ["192.168.0.1", "127.0.0.1"] ["a", "b", "c"] [-1, 0, 1, 2] [{"name":"Andy","age":18},{"name":"Tim","age":28}] [1, 2, 3, 4] [128, 129, -129, -130] ["d", "e", "f"] [0, 1, 2, 3] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] debug \N This string can be quite lengthy string1 2022-08-08T20:10:10 text#1 3.14 2022-08-08T00:00 2022-08-08T12:10:10 1659931810000 2022-08-08T12:10:10 2022-08-08T20:10:10 12345 + +-- !sql_7_12 -- +[1, 0, 1, 1] [1, -2, -3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ["192.168.0.1", "127.0.0.1"] ["a", "b", "c"] [-1, 0, 1, 2] [{"name":"Andy","age":18},{"name":"Tim","age":28}] [1, 2, 3, 4] [128, 129, -129, -130] ["d", "e", "f"] [0, 1, 2, 3] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] \N \N string2 2022-08-08T12:10:10 text2 4.0 2022-08-08T00:00 2022-08-09T12:10:10 1660018210000 2022-08-09T12:10:10 2022-08-09T12:10:10 2222 + +-- !sql_7_13 -- +[1, 0, 1, 1] [1, -2, -3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ["192.168.0.1", "127.0.0.1"] ["a", "b", "c"] [-1, 0, 1, 2] [{"name":"Andy","age":18},{"name":"Tim","age":28}] [1, 2, 3, 4] [128, 129, -129, -130] ["d", "e", "f"] [0, 1, 2, 3] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] \N I'm not null or empty \N string3 2022-08-09T00:40:10 text3_4*5 5.0 2022-08-08T00:00 2022-08-10T12:10:10 1660104610000 2022-08-10T12:10:10 2022-08-10T20:10:10 3333 + +-- !sql_7_14 -- +[1, 0, 1, 1] [1, -2, -3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ["192.168.0.1", "127.0.0.1"] ["a", "b", "c"] [-1, 0, 1, 2] [{"name":"Andy","age":18},{"name":"Tim","age":28}] [1, 2, 3, 4] [128, 129, -129, -130] ["d", "e", "f"] [0, 1, 2, 3] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] \N \N \N string4 2022-08-08T20:10:10 text3_4*5 6.0 2022-08-08T00:00 2022-08-11T12:10:10 1660191010000 2022-08-11T12:10:10 2022-08-11T11:10:10 4444 + +-- !sql_7_15 -- +2022-08-08T20:10:10 + +-- !sql_7_16 -- +2022-08-08T12:10:10 + +-- !sql_7_17 -- +2022-08-09T00:40:10 + +-- !sql_7_18 -- +2022-08-08T20:10:10 + +-- !sql_7_19 -- +I'm not null or empty + +-- !sql_7_20 -- + +I'm not null or empty + +-- !sql_7_21 -- +I'm not null or empty + +-- !sql_7_22 -- +[1, 0, 1, 1] [1, -2, -3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ["192.168.0.1", "127.0.0.1"] ["a", "b", "c"] [-1, 0, 1, 2] [{"name":"Andy","age":18},{"name":"Tim","age":28}] [1, 2, 3, 4] [128, 129, -129, -130] ["d", "e", "f"] [0, 1, 2, 3] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] debug \N This string can be quite lengthy string1 2022-08-08T20:10:10 text#1 3.14 2022-08-08T00:00 2022-08-08T12:10:10 1659931810000 2022-08-08T12:10:10 2022-08-08T20:10:10 12345 + +-- !sql_7_23 -- +[1, 0, 1, 1] [1, -2, -3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ["192.168.0.1", "127.0.0.1"] ["a", "b", "c"] [-1, 0, 1, 2] [{"name":"Andy","age":18},{"name":"Tim","age":28}] [1, 2, 3, 4] [128, 129, -129, -130] ["d", "e", "f"] [0, 1, 2, 3] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] \N \N \N string4 2022-08-08T20:10:10 text3_4*5 6.0 2022-08-08T00:00 2022-08-11T12:10:10 1660191010000 2022-08-11T12:10:10 2022-08-11T11:10:10 4444 +[1, 0, 1, 1] [1, -2, -3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ["192.168.0.1", "127.0.0.1"] ["a", "b", "c"] [-1, 0, 1, 2] [{"name":"Andy","age":18},{"name":"Tim","age":28}] [1, 2, 3, 4] [128, 129, -129, -130] ["d", "e", "f"] [0, 1, 2, 3] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] \N \N string2 2022-08-08T12:10:10 text2 4.0 2022-08-08T00:00 2022-08-09T12:10:10 1660018210000 2022-08-09T12:10:10 2022-08-09T12:10:10 2222 +[1, 0, 1, 1] [1, -2, -3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ["192.168.0.1", "127.0.0.1"] ["a", "b", "c"] [-1, 0, 1, 2] [{"name":"Andy","age":18},{"name":"Tim","age":28}] [1, 2, 3, 4] [128, 129, -129, -130] ["d", "e", "f"] [0, 1, 2, 3] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] \N I'm not null or empty \N string3 2022-08-09T00:40:10 text3_4*5 5.0 2022-08-08T00:00 2022-08-10T12:10:10 1660104610000 2022-08-10T12:10:10 2022-08-10T20:10:10 3333 +[1, 0, 1, 1] [1, -2, -3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ["192.168.0.1", "127.0.0.1"] ["a", "b", "c"] [-1, 0, 1, 2] [{"name":"Andy","age":18},{"name":"Tim","age":28}] [1, 2, 3, 4] [128, 129, -129, -130] ["d", "e", "f"] [0, 1, 2, 3] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] debug \N This string can be quite lengthy string1 2022-08-08T20:10:10 text#1 3.14 2022-08-08T00:00 2022-08-08T12:10:10 1659931810000 2022-08-08T12:10:10 2022-08-08T20:10:10 12345 + +-- !sql_7_24 -- +[1, 0, 1, 1] [1, -2, -3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ["192.168.0.1", "127.0.0.1"] ["a", "b", "c"] [-1, 0, 1, 2] [{"name":"Andy","age":18},{"name":"Tim","age":28}] [1, 2, 3, 4] [128, 129, -129, -130] ["d", "e", "f"] [0, 1, 2, 3] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] \N \N \N string4 2022-08-08T20:10:10 text3_4*5 6.0 2022-08-08T00:00 2022-08-11T12:10:10 1660191010000 2022-08-11T12:10:10 2022-08-11T11:10:10 4444 +[1, 0, 1, 1] [1, -2, -3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ["192.168.0.1", "127.0.0.1"] ["a", "b", "c"] [-1, 0, 1, 2] [{"name":"Andy","age":18},{"name":"Tim","age":28}] [1, 2, 3, 4] [128, 129, -129, -130] ["d", "e", "f"] [0, 1, 2, 3] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] \N \N string2 2022-08-08T12:10:10 text2 4.0 2022-08-08T00:00 2022-08-09T12:10:10 1660018210000 2022-08-09T12:10:10 2022-08-09T12:10:10 2222 +[1, 0, 1, 1] [1, -2, -3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ["192.168.0.1", "127.0.0.1"] ["a", "b", "c"] [-1, 0, 1, 2] [{"name":"Andy","age":18},{"name":"Tim","age":28}] [1, 2, 3, 4] [128, 129, -129, -130] ["d", "e", "f"] [0, 1, 2, 3] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] \N I'm not null or empty \N string3 2022-08-09T00:40:10 text3_4*5 5.0 2022-08-08T00:00 2022-08-10T12:10:10 1660104610000 2022-08-10T12:10:10 2022-08-10T20:10:10 3333 +[1, 0, 1, 1] [1, -2, -3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ["192.168.0.1", "127.0.0.1"] ["a", "b", "c"] [-1, 0, 1, 2] [{"name":"Andy","age":18},{"name":"Tim","age":28}] [1, 2, 3, 4] [128, 129, -129, -130] ["d", "e", "f"] [0, 1, 2, 3] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] debug \N This string can be quite lengthy string1 2022-08-08T20:10:10 text#1 3.14 2022-08-08T00:00 2022-08-08T12:10:10 1659931810000 2022-08-08T12:10:10 2022-08-08T20:10:10 12345 + +-- !sql_7_25 -- +[{"name":"Andy","age":18},{"name":"Tim","age":28}] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] "Andy" "White" +[{"name":"Andy","age":18},{"name":"Tim","age":28}] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] "Andy" "White" +[{"name":"Andy","age":18},{"name":"Tim","age":28}] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] "Andy" "White" +[{"name":"Andy","age":18},{"name":"Tim","age":28}] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] "Andy" "White" + +-- !sql_7_26 -- +value1 value2 + +-- !sql_8_01 -- +[1, 0, 1, 1] [1, -2, -3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ["192.168.0.1", "127.0.0.1"] ["a", "b", "c"] [-1, 0, 1, 2] [{"name":"Andy","age":18},{"name":"Tim","age":28}] [1, 2, 3, 4] [128, 129, -129, -130] ["d", "e", "f"] [0, 1, 2, 3] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] debug \N This string can be quite lengthy string1 2022-08-08T20:10:10 text#1 3.14 2022-08-08T00:00 2022-08-08T12:10:10 1659931810000 2022-08-08T12:10:10 2022-08-08T20:10:10 12345 + +-- !sql_8_02 -- +[1, 0, 1, 1] [1, -2, -3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ["192.168.0.1", "127.0.0.1"] ["a", "b", "c"] [-1, 0, 1, 2] [{"name":"Andy","age":18},{"name":"Tim","age":28}] [1, 2, 3, 4] [128, 129, -129, -130] ["d", "e", "f"] [0, 1, 2, 3] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] \N \N \N string4 2022-08-08T20:10:10 text3_4*5 6.0 2022-08-08T00:00 2022-08-11T12:10:10 1660191010000 2022-08-11T12:10:10 2022-08-11T11:10:10 4444 +[1, 0, 1, 1] [1, -2, -3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ["192.168.0.1", "127.0.0.1"] ["a", "b", "c"] [-1, 0, 1, 2] [{"name":"Andy","age":18},{"name":"Tim","age":28}] [1, 2, 3, 4] [128, 129, -129, -130] ["d", "e", "f"] [0, 1, 2, 3] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] \N \N string2 2022-08-08T12:10:10 text2 4.0 2022-08-08T00:00 2022-08-09T12:10:10 1660018210000 2022-08-09T12:10:10 2022-08-09T12:10:10 2222 +[1, 0, 1, 1] [1, -2, -3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ["192.168.0.1", "127.0.0.1"] ["a", "b", "c"] [-1, 0, 1, 2] [{"name":"Andy","age":18},{"name":"Tim","age":28}] [1, 2, 3, 4] [128, 129, -129, -130] ["d", "e", "f"] [0, 1, 2, 3] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] \N I'm not null or empty \N string3 2022-08-09T00:40:10 text3_4*5 5.0 2022-08-08T00:00 2022-08-10T12:10:10 1660104610000 2022-08-10T12:10:10 2022-08-10T20:10:10 3333 +[1, 0, 1, 1] [1, -2, -3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ["192.168.0.1", "127.0.0.1"] ["a", "b", "c"] [-1, 0, 1, 2] [{"name":"Andy","age":18},{"name":"Tim","age":28}] [1, 2, 3, 4] [128, 129, -129, -130] ["d", "e", "f"] [0, 1, 2, 3] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] debug \N This string can be quite lengthy string1 2022-08-08T20:10:10 text#1 3.14 2022-08-08T00:00 2022-08-08T12:10:10 1659931810000 2022-08-08T12:10:10 2022-08-08T20:10:10 12345 + +-- !sql_8_03 -- +true 1 128 32768 -1 0 1.0 1.0 1.0 1.0 2020-01-01 2020-01-01T12:00 a d 192.168.0.1 {"name":"Andy","age":18} +true 1 128 32768 -1 0 1.0 1.0 1.0 1.0 2020-01-01 2020-01-01T12:00 a d 192.168.0.1 {"name":"Andy","age":18} +true 1 128 32768 -1 0 1.0 1.0 1.0 1.0 2020-01-01 2020-01-01T12:00 a d 192.168.0.1 {"name":"Andy","age":18} +true 1 128 32768 -1 0 1.0 1.0 1.0 1.0 2020-01-01 2020-01-01T12:00 a d 192.168.0.1 {"name":"Andy","age":18} + +-- !sql_8_04 -- +true 1 128 32768 -1 0 1.0 1.0 1.0 1.0 2020-01-01 2020-01-01T12:00 a d 192.168.0.1 {"name":"Andy","age":18} +true 1 128 32768 -1 0 1.0 1.0 1.0 1.0 2020-01-01 2020-01-01T12:00 a d 192.168.0.1 {"name":"Andy","age":18} +true 1 128 32768 -1 0 1.0 1.0 1.0 1.0 2020-01-01 2020-01-01T12:00 a d 192.168.0.1 {"name":"Andy","age":18} +true 1 128 32768 -1 0 1.0 1.0 1.0 1.0 2020-01-01 2020-01-01T12:00 a d 192.168.0.1 {"name":"Andy","age":18} +true 1 128 32768 -1 0 1.0 1.0 1.0 1.0 2020-01-01 2020-01-01T12:00 a d 192.168.0.1 {"name":"Andy","age":18} +true 1 128 32768 -1 0 1.0 1.0 1.0 1.0 2020-01-01 2020-01-01T12:00 a d 192.168.0.1 {"name":"Andy","age":18} +true 1 128 32768 -1 0 1.0 1.0 1.0 1.0 2020-01-01 2020-01-01T12:00 a d 192.168.0.1 {"name":"Andy","age":18} +true 1 128 32768 -1 0 1.0 1.0 1.0 1.0 2020-01-01 2020-01-01T12:00 a d 192.168.0.1 {"name":"Andy","age":18} + +-- !sql_8_05 -- +[1, 0, 1, 1] [1, -2, -3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ["192.168.0.1", "127.0.0.1"] ["a", "b", "c"] [-1, 0, 1, 2] [{"name":"Andy","age":18},{"name":"Tim","age":28}] [1, 2, 3, 4] [128, 129, -129, -130] ["d", "e", "f"] [0, 1, 2, 3] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] debug \N This string can be quite lengthy string1 2022-08-08T20:10:10 text#1 3.14 2022-08-08T00:00 2022-08-08T12:10:10 1659931810000 2022-08-08T12:10:10 2022-08-08T20:10:10 12345 + +-- !sql_8_06 -- +[1, 0, 1, 1] [1, -2, -3, 4] [128, 129, -129, -130] [32768, 32769, -32769, -32770] [-1, 0, 1, 2] [0, 1, 2, 3] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [1, 2, 3, 4] [1, 2, 3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] ["a", "b", "c"] ["d", "e", "f"] ["192.168.0.1", "127.0.0.1"] [{"name":"Andy","age":18},{"name":"Tim","age":28}] +[1, 0, 1, 1] [1, -2, -3, 4] [128, 129, -129, -130] [32768, 32769, -32769, -32770] [-1, 0, 1, 2] [0, 1, 2, 3] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [1, 2, 3, 4] [1, 2, 3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] ["a", "b", "c"] ["d", "e", "f"] ["192.168.0.1", "127.0.0.1"] [{"name":"Andy","age":18},{"name":"Tim","age":28}] +[1, 0, 1, 1] [1, -2, -3, 4] [128, 129, -129, -130] [32768, 32769, -32769, -32770] [-1, 0, 1, 2] [0, 1, 2, 3] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [1, 2, 3, 4] [1, 2, 3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] ["a", "b", "c"] ["d", "e", "f"] ["192.168.0.1", "127.0.0.1"] [{"name":"Andy","age":18},{"name":"Tim","age":28}] +[1, 0, 1, 1] [1, -2, -3, 4] [128, 129, -129, -130] [32768, 32769, -32769, -32770] [-1, 0, 1, 2] [0, 1, 2, 3] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [1, 2, 3, 4] [1, 2, 3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] ["a", "b", "c"] ["d", "e", "f"] ["192.168.0.1", "127.0.0.1"] [{"name":"Andy","age":18},{"name":"Tim","age":28}] + +-- !sql_8_07 -- +[1, 0, 1, 1] [1, -2, -3, 4] [128, 129, -129, -130] [32768, 32769, -32769, -32770] [-1, 0, 1, 2] [0, 1, 2, 3] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [1, 2, 3, 4] [1, 2, 3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] ["a", "b", "c"] ["d", "e", "f"] ["192.168.0.1", "127.0.0.1"] [{"name":"Andy","age":18},{"name":"Tim","age":28}] +[1, 0, 1, 1] [1, -2, -3, 4] [128, 129, -129, -130] [32768, 32769, -32769, -32770] [-1, 0, 1, 2] [0, 1, 2, 3] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [1, 2, 3, 4] [1, 2, 3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] ["a", "b", "c"] ["d", "e", "f"] ["192.168.0.1", "127.0.0.1"] [{"name":"Andy","age":18},{"name":"Tim","age":28}] +[1, 0, 1, 1] [1, -2, -3, 4] [128, 129, -129, -130] [32768, 32769, -32769, -32770] [-1, 0, 1, 2] [0, 1, 2, 3] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [1, 2, 3, 4] [1, 2, 3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] ["a", "b", "c"] ["d", "e", "f"] ["192.168.0.1", "127.0.0.1"] [{"name":"Andy","age":18},{"name":"Tim","age":28}] +[1, 0, 1, 1] [1, -2, -3, 4] [128, 129, -129, -130] [32768, 32769, -32769, -32770] [-1, 0, 1, 2] [0, 1, 2, 3] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [1, 2, 3, 4] [1, 2, 3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] ["a", "b", "c"] ["d", "e", "f"] ["192.168.0.1", "127.0.0.1"] [{"name":"Andy","age":18},{"name":"Tim","age":28}] +[1, 0, 1, 1] [1, -2, -3, 4] [128, 129, -129, -130] [32768, 32769, -32769, -32770] [-1, 0, 1, 2] [0, 1, 2, 3] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [1, 2, 3, 4] [1, 2, 3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] ["a", "b", "c"] ["d", "e", "f"] ["192.168.0.1", "127.0.0.1"] [{"name":"Andy","age":18},{"name":"Tim","age":28}] +[1, 0, 1, 1] [1, -2, -3, 4] [128, 129, -129, -130] [32768, 32769, -32769, -32770] [-1, 0, 1, 2] [0, 1, 2, 3] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [1, 2, 3, 4] [1, 2, 3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] ["a", "b", "c"] ["d", "e", "f"] ["192.168.0.1", "127.0.0.1"] [{"name":"Andy","age":18},{"name":"Tim","age":28}] +[1, 0, 1, 1] [1, -2, -3, 4] [128, 129, -129, -130] [32768, 32769, -32769, -32770] [-1, 0, 1, 2] [0, 1, 2, 3] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [1, 2, 3, 4] [1, 2, 3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] ["a", "b", "c"] ["d", "e", "f"] ["192.168.0.1", "127.0.0.1"] [{"name":"Andy","age":18},{"name":"Tim","age":28}] +[1, 0, 1, 1] [1, -2, -3, 4] [128, 129, -129, -130] [32768, 32769, -32769, -32770] [-1, 0, 1, 2] [0, 1, 2, 3] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [1, 2, 3, 4] [1, 2, 3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] ["a", "b", "c"] ["d", "e", "f"] ["192.168.0.1", "127.0.0.1"] [{"name":"Andy","age":18},{"name":"Tim","age":28}] + +-- !sql_8_08 -- +value1 value2 + +-- !sql_8_09 -- +[1, 0, 1, 1] [1, -2, -3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ["192.168.0.1", "127.0.0.1"] ["a", "b", "c"] [-1, 0, 1, 2] [{"name":"Andy","age":18},{"name":"Tim","age":28}] [1, 2, 3, 4] [128, 129, -129, -130] ["d", "e", "f"] [0, 1, 2, 3] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] debug \N This string can be quite lengthy string1 2022-08-08T20:10:10 text#1 3.14 2022-08-08T00:00 2022-08-08T12:10:10 1659931810000 2022-08-08T12:10:10 2022-08-08T20:10:10 12345 + +-- !sql_8_10 -- +[1, 0, 1, 1] [1, -2, -3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ["192.168.0.1", "127.0.0.1"] ["a", "b", "c"] [-1, 0, 1, 2] [{"name":"Andy","age":18},{"name":"Tim","age":28}] [1, 2, 3, 4] [128, 129, -129, -130] ["d", "e", "f"] [0, 1, 2, 3] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] \N \N string2 2022-08-08T12:10:10 text2 4.0 2022-08-08T00:00 2022-08-09T12:10:10 1660018210000 2022-08-09T12:10:10 2022-08-09T12:10:10 2222 + +-- !sql_8_11 -- +[1, 0, 1, 1] [1, -2, -3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ["192.168.0.1", "127.0.0.1"] ["a", "b", "c"] [-1, 0, 1, 2] [{"name":"Andy","age":18},{"name":"Tim","age":28}] [1, 2, 3, 4] [128, 129, -129, -130] ["d", "e", "f"] [0, 1, 2, 3] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] \N I'm not null or empty \N string3 2022-08-09T00:40:10 text3_4*5 5.0 2022-08-08T00:00 2022-08-10T12:10:10 1660104610000 2022-08-10T12:10:10 2022-08-10T20:10:10 3333 + +-- !sql_8_12 -- +[1, 0, 1, 1] [1, -2, -3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ["192.168.0.1", "127.0.0.1"] ["a", "b", "c"] [-1, 0, 1, 2] [{"name":"Andy","age":18},{"name":"Tim","age":28}] [1, 2, 3, 4] [128, 129, -129, -130] ["d", "e", "f"] [0, 1, 2, 3] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] \N \N \N string4 2022-08-08T20:10:10 text3_4*5 6.0 2022-08-08T00:00 2022-08-11T12:10:10 1660191010000 2022-08-11T12:10:10 2022-08-11T11:10:10 4444 + +-- !sql_8_13 -- +2022-08-08T20:10:10 + +-- !sql_8_14 -- +2022-08-08T12:10:10 + +-- !sql_8_15 -- +2022-08-09T00:40:10 + +-- !sql_8_16 -- +2022-08-08T20:10:10 + +-- !sql_8_17 -- +I'm not null or empty + +-- !sql_8_18 -- + +I'm not null or empty + +-- !sql_8_19 -- +I'm not null or empty + +-- !sql_8_20 -- +[1, 0, 1, 1] [1, -2, -3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ["192.168.0.1", "127.0.0.1"] ["a", "b", "c"] [-1, 0, 1, 2] [{"name":"Andy","age":18},{"name":"Tim","age":28}] [1, 2, 3, 4] [128, 129, -129, -130] ["d", "e", "f"] [0, 1, 2, 3] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] debug \N This string can be quite lengthy string1 2022-08-08T20:10:10 text#1 3.14 2022-08-08T00:00 2022-08-08T12:10:10 1659931810000 2022-08-08T12:10:10 2022-08-08T20:10:10 12345 + +-- !sql_8_21 -- +[1, 0, 1, 1] [1, -2, -3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ["192.168.0.1", "127.0.0.1"] ["a", "b", "c"] [-1, 0, 1, 2] [{"name":"Andy","age":18},{"name":"Tim","age":28}] [1, 2, 3, 4] [128, 129, -129, -130] ["d", "e", "f"] [0, 1, 2, 3] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] \N \N \N string4 2022-08-08T20:10:10 text3_4*5 6.0 2022-08-08T00:00 2022-08-11T12:10:10 1660191010000 2022-08-11T12:10:10 2022-08-11T11:10:10 4444 +[1, 0, 1, 1] [1, -2, -3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ["192.168.0.1", "127.0.0.1"] ["a", "b", "c"] [-1, 0, 1, 2] [{"name":"Andy","age":18},{"name":"Tim","age":28}] [1, 2, 3, 4] [128, 129, -129, -130] ["d", "e", "f"] [0, 1, 2, 3] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] \N \N string2 2022-08-08T12:10:10 text2 4.0 2022-08-08T00:00 2022-08-09T12:10:10 1660018210000 2022-08-09T12:10:10 2022-08-09T12:10:10 2222 +[1, 0, 1, 1] [1, -2, -3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ["192.168.0.1", "127.0.0.1"] ["a", "b", "c"] [-1, 0, 1, 2] [{"name":"Andy","age":18},{"name":"Tim","age":28}] [1, 2, 3, 4] [128, 129, -129, -130] ["d", "e", "f"] [0, 1, 2, 3] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] \N I'm not null or empty \N string3 2022-08-09T00:40:10 text3_4*5 5.0 2022-08-08T00:00 2022-08-10T12:10:10 1660104610000 2022-08-10T12:10:10 2022-08-10T20:10:10 3333 +[1, 0, 1, 1] [1, -2, -3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ["192.168.0.1", "127.0.0.1"] ["a", "b", "c"] [-1, 0, 1, 2] [{"name":"Andy","age":18},{"name":"Tim","age":28}] [1, 2, 3, 4] [128, 129, -129, -130] ["d", "e", "f"] [0, 1, 2, 3] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] debug \N This string can be quite lengthy string1 2022-08-08T20:10:10 text#1 3.14 2022-08-08T00:00 2022-08-08T12:10:10 1659931810000 2022-08-08T12:10:10 2022-08-08T20:10:10 12345 + +-- !sql_8_22 -- +[1, 0, 1, 1] [1, -2, -3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ["192.168.0.1", "127.0.0.1"] ["a", "b", "c"] [-1, 0, 1, 2] [{"name":"Andy","age":18},{"name":"Tim","age":28}] [1, 2, 3, 4] [128, 129, -129, -130] ["d", "e", "f"] [0, 1, 2, 3] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] \N \N \N string4 2022-08-08T20:10:10 text3_4*5 6.0 2022-08-08T00:00 2022-08-11T12:10:10 1660191010000 2022-08-11T12:10:10 2022-08-11T11:10:10 4444 +[1, 0, 1, 1] [1, -2, -3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ["192.168.0.1", "127.0.0.1"] ["a", "b", "c"] [-1, 0, 1, 2] [{"name":"Andy","age":18},{"name":"Tim","age":28}] [1, 2, 3, 4] [128, 129, -129, -130] ["d", "e", "f"] [0, 1, 2, 3] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] \N \N string2 2022-08-08T12:10:10 text2 4.0 2022-08-08T00:00 2022-08-09T12:10:10 1660018210000 2022-08-09T12:10:10 2022-08-09T12:10:10 2222 +[1, 0, 1, 1] [1, -2, -3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ["192.168.0.1", "127.0.0.1"] ["a", "b", "c"] [-1, 0, 1, 2] [{"name":"Andy","age":18},{"name":"Tim","age":28}] [1, 2, 3, 4] [128, 129, -129, -130] ["d", "e", "f"] [0, 1, 2, 3] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] \N I'm not null or empty \N string3 2022-08-09T00:40:10 text3_4*5 5.0 2022-08-08T00:00 2022-08-10T12:10:10 1660104610000 2022-08-10T12:10:10 2022-08-10T20:10:10 3333 +[1, 0, 1, 1] [1, -2, -3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ["192.168.0.1", "127.0.0.1"] ["a", "b", "c"] [-1, 0, 1, 2] [{"name":"Andy","age":18},{"name":"Tim","age":28}] [1, 2, 3, 4] [128, 129, -129, -130] ["d", "e", "f"] [0, 1, 2, 3] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] debug \N This string can be quite lengthy string1 2022-08-08T20:10:10 text#1 3.14 2022-08-08T00:00 2022-08-08T12:10:10 1659931810000 2022-08-08T12:10:10 2022-08-08T20:10:10 12345 + +-- !sql_8_23 -- +[{"name":"Andy","age":18},{"name":"Tim","age":28}] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] "Andy" "White" +[{"name":"Andy","age":18},{"name":"Tim","age":28}] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] "Andy" "White" +[{"name":"Andy","age":18},{"name":"Tim","age":28}] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] "Andy" "White" +[{"name":"Andy","age":18},{"name":"Tim","age":28}] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] "Andy" "White" + +-- !sql01 -- +["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [-1, 0, 1, 2] [0, 1, 2, 3] ["d", "e", "f"] [128, 129, -129, -130] ["192.168.0.1", "127.0.0.1"] string1 [1, 2, 3, 4] 2022-08-08 2022-08-08T12:10:10 text#1 ["2020-01-01", "2020-01-02"] 3.14 [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] ["a", "b", "c"] [{"name":"Andy","age":18},{"name":"Tim","age":28}] 2022-08-08T12:10:10 2022-08-08T12:10:10 2022-08-08T20:10:10 [1, -2, -3, 4] [1, 0, 1, 1] [32768, 32769, -32769, -32770] \N [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] + +-- !sql02 -- +["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [-1, 0, 1, 2] [0, 1, 2, 3] ["d", "e", "f"] [128, 129, -129, -130] ["192.168.0.1", "127.0.0.1"] string1 [1, 2, 3, 4] 2022-08-08 2022-08-08T12:10:10 text#1 ["2020-01-01", "2020-01-02"] 3.14 [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] ["a", "b", "c"] [{"name":"Andy","age":18},{"name":"Tim","age":28}] 2022-08-08T12:10:10 2022-08-08T12:10:10 2022-08-08T20:10:10 [1, -2, -3, 4] [1, 0, 1, 1] [32768, 32769, -32769, -32770] \N [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] + +-- !sql03 -- +2022-08-08 2022-08-08T12:10:10 2022-08-08T12:10:10 2022-08-08T04:10:10 2022-08-08T20:10:10 +2022-08-08 2022-08-09T12:10:10 2022-08-09T12:10:10 2022-08-09T12:10:10 2022-08-09T12:10:10 +2022-08-08 2022-08-10T12:10:10 2022-08-10T12:10:10 2022-08-10T04:10:10 2022-08-10T20:10:10 +2022-08-08 2022-08-11T12:10:10 2022-08-11T12:10:10 2022-08-11T12:10:10 2022-08-11T11:10:10 + +-- !sql04 -- +I'm not null or empty + +-- !sql05 -- + +I'm not null or empty + +-- !sql06 -- +\N +\N + +I'm not null or empty + +-- !sql07 -- +["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [-1, 0, 1, 2] [0, 1, 2, 3] ["d", "e", "f"] [128, 129, -129, -130] ["192.168.0.1", "127.0.0.1"] string1 [1, 2, 3, 4] 2022-08-08 2022-08-08T12:10:10 text#1 ["2020-01-01", "2020-01-02"] 3.14 [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] ["a", "b", "c"] [{"name":"Andy","age":18},{"name":"Tim","age":28}] 2022-08-08T12:10:10 2022-08-08T12:10:10 2022-08-08T20:10:10 [1, -2, -3, 4] [1, 0, 1, 1] [32768, 32769, -32769, -32770] \N [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] +["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [-1, 0, 1, 2] [0, 1, 2, 3] ["d", "e", "f"] [128, 129, -129, -130] ["192.168.0.1", "127.0.0.1"] string2 [1, 2, 3, 4] 2022-08-08 2022-08-09T12:10:10 text2 ["2020-01-01", "2020-01-02"] 4.0 [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] ["a", "b", "c"] [{"name":"Andy","age":18},{"name":"Tim","age":28}] 2022-08-09T12:10:10 2022-08-09T12:10:10 2022-08-09T12:10:10 [1, -2, -3, 4] [1, 0, 1, 1] [32768, 32769, -32769, -32770] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] +["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [-1, 0, 1, 2] [0, 1, 2, 3] ["d", "e", "f"] [128, 129, -129, -130] ["192.168.0.1", "127.0.0.1"] string3 [1, 2, 3, 4] 2022-08-08 2022-08-10T12:10:10 text3_4*5 ["2020-01-01", "2020-01-02"] 5.0 [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] ["a", "b", "c"] [{"name":"Andy","age":18},{"name":"Tim","age":28}] 2022-08-10T12:10:10 2022-08-10T12:10:10 2022-08-10T20:10:10 [1, -2, -3, 4] [1, 0, 1, 1] [32768, 32769, -32769, -32770] I'm not null or empty [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] +["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [-1, 0, 1, 2] [0, 1, 2, 3] ["d", "e", "f"] [128, 129, -129, -130] ["192.168.0.1", "127.0.0.1"] string4 [1, 2, 3, 4] 2022-08-08 2022-08-11T12:10:10 text3_4*5 ["2020-01-01", "2020-01-02"] 6.0 [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] ["a", "b", "c"] [{"name":"Andy","age":18},{"name":"Tim","age":28}] 2022-08-11T12:10:10 2022-08-11T12:10:10 2022-08-11T11:10:10 [1, -2, -3, 4] [1, 0, 1, 1] [32768, 32769, -32769, -32770] \N [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] + +-- !sql08 -- +[{"name":"Andy","age":18},{"name":"Tim","age":28}] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] "Andy" "White" +[{"name":"Andy","age":18},{"name":"Tim","age":28}] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] "Andy" "White" +[{"name":"Andy","age":18},{"name":"Tim","age":28}] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] "Andy" "White" +[{"name":"Andy","age":18},{"name":"Tim","age":28}] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] "Andy" "White" + +-- !sql20 -- +["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [-1, 0, 1, 2] [0, 1, 2, 3] ["d", "e", "f"] [128, 129, -129, -130] ["192.168.0.1", "127.0.0.1"] string1 [1, 2, 3, 4] 2022-08-08 2022-08-08T12:10:10 text#1 ["2020-01-01", "2020-01-02"] 3.14 [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] ["a", "b", "c"] [{"name":"Andy","age":18},{"name":"Tim","age":28}] 2022-08-08T12:10:10 2022-08-08T12:10:10 2022-08-08T20:10:10 [1, -2, -3, 4] [1, 0, 1, 1] [32768, 32769, -32769, -32770] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] + +-- !sql21 -- +["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [-1, 0, 1, 2] [0, 1, 2, 3] ["d", "e", "f"] [128, 129, -129, -130] ["192.168.0.1", "127.0.0.1"] string1 [1, 2, 3, 4] 2022-08-08 2022-08-08T12:10:10 text#1 ["2020-01-01", "2020-01-02"] 3.14 [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] ["a", "b", "c"] [{"name":"Andy","age":18},{"name":"Tim","age":28}] 2022-08-08T12:10:10 2022-08-08T12:10:10 2022-08-08T20:10:10 [1, -2, -3, 4] [1, 0, 1, 1] [32768, 32769, -32769, -32770] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] + +-- !sql22 -- +2022-08-08 2022-08-08T12:10:10 2022-08-08T12:10:10 2022-08-08T04:10:10 2022-08-08T20:10:10 +2022-08-08 2022-08-09T12:10:10 2022-08-09T12:10:10 2022-08-09T12:10:10 2022-08-09T12:10:10 +2022-08-08 2022-08-10T12:10:10 2022-08-10T12:10:10 2022-08-10T04:10:10 2022-08-10T20:10:10 +2022-08-08 2022-08-11T12:10:10 2022-08-11T12:10:10 2022-08-11T12:10:10 2022-08-11T11:10:10 + +-- !sql23 -- +["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [-1, 0, 1, 2] [0, 1, 2, 3] ["d", "e", "f"] [128, 129, -129, -130] ["192.168.0.1", "127.0.0.1"] string1 [1, 2, 3, 4] 2022-08-08 2022-08-08T12:10:10 text#1 ["2020-01-01", "2020-01-02"] 3.14 [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] ["a", "b", "c"] [{"name":"Andy","age":18},{"name":"Tim","age":28}] 2022-08-08T12:10:10 2022-08-08T12:10:10 2022-08-08T20:10:10 [1, -2, -3, 4] [1, 0, 1, 1] [32768, 32769, -32769, -32770] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] +["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [-1, 0, 1, 2] [0, 1, 2, 3] ["d", "e", "f"] [128, 129, -129, -130] ["192.168.0.1", "127.0.0.1"] string2 [1, 2, 3, 4] 2022-08-08 2022-08-09T12:10:10 text2 ["2020-01-01", "2020-01-02"] 4.0 [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] ["a", "b", "c"] [{"name":"Andy","age":18},{"name":"Tim","age":28}] 2022-08-09T12:10:10 2022-08-09T12:10:10 2022-08-09T12:10:10 [1, -2, -3, 4] [1, 0, 1, 1] [32768, 32769, -32769, -32770] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] +["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [-1, 0, 1, 2] [0, 1, 2, 3] ["d", "e", "f"] [128, 129, -129, -130] ["192.168.0.1", "127.0.0.1"] string3 [1, 2, 3, 4] 2022-08-08 2022-08-10T12:10:10 text3_4*5 ["2020-01-01", "2020-01-02"] 5.0 [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] ["a", "b", "c"] [{"name":"Andy","age":18},{"name":"Tim","age":28}] 2022-08-10T12:10:10 2022-08-10T12:10:10 2022-08-10T20:10:10 [1, -2, -3, 4] [1, 0, 1, 1] [32768, 32769, -32769, -32770] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] +["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [-1, 0, 1, 2] [0, 1, 2, 3] ["d", "e", "f"] [128, 129, -129, -130] ["192.168.0.1", "127.0.0.1"] string4 [1, 2, 3, 4] 2022-08-08 2022-08-11T12:10:10 text3_4*5 ["2020-01-01", "2020-01-02"] 6.0 [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] ["a", "b", "c"] [{"name":"Andy","age":18},{"name":"Tim","age":28}] 2022-08-11T12:10:10 2022-08-11T12:10:10 2022-08-11T11:10:10 [1, -2, -3, 4] [1, 0, 1, 1] [32768, 32769, -32769, -32770] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] + +-- !sql24 -- +[{"name":"Andy","age":18},{"name":"Tim","age":28}] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] "Andy" "White" +[{"name":"Andy","age":18},{"name":"Tim","age":28}] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] "Andy" "White" +[{"name":"Andy","age":18},{"name":"Tim","age":28}] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] "Andy" "White" +[{"name":"Andy","age":18},{"name":"Tim","age":28}] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] "Andy" "White" + -- !sql_5_02 -- [1, 0, 1, 1] [1, -2, -3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ["192.168.0.1", "127.0.0.1"] ["a", "b", "c"] [-1, 0, 1, 2] [{"name":"Andy","age":18},{"name":"Tim","age":28}] [1, 2, 3, 4] [128, 129, -129, -130] ["d", "e", "f"] [0, 1, 2, 3] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] \N string1 text#1 3.14 2022-08-08T00:00 12345 2022-08-08T20:10:10 @@ -227,6 +703,10 @@ true 1 128 32768 -1 0 1.0 1.0 1.0 1.0 2020-01-01 2020-01-01T12:00 a d 192.168.0. true 1 128 32768 -1 0 1.0 1.0 1.0 1.0 2020-01-01 2020-01-01T12:00 a d 192.168.0.1 {"name":"Andy","age":18} true 1 128 32768 -1 0 1.0 1.0 1.0 1.0 2020-01-01 2020-01-01T12:00 a d 192.168.0.1 {"name":"Andy","age":18} true 1 128 32768 -1 0 1.0 1.0 1.0 1.0 2020-01-01 2020-01-01T12:00 a d 192.168.0.1 {"name":"Andy","age":18} +true 1 128 32768 -1 0 1.0 1.0 1.0 1.0 2020-01-01 2020-01-01T12:00 a d 192.168.0.1 {"name":"Andy","age":18} +true 1 128 32768 -1 0 1.0 1.0 1.0 1.0 2020-01-01 2020-01-01T12:00 a d 192.168.0.1 {"name":"Andy","age":18} +true 1 128 32768 -1 0 1.0 1.0 1.0 1.0 2020-01-01 2020-01-01T12:00 a d 192.168.0.1 {"name":"Andy","age":18} +true 1 128 32768 -1 0 1.0 1.0 1.0 1.0 2020-01-01 2020-01-01T12:00 a d 192.168.0.1 {"name":"Andy","age":18} -- !sql_7_07 -- [1, 0, 1, 1] [1, -2, -3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ["192.168.0.1", "127.0.0.1"] ["a", "b", "c"] [-1, 0, 1, 2] [{"name":"Andy","age":18},{"name":"Tim","age":28}] [1, 2, 3, 4] [128, 129, -129, -130] ["d", "e", "f"] [0, 1, 2, 3] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] debug \N This string can be quite lengthy string1 2022-08-08T20:10:10 text#1 3.14 2022-08-08T00:00 2022-08-08T12:10:10 1659931810000 2022-08-08T12:10:10 2022-08-08T20:10:10 12345 @@ -242,6 +722,10 @@ true 1 128 32768 -1 0 1.0 1.0 1.0 1.0 2020-01-01 2020-01-01T12:00 a d 192.168.0. [1, 0, 1, 1] [1, -2, -3, 4] [128, 129, -129, -130] [32768, 32769, -32769, -32770] [-1, 0, 1, 2] [0, 1, 2, 3] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [1, 2, 3, 4] [1, 2, 3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] ["a", "b", "c"] ["d", "e", "f"] ["192.168.0.1", "127.0.0.1"] [{"name":"Andy","age":18},{"name":"Tim","age":28}] [1, 0, 1, 1] [1, -2, -3, 4] [128, 129, -129, -130] [32768, 32769, -32769, -32770] [-1, 0, 1, 2] [0, 1, 2, 3] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [1, 2, 3, 4] [1, 2, 3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] ["a", "b", "c"] ["d", "e", "f"] ["192.168.0.1", "127.0.0.1"] [{"name":"Andy","age":18},{"name":"Tim","age":28}] [1, 0, 1, 1] [1, -2, -3, 4] [128, 129, -129, -130] [32768, 32769, -32769, -32770] [-1, 0, 1, 2] [0, 1, 2, 3] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [1, 2, 3, 4] [1, 2, 3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] ["a", "b", "c"] ["d", "e", "f"] ["192.168.0.1", "127.0.0.1"] [{"name":"Andy","age":18},{"name":"Tim","age":28}] +[1, 0, 1, 1] [1, -2, -3, 4] [128, 129, -129, -130] [32768, 32769, -32769, -32770] [-1, 0, 1, 2] [0, 1, 2, 3] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [1, 2, 3, 4] [1, 2, 3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] ["a", "b", "c"] ["d", "e", "f"] ["192.168.0.1", "127.0.0.1"] [{"name":"Andy","age":18},{"name":"Tim","age":28}] +[1, 0, 1, 1] [1, -2, -3, 4] [128, 129, -129, -130] [32768, 32769, -32769, -32770] [-1, 0, 1, 2] [0, 1, 2, 3] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [1, 2, 3, 4] [1, 2, 3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] ["a", "b", "c"] ["d", "e", "f"] ["192.168.0.1", "127.0.0.1"] [{"name":"Andy","age":18},{"name":"Tim","age":28}] +[1, 0, 1, 1] [1, -2, -3, 4] [128, 129, -129, -130] [32768, 32769, -32769, -32770] [-1, 0, 1, 2] [0, 1, 2, 3] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [1, 2, 3, 4] [1, 2, 3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] ["a", "b", "c"] ["d", "e", "f"] ["192.168.0.1", "127.0.0.1"] [{"name":"Andy","age":18},{"name":"Tim","age":28}] +[1, 0, 1, 1] [1, -2, -3, 4] [128, 129, -129, -130] [32768, 32769, -32769, -32770] [-1, 0, 1, 2] [0, 1, 2, 3] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [1, 2, 3, 4] [1, 2, 3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] ["a", "b", "c"] ["d", "e", "f"] ["192.168.0.1", "127.0.0.1"] [{"name":"Andy","age":18},{"name":"Tim","age":28}] -- !sql_7_10 -- value1 value2 @@ -324,6 +808,10 @@ true 1 128 32768 -1 0 1.0 1.0 1.0 1.0 2020-01-01 2020-01-01T12:00 a d 192.168.0. true 1 128 32768 -1 0 1.0 1.0 1.0 1.0 2020-01-01 2020-01-01T12:00 a d 192.168.0.1 {"name":"Andy","age":18} true 1 128 32768 -1 0 1.0 1.0 1.0 1.0 2020-01-01 2020-01-01T12:00 a d 192.168.0.1 {"name":"Andy","age":18} true 1 128 32768 -1 0 1.0 1.0 1.0 1.0 2020-01-01 2020-01-01T12:00 a d 192.168.0.1 {"name":"Andy","age":18} +true 1 128 32768 -1 0 1.0 1.0 1.0 1.0 2020-01-01 2020-01-01T12:00 a d 192.168.0.1 {"name":"Andy","age":18} +true 1 128 32768 -1 0 1.0 1.0 1.0 1.0 2020-01-01 2020-01-01T12:00 a d 192.168.0.1 {"name":"Andy","age":18} +true 1 128 32768 -1 0 1.0 1.0 1.0 1.0 2020-01-01 2020-01-01T12:00 a d 192.168.0.1 {"name":"Andy","age":18} +true 1 128 32768 -1 0 1.0 1.0 1.0 1.0 2020-01-01 2020-01-01T12:00 a d 192.168.0.1 {"name":"Andy","age":18} -- !sql_8_05 -- [1, 0, 1, 1] [1, -2, -3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ["192.168.0.1", "127.0.0.1"] ["a", "b", "c"] [-1, 0, 1, 2] [{"name":"Andy","age":18},{"name":"Tim","age":28}] [1, 2, 3, 4] [128, 129, -129, -130] ["d", "e", "f"] [0, 1, 2, 3] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] debug \N This string can be quite lengthy string1 2022-08-08T20:10:10 text#1 3.14 2022-08-08T00:00 2022-08-08T12:10:10 1659931810000 2022-08-08T12:10:10 2022-08-08T20:10:10 12345 @@ -339,6 +827,10 @@ true 1 128 32768 -1 0 1.0 1.0 1.0 1.0 2020-01-01 2020-01-01T12:00 a d 192.168.0. [1, 0, 1, 1] [1, -2, -3, 4] [128, 129, -129, -130] [32768, 32769, -32769, -32770] [-1, 0, 1, 2] [0, 1, 2, 3] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [1, 2, 3, 4] [1, 2, 3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] ["a", "b", "c"] ["d", "e", "f"] ["192.168.0.1", "127.0.0.1"] [{"name":"Andy","age":18},{"name":"Tim","age":28}] [1, 0, 1, 1] [1, -2, -3, 4] [128, 129, -129, -130] [32768, 32769, -32769, -32770] [-1, 0, 1, 2] [0, 1, 2, 3] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [1, 2, 3, 4] [1, 2, 3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] ["a", "b", "c"] ["d", "e", "f"] ["192.168.0.1", "127.0.0.1"] [{"name":"Andy","age":18},{"name":"Tim","age":28}] [1, 0, 1, 1] [1, -2, -3, 4] [128, 129, -129, -130] [32768, 32769, -32769, -32770] [-1, 0, 1, 2] [0, 1, 2, 3] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [1, 2, 3, 4] [1, 2, 3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] ["a", "b", "c"] ["d", "e", "f"] ["192.168.0.1", "127.0.0.1"] [{"name":"Andy","age":18},{"name":"Tim","age":28}] +[1, 0, 1, 1] [1, -2, -3, 4] [128, 129, -129, -130] [32768, 32769, -32769, -32770] [-1, 0, 1, 2] [0, 1, 2, 3] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [1, 2, 3, 4] [1, 2, 3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] ["a", "b", "c"] ["d", "e", "f"] ["192.168.0.1", "127.0.0.1"] [{"name":"Andy","age":18},{"name":"Tim","age":28}] +[1, 0, 1, 1] [1, -2, -3, 4] [128, 129, -129, -130] [32768, 32769, -32769, -32770] [-1, 0, 1, 2] [0, 1, 2, 3] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [1, 2, 3, 4] [1, 2, 3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] ["a", "b", "c"] ["d", "e", "f"] ["192.168.0.1", "127.0.0.1"] [{"name":"Andy","age":18},{"name":"Tim","age":28}] +[1, 0, 1, 1] [1, -2, -3, 4] [128, 129, -129, -130] [32768, 32769, -32769, -32770] [-1, 0, 1, 2] [0, 1, 2, 3] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [1, 2, 3, 4] [1, 2, 3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] ["a", "b", "c"] ["d", "e", "f"] ["192.168.0.1", "127.0.0.1"] [{"name":"Andy","age":18},{"name":"Tim","age":28}] +[1, 0, 1, 1] [1, -2, -3, 4] [128, 129, -129, -130] [32768, 32769, -32769, -32770] [-1, 0, 1, 2] [0, 1, 2, 3] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [1, 2, 3, 4] [1, 2, 3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] ["a", "b", "c"] ["d", "e", "f"] ["192.168.0.1", "127.0.0.1"] [{"name":"Andy","age":18},{"name":"Tim","age":28}] -- !sql_8_08 -- value1 value2 diff --git a/regression-test/suites/correctness_p0/test_colocate_join_of_column_order.groovy b/regression-test/suites/correctness_p0/test_colocate_join_of_column_order.groovy new file mode 100644 index 000000000000000..663b7da02d62da0 --- /dev/null +++ b/regression-test/suites/correctness_p0/test_colocate_join_of_column_order.groovy @@ -0,0 +1,82 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_colocate_join_of_column_order") { + sql """ DROP TABLE IF EXISTS `test_colocate_join_of_column_order_t1`; """ + // distributed by k1,k2 + sql """ + CREATE TABLE IF NOT EXISTS `test_colocate_join_of_column_order_t1` ( + `k1` varchar(64) NULL, + `k2` varchar(64) NULL, + `v` int NULL + ) ENGINE=OLAP + DUPLICATE KEY(`k1`,`k2`) + COMMENT 'OLAP' + DISTRIBUTED BY HASH(`k1`,`k2`) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1", + "colocate_with" = "group_column_order" + ); + """ + sql """ DROP TABLE IF EXISTS `test_colocate_join_of_column_order_t2`; """ + // distributed by k2,k1 + sql """ + CREATE TABLE IF NOT EXISTS `test_colocate_join_of_column_order_t2` ( + `k1` varchar(64) NULL, + `k2` varchar(64) NULL, + `v` int NULL + ) ENGINE=OLAP + DUPLICATE KEY(`k1`,`k2`) + COMMENT 'OLAP' + DISTRIBUTED BY HASH(`k2`,`k1`) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1", + "colocate_with" = "group_column_order" + ); + """ + sql """insert into test_colocate_join_of_column_order_t1 values('k1','k2',11);""" + sql """insert into test_colocate_join_of_column_order_t2 values('k1','k2',11);""" + + sql """set enable_nereids_planner=true; """ + explain { + sql("select * from test_colocate_join_of_column_order_t1 a join test_colocate_join_of_column_order_t2 b on a.k1=b.k1 and a.k2=b.k2;") + notContains "COLOCATE" + } + explain { + sql("select * from test_colocate_join_of_column_order_t1 a join test_colocate_join_of_column_order_t2 b on a.k1=b.k2;") + notContains "COLOCATE" + } + explain { + sql("select * from test_colocate_join_of_column_order_t1 a join test_colocate_join_of_column_order_t2 b on a.k1=b.k1;") + notContains "COLOCATE" + } + explain { + sql("select * from test_colocate_join_of_column_order_t1 a join test_colocate_join_of_column_order_t2 b on a.k1=b.k2 and a.v=b.v;") + notContains "COLOCATE" + } + explain { + sql("select * from test_colocate_join_of_column_order_t1 a join test_colocate_join_of_column_order_t2 b on a.k1=b.k2 and a.k2=b.k1;") + contains "COLOCATE" + } + explain { + sql("select * from test_colocate_join_of_column_order_t1 a join test_colocate_join_of_column_order_t2 b on a.k1=b.k2 and a.k2=b.k1 and a.v=b.v;") + contains "COLOCATE" + } + + sql """ DROP TABLE IF EXISTS `test_colocate_join_of_column_order_t1`; """ + sql """ DROP TABLE IF EXISTS `test_colocate_join_of_column_order_t2`; """ +} diff --git a/regression-test/suites/external_table_p0/es/test_es_query.groovy b/regression-test/suites/external_table_p0/es/test_es_query.groovy index 43e17b2b53b72f5..0f9839aea215d01 100644 --- a/regression-test/suites/external_table_p0/es/test_es_query.groovy +++ b/regression-test/suites/external_table_p0/es/test_es_query.groovy @@ -176,139 +176,165 @@ suite("test_es_query", "p0,external,es,external_docker,external_docker_es") { order_qt_sql23 """select * from test_v2 where esquery(c_long, '{"term":{"c_long":"-1"}}');""" order_qt_sql24 """select c_person, c_user, json_extract(c_person, '\$.[0].name'), json_extract(c_user, '\$.[1].last') from test_v2;""" - sql """switch test_es_query_es5""" - order_qt_sql_5_02 """select * from test1 where test2='text#1'""" - order_qt_sql_5_03 """select * from test2_20220808 where test4 >= '2022-08-08 00:00:00' and test4 < '2022-08-08 23:59:59'""" - order_qt_sql_5_04 """select * from test2_20220808 where substring(test2, 2) = 'ext2'""" - order_qt_sql_5_05 """select c_bool[1], c_byte[1], c_short[1], c_integer[1], c_long[1], c_unsigned_long[1], c_float[1], c_half_float[1], c_double[1], c_scaled_float[1], c_date[1], c_datetime[1], c_keyword[1], c_text[1], c_ip[1], json_extract(c_person, '\$.[0]') from test1""" - order_qt_sql_5_06 """select c_bool[1], c_byte[1], c_short[1], c_integer[1], c_long[1], c_unsigned_long[1], c_float[1], c_half_float[1], c_double[1], c_scaled_float[1], c_date[1], c_datetime[1], c_keyword[1], c_text[1], c_ip[1], json_extract(c_person, '\$.[0]') from test2_20220808""" - order_qt_sql_5_07 """select * from test1 where esquery(test2, '{"match":{"test2":"text#1"}}')""" - order_qt_sql_5_08 """select c_bool, c_byte, c_short, c_integer, c_long, c_unsigned_long, c_float, c_half_float, c_double, c_scaled_float, c_date, c_datetime, c_keyword, c_text, c_ip, c_person from test1""" - order_qt_sql_5_09 """select c_bool, c_byte, c_short, c_integer, c_long, c_unsigned_long, c_float, c_half_float, c_double, c_scaled_float, c_date, c_datetime, c_keyword, c_text, c_ip, c_person from test2_20220808""" - order_qt_sql_5_10 """select * from test1 where test1='string1'""" - order_qt_sql_5_11 """select * from test1 where test1='string2'""" - order_qt_sql_5_12 """select * from test1 where test1='string3'""" - order_qt_sql_5_13 """select test6 from test1 where test1='string1'""" - order_qt_sql_5_14 """select test6 from test1 where test1='string2'""" - order_qt_sql_5_15 """select test6 from test1 where test1='string3'""" - order_qt_sql_5_16 """select message from test1 where message != ''""" - order_qt_sql_5_17 """select message from test1 where message is not null""" - order_qt_sql_5_18 """select message from test1 where not_null_or_empty(message)""" - order_qt_sql_5_19 """select * from test1 where esquery(c_unsigned_long, '{"match":{"c_unsigned_long":0}}')""" - order_qt_sql_5_20 """select c_person, c_user, json_extract(c_person, '\$.[0].name'), json_extract(c_user, '\$.[1].last') from test1;""" + def query_catalogs = { -> + sql """switch internal""" + sql """use regression_test_external_table_p0_es""" + order_qt_sql01 """select * from test_v1 where test2='text#1'""" + order_qt_sql02 """select * from test_v1 where esquery(test2, '{"match":{"test2":"text#1"}}')""" + order_qt_sql03 """select test4,test5,test6,test7,test8 from test_v1 order by test8""" + order_qt_sql04 """select message from test_v1 where message != ''""" + order_qt_sql05 """select message from test_v1 where message is not null""" + order_qt_sql06 """select message from test_v1 where not_null_or_empty(message)""" + order_qt_sql07 """select * from test_v1 where esquery(c_datetime, '{"term":{"c_datetime":"2020-01-01 12:00:00"}}');""" + order_qt_sql08 """select c_person, c_user, json_extract(c_person, '\$.[0].name'), json_extract(c_user, '\$.[1].last') from test_v1;""" + + order_qt_sql20 """select * from test_v2 where test2='text#1'""" + order_qt_sql21 """select * from test_v2 where esquery(test2, '{"match":{"test2":"text#1"}}')""" + order_qt_sql22 """select test4,test5,test6,test7,test8 from test_v2 order by test8""" + order_qt_sql23 """select * from test_v2 where esquery(c_long, '{"term":{"c_long":"-1"}}');""" + order_qt_sql24 """select c_person, c_user, json_extract(c_person, '\$.[0].name'), json_extract(c_user, '\$.[1].last') from test_v2;""" - sql """switch test_es_query_es6""" - // order_qt_sql_6_01 """show tables""" - order_qt_sql_6_02 """select * from test1 where test2='text#1'""" - order_qt_sql_6_03 """select * from test2_20220808 where test4 >= '2022-08-08 00:00:00' and test4 < '2022-08-08 23:59:59'""" - order_qt_sql_6_04 """select * from test2_20220808 where substring(test2, 2) = 'ext2'""" - order_qt_sql_6_05 """select c_bool[1], c_byte[1], c_short[1], c_integer[1], c_long[1], c_unsigned_long[1], c_float[1], c_half_float[1], c_double[1], c_scaled_float[1], c_date[1], c_datetime[1], c_keyword[1], c_text[1], c_ip[1], json_extract(c_person, '\$.[0]') from test1""" - order_qt_sql_6_06 """select c_bool[1], c_byte[1], c_short[1], c_integer[1], c_long[1], c_unsigned_long[1], c_float[1], c_half_float[1], c_double[1], c_scaled_float[1], c_date[1], c_datetime[1], c_keyword[1], c_text[1], c_ip[1], json_extract(c_person, '\$.[0]') from test2_20220808""" - order_qt_sql_6_07 """select * from test1 where esquery(test2, '{"match":{"test2":"text#1"}}')""" - order_qt_sql_6_08 """select c_bool, c_byte, c_short, c_integer, c_long, c_unsigned_long, c_float, c_half_float, c_double, c_scaled_float, c_date, c_datetime, c_keyword, c_text, c_ip, c_person from test1""" - order_qt_sql_6_09 """select c_bool, c_byte, c_short, c_integer, c_long, c_unsigned_long, c_float, c_half_float, c_double, c_scaled_float, c_date, c_datetime, c_keyword, c_text, c_ip, c_person from test2_20220808""" - order_qt_sql_6_10 """select * from test1 where test1='string1'""" - order_qt_sql_6_11 """select * from test1 where test1='string2'""" - order_qt_sql_6_12 """select * from test1 where test1='string3'""" - order_qt_sql_6_13 """select test6 from test1 where test1='string1'""" - order_qt_sql_6_14 """select test6 from test1 where test1='string2'""" - order_qt_sql_6_15 """select test6 from test1 where test1='string3'""" - order_qt_sql_6_16 """select message from test1 where message != ''""" - order_qt_sql_6_17 """select message from test1 where message is not null""" - order_qt_sql_6_18 """select message from test1 where not_null_or_empty(message)""" - order_qt_sql_6_19 """select * from test1 where esquery(c_person, '{"match":{"c_person.name":"Andy"}}')""" - order_qt_sql_6_20 """select c_person, c_user, json_extract(c_person, '\$.[0].name'), json_extract(c_user, '\$.[1].last') from test1;""" + sql """switch test_es_query_es5""" + order_qt_sql_5_02 """select * from test1 where test2='text#1'""" + order_qt_sql_5_03 """select * from test2_20220808 where test4 >= '2022-08-08 00:00:00' and test4 < '2022-08-08 23:59:59'""" + order_qt_sql_5_04 """select * from test2_20220808 where substring(test2, 2) = 'ext2'""" + order_qt_sql_5_05 """select c_bool[1], c_byte[1], c_short[1], c_integer[1], c_long[1], c_unsigned_long[1], c_float[1], c_half_float[1], c_double[1], c_scaled_float[1], c_date[1], c_datetime[1], c_keyword[1], c_text[1], c_ip[1], json_extract(c_person, '\$.[0]') from test1""" + order_qt_sql_5_06 """select c_bool[1], c_byte[1], c_short[1], c_integer[1], c_long[1], c_unsigned_long[1], c_float[1], c_half_float[1], c_double[1], c_scaled_float[1], c_date[1], c_datetime[1], c_keyword[1], c_text[1], c_ip[1], json_extract(c_person, '\$.[0]') from test2_20220808""" + order_qt_sql_5_07 """select * from test1 where esquery(test2, '{"match":{"test2":"text#1"}}')""" + order_qt_sql_5_08 """select c_bool, c_byte, c_short, c_integer, c_long, c_unsigned_long, c_float, c_half_float, c_double, c_scaled_float, c_date, c_datetime, c_keyword, c_text, c_ip, c_person from test1""" + order_qt_sql_5_09 """select c_bool, c_byte, c_short, c_integer, c_long, c_unsigned_long, c_float, c_half_float, c_double, c_scaled_float, c_date, c_datetime, c_keyword, c_text, c_ip, c_person from test2_20220808""" + order_qt_sql_5_10 """select * from test1 where test1='string1'""" + order_qt_sql_5_11 """select * from test1 where test1='string2'""" + order_qt_sql_5_12 """select * from test1 where test1='string3'""" + order_qt_sql_5_13 """select test6 from test1 where test1='string1'""" + order_qt_sql_5_14 """select test6 from test1 where test1='string2'""" + order_qt_sql_5_15 """select test6 from test1 where test1='string3'""" + order_qt_sql_5_16 """select message from test1 where message != ''""" + order_qt_sql_5_17 """select message from test1 where message is not null""" + order_qt_sql_5_18 """select message from test1 where not_null_or_empty(message)""" + order_qt_sql_5_19 """select * from test1 where esquery(c_unsigned_long, '{"match":{"c_unsigned_long":0}}')""" + order_qt_sql_5_20 """select c_person, c_user, json_extract(c_person, '\$.[0].name'), json_extract(c_user, '\$.[1].last') from test1;""" - List> tables6N = sql """show tables""" - boolean notContainHide = true - tables6N.forEach { - if (it[0] == ".hide"){ - notContainHide = false + sql """switch test_es_query_es6""" + // order_qt_sql_6_01 """show tables""" + order_qt_sql_6_02 """select * from test1 where test2='text#1'""" + order_qt_sql_6_03 """select * from test2_20220808 where test4 >= '2022-08-08 00:00:00' and test4 < '2022-08-08 23:59:59'""" + order_qt_sql_6_04 """select * from test2_20220808 where substring(test2, 2) = 'ext2'""" + order_qt_sql_6_05 """select c_bool[1], c_byte[1], c_short[1], c_integer[1], c_long[1], c_unsigned_long[1], c_float[1], c_half_float[1], c_double[1], c_scaled_float[1], c_date[1], c_datetime[1], c_keyword[1], c_text[1], c_ip[1], json_extract(c_person, '\$.[0]') from test1""" + order_qt_sql_6_06 """select c_bool[1], c_byte[1], c_short[1], c_integer[1], c_long[1], c_unsigned_long[1], c_float[1], c_half_float[1], c_double[1], c_scaled_float[1], c_date[1], c_datetime[1], c_keyword[1], c_text[1], c_ip[1], json_extract(c_person, '\$.[0]') from test2_20220808""" + order_qt_sql_6_07 """select * from test1 where esquery(test2, '{"match":{"test2":"text#1"}}')""" + order_qt_sql_6_08 """select c_bool, c_byte, c_short, c_integer, c_long, c_unsigned_long, c_float, c_half_float, c_double, c_scaled_float, c_date, c_datetime, c_keyword, c_text, c_ip, c_person from test1""" + order_qt_sql_6_09 """select c_bool, c_byte, c_short, c_integer, c_long, c_unsigned_long, c_float, c_half_float, c_double, c_scaled_float, c_date, c_datetime, c_keyword, c_text, c_ip, c_person from test2_20220808""" + order_qt_sql_6_10 """select * from test1 where test1='string1'""" + order_qt_sql_6_11 """select * from test1 where test1='string2'""" + order_qt_sql_6_12 """select * from test1 where test1='string3'""" + order_qt_sql_6_13 """select test6 from test1 where test1='string1'""" + order_qt_sql_6_14 """select test6 from test1 where test1='string2'""" + order_qt_sql_6_15 """select test6 from test1 where test1='string3'""" + order_qt_sql_6_16 """select message from test1 where message != ''""" + order_qt_sql_6_17 """select message from test1 where message is not null""" + order_qt_sql_6_18 """select message from test1 where not_null_or_empty(message)""" + order_qt_sql_6_19 """select * from test1 where esquery(c_person, '{"match":{"c_person.name":"Andy"}}')""" + order_qt_sql_6_20 """select c_person, c_user, json_extract(c_person, '\$.[0].name'), json_extract(c_user, '\$.[1].last') from test1;""" + + List> tables6N = sql """show tables""" + boolean notContainHide = true + tables6N.forEach { + if (it[0] == ".hide"){ + notContainHide = false + } } - } - assertTrue(notContainHide) + assertTrue(notContainHide) - sql """switch es6_hide""" - List> tables6Y = sql """show tables""" - boolean containHide = false - tables6Y.forEach { - if (it[0] == ".hide"){ - containHide = true + sql """switch es6_hide""" + List> tables6Y = sql """show tables""" + boolean containHide = false + tables6Y.forEach { + if (it[0] == ".hide"){ + containHide = true + } } - } - assertTrue(containHide) + assertTrue(containHide) - sql """switch test_es_query_es7""" - // order_qt_sql_7_01 """show tables""" - order_qt_sql_7_02 """select * from test1 where test2='text#1'""" - order_qt_sql_7_03 """select * from test2_20220808 where test4 >= '2022-08-08 00:00:00' and test4 < '2022-08-08 23:59:59'""" - order_qt_sql_7_04 """select * from test2_20220808 where substring(test2, 2) = 'ext2'""" - order_qt_sql_7_05 """select c_bool[1], c_byte[1], c_short[1], c_integer[1], c_long[1], c_unsigned_long[1], c_float[1], c_half_float[1], c_double[1], c_scaled_float[1], c_date[1], c_datetime[1], c_keyword[1], c_text[1], c_ip[1], json_extract(c_person, '\$.[0]') from test1""" - order_qt_sql_7_06 """select c_bool[1], c_byte[1], c_short[1], c_integer[1], c_long[1], c_unsigned_long[1], c_float[1], c_half_float[1], c_double[1], c_scaled_float[1], c_date[1], c_datetime[1], c_keyword[1], c_text[1], c_ip[1], json_extract(c_person, '\$.[0]') from test2""" - order_qt_sql_7_07 """select * from test1 where esquery(test2, '{"match":{"test2":"text#1"}}')""" - order_qt_sql_7_08 """select c_bool, c_byte, c_short, c_integer, c_long, c_unsigned_long, c_float, c_half_float, c_double, c_scaled_float, c_date, c_datetime, c_keyword, c_text, c_ip, c_person from test1""" - order_qt_sql_7_09 """select c_bool, c_byte, c_short, c_integer, c_long, c_unsigned_long, c_float, c_half_float, c_double, c_scaled_float, c_date, c_datetime, c_keyword, c_text, c_ip, c_person from test2""" - order_qt_sql_7_10 """select * from test3_20231005""" - order_qt_sql_7_11 """select * from test1 where test1='string1'""" - order_qt_sql_7_12 """select * from test1 where test1='string2'""" - order_qt_sql_7_13 """select * from test1 where test1='string3'""" - order_qt_sql_7_14 """select * from test1 where test1='string4'""" - order_qt_sql_7_15 """select test10 from test1 where test1='string1'""" - order_qt_sql_7_16 """select test10 from test1 where test1='string2'""" - order_qt_sql_7_17 """select test10 from test1 where test1='string3'""" - order_qt_sql_7_18 """select test10 from test1 where test1='string4'""" - order_qt_sql_7_19 """select message from test1 where message != ''""" - order_qt_sql_7_20 """select message from test1 where message is not null""" - order_qt_sql_7_21 """select message from test1 where not_null_or_empty(message)""" - order_qt_sql_7_22 """select * from test1 where esquery(my_wildcard, '{ "wildcard": { "my_wildcard": { "value":"*quite*lengthy" } } }');""" - order_qt_sql_7_23 """select * from test1 where level = 'debug'""" - order_qt_sql_7_24 """select * from test1 where esquery(c_float, '{"match":{"c_float":1.1}}')""" - order_qt_sql_7_25 """select c_person, c_user, json_extract(c_person, '\$.[0].name'), json_extract(c_user, '\$.[1].last') from test1;""" + sql """switch test_es_query_es7""" + // order_qt_sql_7_01 """show tables""" + order_qt_sql_7_02 """select * from test1 where test2='text#1'""" + order_qt_sql_7_03 """select * from test2_20220808 where test4 >= '2022-08-08 00:00:00' and test4 < '2022-08-08 23:59:59'""" + order_qt_sql_7_04 """select * from test2_20220808 where substring(test2, 2) = 'ext2'""" + order_qt_sql_7_05 """select c_bool[1], c_byte[1], c_short[1], c_integer[1], c_long[1], c_unsigned_long[1], c_float[1], c_half_float[1], c_double[1], c_scaled_float[1], c_date[1], c_datetime[1], c_keyword[1], c_text[1], c_ip[1], json_extract(c_person, '\$.[0]') from test1""" + order_qt_sql_7_06 """select c_bool[1], c_byte[1], c_short[1], c_integer[1], c_long[1], c_unsigned_long[1], c_float[1], c_half_float[1], c_double[1], c_scaled_float[1], c_date[1], c_datetime[1], c_keyword[1], c_text[1], c_ip[1], json_extract(c_person, '\$.[0]') from test2""" + order_qt_sql_7_07 """select * from test1 where esquery(test2, '{"match":{"test2":"text#1"}}')""" + order_qt_sql_7_08 """select c_bool, c_byte, c_short, c_integer, c_long, c_unsigned_long, c_float, c_half_float, c_double, c_scaled_float, c_date, c_datetime, c_keyword, c_text, c_ip, c_person from test1""" + order_qt_sql_7_09 """select c_bool, c_byte, c_short, c_integer, c_long, c_unsigned_long, c_float, c_half_float, c_double, c_scaled_float, c_date, c_datetime, c_keyword, c_text, c_ip, c_person from test2""" + order_qt_sql_7_10 """select * from test3_20231005""" + order_qt_sql_7_11 """select * from test1 where test1='string1'""" + order_qt_sql_7_12 """select * from test1 where test1='string2'""" + order_qt_sql_7_13 """select * from test1 where test1='string3'""" + order_qt_sql_7_14 """select * from test1 where test1='string4'""" + order_qt_sql_7_15 """select test10 from test1 where test1='string1'""" + order_qt_sql_7_16 """select test10 from test1 where test1='string2'""" + order_qt_sql_7_17 """select test10 from test1 where test1='string3'""" + order_qt_sql_7_18 """select test10 from test1 where test1='string4'""" + order_qt_sql_7_19 """select message from test1 where message != ''""" + order_qt_sql_7_20 """select message from test1 where message is not null""" + order_qt_sql_7_21 """select message from test1 where not_null_or_empty(message)""" + order_qt_sql_7_22 """select * from test1 where esquery(my_wildcard, '{ "wildcard": { "my_wildcard": { "value":"*quite*lengthy" } } }');""" + order_qt_sql_7_23 """select * from test1 where level = 'debug'""" + order_qt_sql_7_24 """select * from test1 where esquery(c_float, '{"match":{"c_float":1.1}}')""" + order_qt_sql_7_25 """select c_person, c_user, json_extract(c_person, '\$.[0].name'), json_extract(c_user, '\$.[1].last') from test1;""" - List> tables7N = sql """show tables""" - boolean notContainHide7 = true - tables7N.forEach { - if (it[0] == ".hide"){ - notContainHide7 = false + List> tables7N = sql """show tables""" + boolean notContainHide7 = true + tables7N.forEach { + if (it[0] == ".hide"){ + notContainHide7 = false + } } - } - assertTrue(notContainHide7) + assertTrue(notContainHide7) - sql """switch es7_hide""" - List> tables7Y = sql """show tables""" - boolean containeHide7 = false - tables7Y.forEach { - if (it[0] == (".hide")){ - containeHide7 = true + sql """switch es7_hide""" + List> tables7Y = sql """show tables""" + boolean containeHide7 = false + tables7Y.forEach { + if (it[0] == (".hide")){ + containeHide7 = true + } } + assertTrue(containeHide7) + + order_qt_sql_7_26 """select * from test3_20231005""" + + sql """switch test_es_query_es8""" + order_qt_sql_8_01 """select * from test1 where test2='text#1'""" + order_qt_sql_8_02 """select * from test2_20220808 where test4 >= '2022-08-08 00:00:00' and test4 < '2022-08-08 23:59:59'""" + order_qt_sql_8_03 """select c_bool[1], c_byte[1], c_short[1], c_integer[1], c_long[1], c_unsigned_long[1], c_float[1], c_half_float[1], c_double[1], c_scaled_float[1], c_date[1], c_datetime[1], c_keyword[1], c_text[1], c_ip[1], json_extract(c_person, '\$.[0]') from test1""" + order_qt_sql_8_04 """select c_bool[1], c_byte[1], c_short[1], c_integer[1], c_long[1], c_unsigned_long[1], c_float[1], c_half_float[1], c_double[1], c_scaled_float[1], c_date[1], c_datetime[1], c_keyword[1], c_text[1], c_ip[1], json_extract(c_person, '\$.[0]') from test2""" + order_qt_sql_8_05 """select * from test1 where esquery(test2, '{"match":{"test2":"text#1"}}')""" + order_qt_sql_8_06 """select c_bool, c_byte, c_short, c_integer, c_long, c_unsigned_long, c_float, c_half_float, c_double, c_scaled_float, c_date, c_datetime, c_keyword, c_text, c_ip, c_person from test1""" + order_qt_sql_8_07 """select c_bool, c_byte, c_short, c_integer, c_long, c_unsigned_long, c_float, c_half_float, c_double, c_scaled_float, c_date, c_datetime, c_keyword, c_text, c_ip, c_person from test2""" + order_qt_sql_8_08 """select * from test3_20231005""" + order_qt_sql_8_09 """select * from test1 where test1='string1'""" + order_qt_sql_8_10 """select * from test1 where test1='string2'""" + order_qt_sql_8_11 """select * from test1 where test1='string3'""" + order_qt_sql_8_12 """select * from test1 where test1='string4'""" + order_qt_sql_8_13 """select test10 from test1 where test1='string1'""" + order_qt_sql_8_14 """select test10 from test1 where test1='string2'""" + order_qt_sql_8_15 """select test10 from test1 where test1='string3'""" + order_qt_sql_8_16 """select test10 from test1 where test1='string4'""" + order_qt_sql_8_17 """select message from test1 where message != ''""" + order_qt_sql_8_18 """select message from test1 where message is not null""" + order_qt_sql_8_19 """select message from test1 where not_null_or_empty(message)""" + order_qt_sql_8_20 """select * from test1 where esquery(my_wildcard, '{ "wildcard": { "my_wildcard": { "value":"*quite*lengthy" } } }');""" + order_qt_sql_8_21 """select * from test1 where level = 'debug'""" + order_qt_sql_8_22 """select * from test1 where esquery(c_ip, '{"match":{"c_ip":"192.168.0.1"}}')""" + order_qt_sql_8_23 """select c_person, c_user, json_extract(c_person, '\$.[0].name'), json_extract(c_user, '\$.[1].last') from test1;""" + } - assertTrue(containeHide7) - order_qt_sql_7_26 """select * from test3_20231005""" + sql """set enable_es_parallel_scroll=true""" + query_catalogs() - sql """switch test_es_query_es8""" - order_qt_sql_8_01 """select * from test1 where test2='text#1'""" - order_qt_sql_8_02 """select * from test2_20220808 where test4 >= '2022-08-08 00:00:00' and test4 < '2022-08-08 23:59:59'""" - order_qt_sql_8_03 """select c_bool[1], c_byte[1], c_short[1], c_integer[1], c_long[1], c_unsigned_long[1], c_float[1], c_half_float[1], c_double[1], c_scaled_float[1], c_date[1], c_datetime[1], c_keyword[1], c_text[1], c_ip[1], json_extract(c_person, '\$.[0]') from test1""" - order_qt_sql_8_04 """select c_bool[1], c_byte[1], c_short[1], c_integer[1], c_long[1], c_unsigned_long[1], c_float[1], c_half_float[1], c_double[1], c_scaled_float[1], c_date[1], c_datetime[1], c_keyword[1], c_text[1], c_ip[1], json_extract(c_person, '\$.[0]') from test2""" - order_qt_sql_8_05 """select * from test1 where esquery(test2, '{"match":{"test2":"text#1"}}')""" - order_qt_sql_8_06 """select c_bool, c_byte, c_short, c_integer, c_long, c_unsigned_long, c_float, c_half_float, c_double, c_scaled_float, c_date, c_datetime, c_keyword, c_text, c_ip, c_person from test1""" - order_qt_sql_8_07 """select c_bool, c_byte, c_short, c_integer, c_long, c_unsigned_long, c_float, c_half_float, c_double, c_scaled_float, c_date, c_datetime, c_keyword, c_text, c_ip, c_person from test2""" - order_qt_sql_8_08 """select * from test3_20231005""" - order_qt_sql_8_09 """select * from test1 where test1='string1'""" - order_qt_sql_8_10 """select * from test1 where test1='string2'""" - order_qt_sql_8_11 """select * from test1 where test1='string3'""" - order_qt_sql_8_12 """select * from test1 where test1='string4'""" - order_qt_sql_8_13 """select test10 from test1 where test1='string1'""" - order_qt_sql_8_14 """select test10 from test1 where test1='string2'""" - order_qt_sql_8_15 """select test10 from test1 where test1='string3'""" - order_qt_sql_8_16 """select test10 from test1 where test1='string4'""" - order_qt_sql_8_17 """select message from test1 where message != ''""" - order_qt_sql_8_18 """select message from test1 where message is not null""" - order_qt_sql_8_19 """select message from test1 where not_null_or_empty(message)""" - order_qt_sql_8_20 """select * from test1 where esquery(my_wildcard, '{ "wildcard": { "my_wildcard": { "value":"*quite*lengthy" } } }');""" - order_qt_sql_8_21 """select * from test1 where level = 'debug'""" - order_qt_sql_8_22 """select * from test1 where esquery(c_ip, '{"match":{"c_ip":"192.168.0.1"}}')""" - order_qt_sql_8_23 """select c_person, c_user, json_extract(c_person, '\$.[0].name'), json_extract(c_user, '\$.[1].last') from test1;""" + sql """set enable_es_parallel_scroll=false""" + query_catalogs() } } diff --git a/samples/datalake/hudi/README.md b/samples/datalake/hudi/README.md new file mode 100644 index 000000000000000..507c5902c5ea869 --- /dev/null +++ b/samples/datalake/hudi/README.md @@ -0,0 +1,207 @@ + + +# Doris+Hudi+MINIO Environments +Launch spark/doris/hive/hudi/minio test environments, and give examples to query hudi in Doris. + +## Launch Docker Compose +**Create Network** +```shell +sudo docker network create -d bridge hudi-net +``` +**Launch all components in docker** +```shell +sudo ./start-hudi-compose.sh +``` +**Login into Spark** +```shell +sudo ./login-spark.sh +``` +**Login into Doris** +```shell +sudo ./login-doris.sh +``` + +## Prepare Hudi Data +There's already a hive table named `customer` in hive default. Create a hudi table from the hive table: +```sql +-- ./login-spark.sh +use default; + +-- create a COW table +CREATE TABLE customer_cow +USING hudi +TBLPROPERTIES ( + type = 'cow', + primaryKey = 'c_custkey', + preCombineField = 'c_name' +) +PARTITIONED BY (c_nationkey) +AS SELECT * FROM customer; + +-- create a MOR table +CREATE TABLE customer_mor +USING hudi +TBLPROPERTIES ( + type = 'mor', + primaryKey = 'c_custkey', + preCombineField = 'c_name' +) +PARTITIONED BY (c_nationkey) +AS SELECT * FROM customer; +``` + +## Query Data +Doris refresh hive catalog in [10min in default](https://doris.apache.org/docs/lakehouse/datalake-analytics/hive/#metadata-cache--refresh), +users can refresh directly to access the hudi table in Doris by `doris> refresh catalog hive;` + +After hudi table is ready in Doris, all operations in hudi table will be detected by Doris, and there's no need to refresh catalog or tables. + +Insert new data into hudi tables in spark-sql: +```sql +spark-sql> insert into customer_cow values (100, "Customer#000000100", "jD2xZzi", "25-430-914-2194", 3471.59, "BUILDING", "cial ideas. final, furious requests", 25); +spark-sql> insert into customer_mor values (100, "Customer#000000100", "jD2xZzi", "25-430-914-2194", 3471.59, "BUILDING", "cial ideas. final, furious requests", 25); +``` +`c_nationkey=25` is a new partition, doris can query the new data at once without refresh: +```sql +doris> use hive.default; +doris> select * from customer_cow where c_custkey = 100; +doris> select * from customer_mor where c_custkey = 100; +``` +Insert a record with `c_custkey=32`(primary key, already in table) will remove the old record: +```sql +spark-sql> insert into customer_cow values (32, "Customer#000000032_update", "jD2xZzi", "25-430-914-2194", 3471.59, "BUILDING", "cial ideas. final, furious requests", 15); +spark-sql> insert into customer_mor values (32, "Customer#000000032_update", "jD2xZzi", "25-430-914-2194", 3471.59, "BUILDING", "cial ideas. final, furious requests", 15); +``` +Query the updated data at once in doris: +```sql +doris> select * from customer_cow where c_custkey = 32; ++-----------+---------------------------+-----------+-----------------+-----------+--------------+-------------------------------------+-------------+ +| c_custkey | c_name | c_address | c_phone | c_acctbal | c_mktsegment | c_comment | c_nationkey | ++-----------+---------------------------+-----------+-----------------+-----------+--------------+-------------------------------------+-------------+ +| 32 | Customer#000000032_update | jD2xZzi | 25-430-914-2194 | 3471.59 | BUILDING | cial ideas. final, furious requests | 15 | ++-----------+---------------------------+-----------+-----------------+-----------+--------------+-------------------------------------+-------------+ +doris> select * from customer_mor where c_custkey = 32; ++-----------+---------------------------+-----------+-----------------+-----------+--------------+-------------------------------------+-------------+ +| c_custkey | c_name | c_address | c_phone | c_acctbal | c_mktsegment | c_comment | c_nationkey | ++-----------+---------------------------+-----------+-----------------+-----------+--------------+-------------------------------------+-------------+ +| 32 | Customer#000000032_update | jD2xZzi | 25-430-914-2194 | 3471.59 | BUILDING | cial ideas. final, furious requests | 15 | ++-----------+---------------------------+-----------+-----------------+-----------+--------------+-------------------------------------+-------------+ +``` + +## Query Optimization +Doris uses native reader(c++) to read the data files of the **COW** table, and uses the Java SDK (By calling hudi-bundle through JNI) to read the data files of the **MOR** table. In upsert scenario, there may still remains base files that have not been updated in the MOR table, which can be read through the native reader. Users can view the execution plan of hudi scan through the explain command, where `hudiNativeReadSplits` indicates how many split files are read through the native reader. +```sql +-- COW table is read natively +doris> explain select * from customer_cow where c_custkey = 32; +| 0:VHUDI_SCAN_NODE(68) | +| table: customer_cow | +| predicates: (c_custkey[#5] = 32) | +| inputSplitNum=101, totalFileSize=45338886, scanRanges=101 | +| partition=26/26 | +| cardinality=1, numNodes=1 | +| pushdown agg=NONE | +| hudiNativeReadSplits=101/101 | + +-- MOR table: because only the base file contains `c_custkey = 32` that is updated, 100 splits are read natively, while the split with log file is read by JNI. +doris> explain select * from customer_mor where c_custkey = 32; +| 0:VHUDI_SCAN_NODE(68) | +| table: customer_mor | +| predicates: (c_custkey[#5] = 32) | +| inputSplitNum=101, totalFileSize=45340731, scanRanges=101 | +| partition=26/26 | +| cardinality=1, numNodes=1 | +| pushdown agg=NONE | +| hudiNativeReadSplits=100/101 | + +-- Use delete statement to see more differences +spark-sql> delete from customer_cow where c_custkey = 64; +doris> explain select * from customer_cow where c_custkey = 64; + +spark-sql> delete from customer_mor where c_custkey = 64; +doris> explain select * from customer_mor where c_custkey = 64; + +-- customer_xxx is partitioned by c_nationkey, we can use the partition column to prune data +doris> explain select * from customer_mor where c_custkey = 64 and c_nationkey = 15; +| 0:VHUDI_SCAN_NODE(68) | +| table: customer_mor | +| predicates: (c_custkey[#5] = 64), (c_nationkey[#12] = 15) | +| inputSplitNum=4, totalFileSize=1798186, scanRanges=4 | +| partition=1/26 | +| cardinality=1, numNodes=1 | +| pushdown agg=NONE | +| hudiNativeReadSplits=3/4 | +``` + +## TimeTravel +See the commit metadata in spark-sql: +```sql +spark-sql> call show_commits(table => 'customer_cow', limit => 10); +20240603033556094 20240603033558249 commit 448833 0 1 1 183 0 0 +20240603015444737 20240603015446588 commit 450238 0 1 1 202 1 0 +20240603015018572 20240603015020503 commit 436692 1 0 1 1 0 0 +20240603013858098 20240603013907467 commit 44902033 100 0 25 18751 0 0 + +spark-sql> call show_commits(table => 'customer_mor', limit => 10); +20240603033745977 20240603033748021 deltacommit 1240 0 1 1 0 0 0 +20240603015451860 20240603015453539 deltacommit 1434 0 1 1 1 1 0 +20240603015058442 20240603015100120 deltacommit 436691 1 0 1 1 0 0 +20240603013918515 20240603013922961 deltacommit 44904040 100 0 25 18751 0 0 +``` +Let's travel to the commit we insert `c_custkey=100` in doris where `c_custkey=32` is not updated: +```sql +doris> select * from customer_cow for time as of '20240603015018572' where c_custkey = 32 or c_custkey = 100; ++-----------+--------------------+---------------------------------------+-----------------+-----------+--------------+--------------------------------------------------+-------------+ +| c_custkey | c_name | c_address | c_phone | c_acctbal | c_mktsegment | c_comment | c_nationkey | ++-----------+--------------------+---------------------------------------+-----------------+-----------+--------------+--------------------------------------------------+-------------+ +| 32 | Customer#000000032 | jD2xZzi UmId,DCtNBLXKj9q0Tlp2iQ6ZcO3J | 25-430-914-2194 | 3471.53 | BUILDING | cial ideas. final, furious requests across the e | 15 | +| 100 | Customer#000000100 | jD2xZzi | 25-430-914-2194 | 3471.59 | BUILDING | cial ideas. final, furious requests | 25 | ++-----------+--------------------+---------------------------------------+-----------------+-----------+--------------+--------------------------------------------------+-------------+ +-- compare with spark-sql +spark-sql> select * from customer_mor timestamp as of '20240603015018572' where c_custkey = 32 or c_custkey = 100; + +doris> select * from customer_mor for time as of '20240603015058442' where c_custkey = 32 or c_custkey = 100; ++-----------+--------------------+---------------------------------------+-----------------+-----------+--------------+--------------------------------------------------+-------------+ +| c_custkey | c_name | c_address | c_phone | c_acctbal | c_mktsegment | c_comment | c_nationkey | ++-----------+--------------------+---------------------------------------+-----------------+-----------+--------------+--------------------------------------------------+-------------+ +| 100 | Customer#000000100 | jD2xZzi | 25-430-914-2194 | 3471.59 | BUILDING | cial ideas. final, furious requests | 25 | +| 32 | Customer#000000032 | jD2xZzi UmId,DCtNBLXKj9q0Tlp2iQ6ZcO3J | 25-430-914-2194 | 3471.53 | BUILDING | cial ideas. final, furious requests across the e | 15 | ++-----------+--------------------+---------------------------------------+-----------------+-----------+--------------+--------------------------------------------------+-------------+ +spark-sql> select * from customer_mor timestamp as of '20240603015058442' where c_custkey = 32 or c_custkey = 100; +``` + +## Incremental Read +Seed the data changed between after inserting `c_custkey=100` +```sql +doris> select * from customer_cow@incr('beginTime'='20240603015018572'); ++-----------+---------------------------+-----------+-----------------+-----------+--------------+-------------------------------------+-------------+ +| c_custkey | c_name | c_address | c_phone | c_acctbal | c_mktsegment | c_comment | c_nationkey | ++-----------+---------------------------+-----------+-----------------+-----------+--------------+-------------------------------------+-------------+ +| 32 | Customer#000000032_update | jD2xZzi | 25-430-914-2194 | 3471.59 | BUILDING | cial ideas. final, furious requests | 15 | ++-----------+---------------------------+-----------+-----------------+-----------+--------------+-------------------------------------+-------------+ +spark-sql> select * from hudi_table_changes('customer_cow', 'latest_state', '20240603015018572'); + +doris> select * from customer_mor@incr('beginTime'='20240603015058442'); ++-----------+---------------------------+-----------+-----------------+-----------+--------------+-------------------------------------+-------------+ +| c_custkey | c_name | c_address | c_phone | c_acctbal | c_mktsegment | c_comment | c_nationkey | ++-----------+---------------------------+-----------+-----------------+-----------+--------------+-------------------------------------+-------------+ +| 32 | Customer#000000032_update | jD2xZzi | 25-430-914-2194 | 3471.59 | BUILDING | cial ideas. final, furious requests | 15 | ++-----------+---------------------------+-----------+-----------------+-----------+--------------+-------------------------------------+-------------+ +spark-sql> select * from hudi_table_changes('customer_mor', 'latest_state', '20240603015058442'); +``` diff --git a/samples/datalake/hudi/data/customer/000000_0.parquet b/samples/datalake/hudi/data/customer/000000_0.parquet new file mode 100644 index 000000000000000..4521df9f9772e1d Binary files /dev/null and b/samples/datalake/hudi/data/customer/000000_0.parquet differ diff --git a/samples/datalake/hudi/data/customer/000001_0.parquet b/samples/datalake/hudi/data/customer/000001_0.parquet new file mode 100644 index 000000000000000..c3f4625c76580b6 Binary files /dev/null and b/samples/datalake/hudi/data/customer/000001_0.parquet differ diff --git a/samples/datalake/hudi/data/customer/000002_0.parquet b/samples/datalake/hudi/data/customer/000002_0.parquet new file mode 100644 index 000000000000000..bd8aeb3b419638c Binary files /dev/null and b/samples/datalake/hudi/data/customer/000002_0.parquet differ diff --git a/samples/datalake/hudi/data/customer/000003_0.parquet b/samples/datalake/hudi/data/customer/000003_0.parquet new file mode 100644 index 000000000000000..1cb94ee0708fb04 Binary files /dev/null and b/samples/datalake/hudi/data/customer/000003_0.parquet differ diff --git a/samples/datalake/hudi/hudi-compose.env b/samples/datalake/hudi/hudi-compose.env new file mode 100644 index 000000000000000..9c693e12510d3d2 --- /dev/null +++ b/samples/datalake/hudi/hudi-compose.env @@ -0,0 +1,21 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +HIVE_THRIFT_PORT=9771 +MINIO_SERVICE_PORT=9772 +DORIS_QUERY_PORT=9773 diff --git a/samples/datalake/hudi/hudi-compose.yml b/samples/datalake/hudi/hudi-compose.yml new file mode 100644 index 000000000000000..59d3045153718bf --- /dev/null +++ b/samples/datalake/hudi/hudi-compose.yml @@ -0,0 +1,109 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +version: "3.9" +services: + metastore_db: + image: postgres:11 + hostname: metastore_db + environment: + POSTGRES_USER: hive + POSTGRES_PASSWORD: hive + POSTGRES_DB: metastore + + hive-metastore: + hostname: hive-metastore + container_name: spark-hudi-hive + image: 'starburstdata/hive:3.1.2-e.18' + volumes: + - './packages/spark-3.4.2-bin-hadoop3:/opt/spark-3.4.2-bin-hadoop3' + - './scripts:/opt/scripts' + ports: + - '${HIVE_THRIFT_PORT}:9083' # Metastore Thrift + environment: + HIVE_METASTORE_DRIVER: org.postgresql.Driver + HIVE_METASTORE_JDBC_URL: jdbc:postgresql://metastore_db:5432/metastore + HIVE_METASTORE_USER: hive + HIVE_METASTORE_PASSWORD: hive + HIVE_METASTORE_WAREHOUSE_DIR: s3://datalake/ + S3_ENDPOINT: http://minio:9000 + S3_ACCESS_KEY: minio + S3_SECRET_KEY: minio123 + S3_PATH_STYLE_ACCESS: "true" + REGION: "" + GOOGLE_CLOUD_KEY_FILE_PATH: "" + AZURE_ADL_CLIENT_ID: "" + AZURE_ADL_CREDENTIAL: "" + AZURE_ADL_REFRESH_URL: "" + AZURE_ABFS_STORAGE_ACCOUNT: "" + AZURE_ABFS_ACCESS_KEY: "" + AZURE_WASB_STORAGE_ACCOUNT: "" + AZURE_ABFS_OAUTH: "" + AZURE_ABFS_OAUTH_TOKEN_PROVIDER: "" + AZURE_ABFS_OAUTH_CLIENT_ID: "" + AZURE_ABFS_OAUTH_SECRET: "" + AZURE_ABFS_OAUTH_ENDPOINT: "" + AZURE_WASB_ACCESS_KEY: "" + HIVE_METASTORE_USERS_IN_ADMIN_ROLE: "hive" + depends_on: + - metastore_db + + doris-hudi-env: + hostname: doris-hudi-env + container_name: doris-hudi-env + image: 'apache/hadoop:3.3.6' + environment: + LD_LIBRARY_PATH: /opt/doris/be/lib + ports: + - '${DORIS_QUERY_PORT}:9030' + volumes: + - './packages/jdk-17.0.2:/opt/jdk-17.0.2' + - './packages/doris-bin:/opt/doris-bin' + - './scripts:/opt/scripts' + command: sh /opt/scripts/start_doris.sh + + minio: + hostname: minio + image: 'minio/minio:RELEASE.2022-05-26T05-48-41Z' + container_name: minio-hudi-storage + ports: + - '${MINIO_SERVICE_PORT}:9000' + environment: + MINIO_ACCESS_KEY: minio + MINIO_SECRET_KEY: minio123 + command: server /data --console-address ":9001" + + # This job will create the "datalake" bucket on Minio + mc-job: + image: 'minio/mc:RELEASE.2022-05-09T04-08-26Z' + volumes: + - './data:/data' + entrypoint: | + /bin/bash -c " + sleep 5; + /usr/bin/mc config --quiet host add myminio http://minio:9000 minio minio123; + /usr/bin/mc mb --quiet myminio/datalake; + /usr/bin/mc mb --quiet myminio/data; + /usr/bin/mc mirror /data myminio/data + " + depends_on: + - minio + +networks: + default: + name: hudi-net + external: true diff --git a/samples/datalake/hudi/login-doris.sh b/samples/datalake/hudi/login-doris.sh new file mode 100755 index 000000000000000..e30a3f2f99f506f --- /dev/null +++ b/samples/datalake/hudi/login-doris.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +docker exec -it spark-hudi-hive mysql -u root -h doris-hudi-env -P 9030 diff --git a/samples/datalake/hudi/login-spark.sh b/samples/datalake/hudi/login-spark.sh new file mode 100755 index 000000000000000..a9309a2e91eb3e4 --- /dev/null +++ b/samples/datalake/hudi/login-spark.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +docker exec -it spark-hudi-hive /opt/scripts/spark-hudi.sh diff --git a/samples/datalake/hudi/scripts/doris-hudi.sql b/samples/datalake/hudi/scripts/doris-hudi.sql new file mode 100644 index 000000000000000..db447c86fc88602 --- /dev/null +++ b/samples/datalake/hudi/scripts/doris-hudi.sql @@ -0,0 +1,11 @@ +CREATE CATALOG `hive` PROPERTIES ( + "type"="hms", + 'hive.metastore.uris' = 'thrift://hive-metastore:9083', + "s3.access_key" = "minio", + "s3.secret_key" = "minio123", + "s3.endpoint" = "http://minio:9000", + "s3.region" = "us-east-1", + "use_path_style" = "true" +); + +ALTER SYSTEM ADD BACKEND 'doris-hudi-env:9050'; diff --git a/samples/datalake/hudi/scripts/hive-minio.sql b/samples/datalake/hudi/scripts/hive-minio.sql new file mode 100644 index 000000000000000..4ca4b27293d85ba --- /dev/null +++ b/samples/datalake/hudi/scripts/hive-minio.sql @@ -0,0 +1,13 @@ +USE default; + +CREATE EXTERNAL TABLE `customer`( + `c_custkey` int, + `c_name` varchar(25), + `c_address` varchar(40), + `c_nationkey` int, + `c_phone` char(15), + `c_acctbal` decimal(12,2), + `c_mktsegment` char(10), + `c_comment` varchar(117)) +STORED AS parquet +LOCATION 's3://data/customer'; diff --git a/samples/datalake/hudi/scripts/spark-hudi.sh b/samples/datalake/hudi/scripts/spark-hudi.sh new file mode 100755 index 000000000000000..cfb3ee06d948973 --- /dev/null +++ b/samples/datalake/hudi/scripts/spark-hudi.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +export SPARK_HOME=/opt/spark-hudi +export HIVE_HOME=/opt/apache-hive-3.1.2-bin +export HADOOP_HOME=/opt/hadoop-3.3.1 + +if [[ ! -d "${SPARK_HOME}" ]]; then + cp -r /opt/spark-3.4.2-bin-hadoop3 "${SPARK_HOME}" +fi + +cp "${HIVE_HOME}"/conf/hive-site.xml "${SPARK_HOME}"/conf/ +cp "${HIVE_HOME}"/lib/postgresql-jdbc.jar "${SPARK_HOME}"/jars/ +cp "${HADOOP_HOME}"/etc/hadoop/core-site.xml "${SPARK_HOME}"/conf/ + +"${SPARK_HOME}"/bin/spark-sql \ + --master local[*] \ + --name "spark-hudi-sql" \ + --conf spark.serializer=org.apache.spark.serializer.KryoSerializer \ + --conf spark.sql.catalog.spark_catalog=org.apache.spark.sql.hudi.catalog.HoodieCatalog \ + --conf spark.sql.extensions=org.apache.spark.sql.hudi.HoodieSparkSessionExtension \ + --conf spark.sql.catalogImplementation=hive diff --git a/samples/datalake/hudi/scripts/start_doris.sh b/samples/datalake/hudi/scripts/start_doris.sh new file mode 100755 index 000000000000000..7204d6451aa8e40 --- /dev/null +++ b/samples/datalake/hudi/scripts/start_doris.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +cp -r /opt/doris-bin /opt/doris + +/opt/doris/fe/bin/start_fe.sh --daemon +/opt/doris/be/bin/start_be.sh --daemon +tail -F /dev/null diff --git a/samples/datalake/hudi/start-hudi-compose.sh b/samples/datalake/hudi/start-hudi-compose.sh new file mode 100755 index 000000000000000..839c5b3e221825a --- /dev/null +++ b/samples/datalake/hudi/start-hudi-compose.sh @@ -0,0 +1,118 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +DORIS_PACKAGE=apache-doris-2.1.4-bin-x64 +DORIS_DOWNLOAD_URL=https://apache-doris-releases.oss-accelerate.aliyuncs.com + +md5_aws_java_sdk="452d1e00efb11bff0ee17c42a6a44a0a" +md5_hadoop_aws="a3e19d42cadd1a6862a41fd276f94382" +md5_hudi_bundle="a9cb8c752d1d7132ef3cfe3ead78a30d" +md5_jdk17="0930efa680ac61e833699ccc36bfc739" +md5_spark="b393d314ffbc03facdc85575197c5db9" +md5_doris="a4d8bc9730aca3a51294e87d7d5b3e8e" + +download_source_file() { + local FILE_PATH="$1" + local EXPECTED_MD5="$2" + local DOWNLOAD_URL="$3" + + echo "Download ${FILE_PATH}" + + if [[ -f "${FILE_PATH}" ]]; then + local FILE_MD5 + FILE_MD5=$(md5sum "${FILE_PATH}" | awk '{ print $1 }') + + if [[ "${FILE_MD5}" = "${EXPECTED_MD5}" ]]; then + echo "${FILE_PATH} is ready!" + else + echo "${FILE_PATH} is broken, Redownloading ..." + rm "${FILE_PATH}" + wget "${DOWNLOAD_URL}"/"${FILE_PATH}" + fi + else + echo "Downloading ${FILE_PATH} ..." + wget "${DOWNLOAD_URL}"/"${FILE_PATH}" + fi +} + +curdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" +cd "${curdir}" || exit + +if [[ ! -d "packages" ]]; then + mkdir packages +fi +cd packages || exit + +download_source_file "aws-java-sdk-bundle-1.12.48.jar" "${md5_aws_java_sdk}" "https://repo1.maven.org/maven2/com/amazonaws/aws-java-sdk-bundle/1.12.48" +download_source_file "hadoop-aws-3.3.1.jar" "${md5_hadoop_aws}" "https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-aws/3.3.1" +download_source_file "hudi-spark3.4-bundle_2.12-0.14.1.jar" "${md5_hudi_bundle}" "https://repo1.maven.org/maven2/org/apache/hudi/hudi-spark3.4-bundle_2.12/0.14.1" +download_source_file "openjdk-17.0.2_linux-x64_bin.tar.gz" "${md5_jdk17}" "https://download.java.net/java/GA/jdk17.0.2/dfd4a8d0985749f896bed50d7138ee7f/8/GPL" +download_source_file "spark-3.4.2-bin-hadoop3.tgz" "${md5_spark}" "https://archive.apache.org/dist/spark/spark-3.4.2" +download_source_file "${DORIS_PACKAGE}.tar.gz" "${md5_doris}" "${DORIS_DOWNLOAD_URL}" + +if [[ ! -f "jdk-17.0.2/SUCCESS" ]]; then + echo "Prepare jdk17 environment" + if [[ -d "jdk-17.0.2" ]]; then + echo "Remove broken jdk-17.0.2" + rm -rf jdk-17.0.2 + fi + echo "Unpackage jdk-17.0.2" + tar xzf openjdk-17.0.2_linux-x64_bin.tar.gz + touch jdk-17.0.2/SUCCESS +fi +if [[ ! -f "spark-3.4.2-bin-hadoop3/SUCCESS" ]]; then + echo "Prepare spark3.4 environment" + if [[ -d "spark-3.4.2-bin-hadoop3" ]]; then + echo "Remove broken spark-3.4.2-bin-hadoop3" + rm -rf spark-3.4.2-bin-hadoop3 + fi + echo "Unpackage spark-3.4.2-bin-hadoop3" + tar -xf spark-3.4.2-bin-hadoop3.tgz + cp aws-java-sdk-bundle-1.12.48.jar spark-3.4.2-bin-hadoop3/jars/ + cp hadoop-aws-3.3.1.jar spark-3.4.2-bin-hadoop3/jars/ + cp hudi-spark3.4-bundle_2.12-0.14.1.jar spark-3.4.2-bin-hadoop3/jars/ + touch spark-3.4.2-bin-hadoop3/SUCCESS +fi +if [[ ! -f "doris-bin/SUCCESS" ]]; then + echo "Prepare ${DORIS_PACKAGE} environment" + if [[ -d "doris-bin" ]]; then + echo "Remove broken ${DORIS_PACKAGE}" + rm -rf doris-bin + fi + echo "Unpackage ${DORIS_PACKAGE}" + tar xzf "${DORIS_PACKAGE}".tar.gz + mv "${DORIS_PACKAGE}" doris-bin + touch doris-bin/SUCCESS +fi + +cd ../ + +docker compose -f hudi-compose.yml --env-file hudi-compose.env up -d +echo "Create hive table ..." +sleep 5 +docker exec -it spark-hudi-hive sh -c "/opt/hadoop-3.3.1/bin/hadoop fs -chmod 777 /tmp/hive" +docker exec -it spark-hudi-hive sh -c "hive -f /opt/scripts/hive-minio.sql" +echo "Build hive catalog in Doris ..." +sleep 5 +docker exec -it spark-hudi-hive sh -c "mysql -u root -h doris-hudi-env -P 9030 < /opt/scripts/doris-hudi.sql" +echo "======================================================" +echo "Success to launch spark+doris+hudi+minio environments!" +echo "./login-spark.sh to login into spark" +echo "./login-doris.sh to login into doris" +echo "======================================================" diff --git a/samples/datalake/hudi/stop-hudi-compose.sh b/samples/datalake/hudi/stop-hudi-compose.sh new file mode 100755 index 000000000000000..7d4237a0581bc6d --- /dev/null +++ b/samples/datalake/hudi/stop-hudi-compose.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +docker compose -f hudi-compose.yml --env-file hudi-compose.env down