From c7df3e0be6b9cfb739b96f840783a6561d07ab60 Mon Sep 17 00:00:00 2001 From: Pxl Date: Fri, 30 Aug 2024 12:43:52 +0800 Subject: [PATCH] [Bug](runtime-filter) support ip rf and use exception to replace dcheck when PrimitiveType to PColumnType (#39985) ## Proposed changes use exception to replace dcheck when PrimitiveType to PColumnType ```cpp *** SIGABRT unknown detail explain (@0x11d3f) received by PID 73023 (TID 74292 OR 0x7fd758225640) from PID 73023; stack trace: *** 0# doris::signal::(anonymous namespace)::FailureSignalHandler(int, siginfo_t*, void*) at /home/zcp/repo_center/doris_master/doris/be/src/common/signal_handler.h:421 1# 0x00007FDDBE6B9520 in /lib/x86_64-linux-gnu/libc.so.6 2# pthread_kill at ./nptl/pthread_kill.c:89 3# raise at ../sysdeps/posix/raise.c:27 4# abort at ./stdlib/abort.c:81 5# 0x000056123F81A94D in /root/output/be/lib/doris_be 6# 0x000056123F80CF8A in /root/output/be/lib/doris_be 7# google::LogMessage::SendToLog() in /root/output/be/lib/doris_be 8# google::LogMessage::Flush() in /root/output/be/lib/doris_be 9# google::LogMessageFatal::~LogMessageFatal() in /root/output/be/lib/doris_be 10# doris::to_proto(doris::PrimitiveType) at /home/zcp/repo_center/doris_master/doris/be/src/exprs/runtime_filter.cpp:114 11# doris::IRuntimeFilter::push_to_remote(doris::TNetworkAddress const*) at /home/zcp/repo_center/doris_master/doris/be/src/exprs/runtime_filter.cpp:1143 12# doris::IRuntimeFilter::publish(bool)::$_0::operator()(doris::IRuntimeFilter*) const at /home/zcp/repo_center/doris_master/doris/be/src/exprs/runtime_filter.cpp:959 13# doris::IRuntimeFilter::publish(bool)::$_2::operator()() const at /home/zcp/repo_center/doris_master/doris/be/src/exprs/runtime_filter.cpp:983 14# doris::IRuntimeFilter::publish(bool) at /home/zcp/repo_center/doris_master/doris/be/src/exprs/runtime_filter.cpp:997 ``` --- be/src/exprs/runtime_filter.cpp | 108 ++++++++++++++---- be/src/olap/predicate_creator.h | 4 +- be/src/runtime/large_int_value.h | 5 + be/src/runtime/primitive_type.h | 4 +- be/src/vec/core/types.h | 6 +- be/src/vec/data_types/data_type_ipv6.cpp | 2 +- be/src/vec/data_types/data_type_ipv6.h | 2 +- be/src/vec/exprs/vexpr.cpp | 13 ++- be/src/vec/exprs/vexpr.h | 16 +++ be/src/vec/olap/olap_data_convertor.cpp | 4 +- be/src/vec/runtime/ipv4_value.h | 18 +-- be/src/vec/runtime/ipv6_value.h | 18 +-- .../serde/data_type_serde_mysql_test.cpp | 4 +- be/test/vec/runtime/ip_value_test.cpp | 16 +-- gensrc/proto/internal_service.proto | 6 +- .../data/datatype_p0/ip/ip_rf/test_ip_rf.out | 31 +++++ .../datatype_p0/ip/ip_rf/test_ip_rf.groovy | 73 ++++++++++++ 17 files changed, 264 insertions(+), 66 deletions(-) create mode 100644 regression-test/data/datatype_p0/ip/ip_rf/test_ip_rf.out create mode 100644 regression-test/suites/datatype_p0/ip/ip_rf/test_ip_rf.groovy diff --git a/be/src/exprs/runtime_filter.cpp b/be/src/exprs/runtime_filter.cpp index f77ceee82b0887..56c414be5931bc 100644 --- a/be/src/exprs/runtime_filter.cpp +++ b/be/src/exprs/runtime_filter.cpp @@ -110,11 +110,14 @@ PColumnType to_proto(PrimitiveType type) { return PColumnType::COLUMN_TYPE_VARCHAR; case TYPE_STRING: return PColumnType::COLUMN_TYPE_STRING; + case TYPE_IPV4: + return PColumnType::COLUMN_TYPE_IPV4; + case TYPE_IPV6: + return PColumnType::COLUMN_TYPE_IPV6; default: - DCHECK(false) << "Invalid type."; + throw Exception(ErrorCode::INTERNAL_ERROR, + "runtime filter meet invalid PrimitiveType type {}", int(type)); } - DCHECK(false); - return PColumnType::COLUMN_TYPE_INT; } // PColumnType->PrimitiveType @@ -160,10 +163,14 @@ PrimitiveType to_primitive_type(PColumnType type) { return TYPE_CHAR; case PColumnType::COLUMN_TYPE_STRING: return TYPE_STRING; + case PColumnType::COLUMN_TYPE_IPV4: + return TYPE_IPV4; + case PColumnType::COLUMN_TYPE_IPV6: + return TYPE_IPV6; default: - DCHECK(false); + throw Exception(ErrorCode::INTERNAL_ERROR, + "runtime filter meet invalid PColumnType type {}", int(type)); } - return TYPE_INT; } // PFilterType -> RuntimeFilterType @@ -554,14 +561,13 @@ class RuntimePredicateWrapper { } Status assign(const PInFilter* in_filter, bool contain_null) { - PrimitiveType type = to_primitive_type(in_filter->column_type()); - _context->hybrid_set.reset(create_set(type)); + _context->hybrid_set.reset(create_set(_column_return_type)); if (contain_null) { _context->hybrid_set->set_null_aware(true); _context->hybrid_set->insert((const void*)nullptr); } - switch (type) { + switch (_column_return_type) { case TYPE_BOOLEAN: { batch_assign(in_filter, [](std::shared_ptr& set, PColumnValue& column) { bool bool_val = column.boolval(); @@ -701,9 +707,27 @@ class RuntimePredicateWrapper { }); break; } + case TYPE_IPV4: { + batch_assign(in_filter, [](std::shared_ptr& set, PColumnValue& column) { + int32_t tmp = column.intval(); + set->insert(&tmp); + }); + break; + } + case TYPE_IPV6: { + batch_assign(in_filter, [](std::shared_ptr& set, PColumnValue& column) { + auto string_val = column.stringval(); + StringParser::ParseResult result; + auto int128_val = StringParser::string_to_int( + string_val.c_str(), string_val.length(), &result); + DCHECK(result == StringParser::PARSE_SUCCESS); + set->insert(&int128_val); + }); + break; + } default: { return Status::InternalError("not support assign to in filter, type: " + - type_to_string(type)); + type_to_string(_column_return_type)); } } return Status::OK(); @@ -726,15 +750,14 @@ class RuntimePredicateWrapper { // used by shuffle runtime filter // assign this filter by protobuf Status assign(const PMinMaxFilter* minmax_filter, bool contain_null) { - PrimitiveType type = to_primitive_type(minmax_filter->column_type()); - _context->minmax_func.reset(create_minmax_filter(type)); + _context->minmax_func.reset(create_minmax_filter(_column_return_type)); if (contain_null) { _context->minmax_func->set_null_aware(true); _context->minmax_func->set_contain_null(); } - switch (type) { + switch (_column_return_type) { case TYPE_BOOLEAN: { bool min_val = minmax_filter->min_val().boolval(); bool max_val = minmax_filter->max_val().boolval(); @@ -850,6 +873,23 @@ class RuntimePredicateWrapper { auto max_val_ref = minmax_filter->max_val().stringval(); return _context->minmax_func->assign(&min_val_ref, &max_val_ref); } + case TYPE_IPV4: { + int tmp_min = minmax_filter->min_val().intval(); + int tmp_max = minmax_filter->max_val().intval(); + return _context->minmax_func->assign(&tmp_min, &tmp_max); + } + case TYPE_IPV6: { + auto min_string_val = minmax_filter->min_val().stringval(); + auto max_string_val = minmax_filter->max_val().stringval(); + StringParser::ParseResult result; + auto min_val = StringParser::string_to_int(min_string_val.c_str(), + min_string_val.length(), &result); + DCHECK(result == StringParser::PARSE_SUCCESS); + auto max_val = StringParser::string_to_int(max_string_val.c_str(), + max_string_val.length(), &result); + DCHECK(result == StringParser::PARSE_SUCCESS); + return _context->minmax_func->assign(&min_val, &max_val); + } default: break; } @@ -1140,7 +1180,7 @@ Status IRuntimeFilter::push_to_remote(const TNetworkAddress* addr) { merge_filter_request->set_filter_id(_filter_id); merge_filter_request->set_is_pipeline(true); auto column_type = _wrapper->column_type(); - merge_filter_request->set_column_type(to_proto(column_type)); + RETURN_IF_CATCH_EXCEPTION(merge_filter_request->set_column_type(to_proto(column_type))); merge_filter_callback->cntl_->set_timeout_ms(wait_time_ms()); if (get_ignored()) { @@ -1411,13 +1451,10 @@ template Status IRuntimeFilter::_create_wrapper(const T* param, std::unique_ptr* wrapper) { int filter_type = param->request->filter_type(); - PrimitiveType column_type = PrimitiveType::INVALID_TYPE; - if (param->request->has_in_filter()) { - column_type = to_primitive_type(param->request->in_filter().column_type()); - } - if (param->request->has_column_type()) { - column_type = to_primitive_type(param->request->column_type()); + if (!param->request->has_column_type()) { + return Status::InternalError("unknown filter column type"); } + PrimitiveType column_type = to_primitive_type(param->request->column_type()); *wrapper = std::make_unique(column_type, get_type(filter_type), param->request->filter_id()); @@ -1637,9 +1674,21 @@ void IRuntimeFilter::to_protobuf(PInFilter* filter) { }); return; } + case TYPE_IPV4: { + batch_copy(filter, it, [](PColumnValue* column, const IPv4* value) { + column->set_intval(*reinterpret_cast(value)); + }); + return; + } + case TYPE_IPV6: { + batch_copy(filter, it, [](PColumnValue* column, const IPv6* value) { + column->set_stringval(LargeIntValue::to_string(*value)); + }); + return; + } default: { - DCHECK(false) << "unknown type"; - break; + throw Exception(ErrorCode::INTERNAL_ERROR, + "runtime filter meet invalid PrimitiveType type {}", int(column_type)); } } } @@ -1753,9 +1802,22 @@ void IRuntimeFilter::to_protobuf(PMinMaxFilter* filter) { filter->mutable_max_val()->set_stringval(*max_string_value); break; } + case TYPE_IPV4: { + filter->mutable_min_val()->set_intval(*reinterpret_cast(min_data)); + filter->mutable_max_val()->set_intval(*reinterpret_cast(max_data)); + return; + } + case TYPE_IPV6: { + filter->mutable_min_val()->set_stringval( + LargeIntValue::to_string(*reinterpret_cast(min_data))); + filter->mutable_max_val()->set_stringval( + LargeIntValue::to_string(*reinterpret_cast(max_data))); + return; + } default: { - DCHECK(false) << "unknown type"; - break; + throw Exception(ErrorCode::INTERNAL_ERROR, + "runtime filter meet invalid PrimitiveType type {}", + int(_wrapper->column_type())); } } } diff --git a/be/src/olap/predicate_creator.h b/be/src/olap/predicate_creator.h index c47a1694f17a1d..1020e985248d72 100644 --- a/be/src/olap/predicate_creator.h +++ b/be/src/olap/predicate_creator.h @@ -242,7 +242,7 @@ std::unique_ptr> get_creator(const FieldType& ty case FieldType::OLAP_FIELD_TYPE_IPV4: { return std::make_unique>( [](const std::string& condition) { - vectorized::IPv4 value; + IPv4 value; bool res = IPv4Value::from_string(value, condition); DCHECK(res); return value; @@ -251,7 +251,7 @@ std::unique_ptr> get_creator(const FieldType& ty case FieldType::OLAP_FIELD_TYPE_IPV6: { return std::make_unique>( [](const std::string& condition) { - vectorized::IPv6 value; + IPv6 value; bool res = IPv6Value::from_string(value, condition); DCHECK(res); return value; diff --git a/be/src/runtime/large_int_value.h b/be/src/runtime/large_int_value.h index 7bc89ea765c150..fbe37ac1f409ab 100644 --- a/be/src/runtime/large_int_value.h +++ b/be/src/runtime/large_int_value.h @@ -24,6 +24,8 @@ #include #include +#include "olap/olap_common.h" + namespace doris { inline const __int128 MAX_INT128 = ~((__int128)0x01 << 127); @@ -36,6 +38,9 @@ class LargeIntValue { } static std::string to_string(__int128 value) { return fmt::format(FMT_COMPILE("{}"), value); } + static std::string to_string(__uint128_t value) { + return fmt::format(FMT_COMPILE("{}"), value); + } }; std::ostream& operator<<(std::ostream& os, __int128 const& value); diff --git a/be/src/runtime/primitive_type.h b/be/src/runtime/primitive_type.h index 59c0d91e432391..fb7c6e317f10a1 100644 --- a/be/src/runtime/primitive_type.h +++ b/be/src/runtime/primitive_type.h @@ -243,13 +243,13 @@ struct PrimitiveTypeTraits { }; template <> struct PrimitiveTypeTraits { - using CppType = vectorized::IPv4; + using CppType = IPv4; using StorageFieldType = CppType; using ColumnType = vectorized::ColumnIPv4; }; template <> struct PrimitiveTypeTraits { - using CppType = vectorized::IPv6; + using CppType = IPv6; using StorageFieldType = CppType; using ColumnType = vectorized::ColumnIPv6; }; diff --git a/be/src/vec/core/types.h b/be/src/vec/core/types.h index 73025b166e5308..f5943fa6e1d31f 100644 --- a/be/src/vec/core/types.h +++ b/be/src/vec/core/types.h @@ -42,6 +42,9 @@ struct decimal12_t; struct uint24_t; struct StringRef; +using IPv4 = uint32_t; +using IPv6 = uint128_t; + namespace vectorized { /// Data types for representing elementary values from a database in RAM. @@ -296,9 +299,6 @@ struct TypeId { /// Not a data type in database, defined just for convenience. using Strings = std::vector; -using IPv4 = uint32_t; -using IPv6 = uint128_t; - template <> inline constexpr bool IsNumber = true; template <> diff --git a/be/src/vec/data_types/data_type_ipv6.cpp b/be/src/vec/data_types/data_type_ipv6.cpp index 4a81727870e4cf..2d524a71728a0b 100755 --- a/be/src/vec/data_types/data_type_ipv6.cpp +++ b/be/src/vec/data_types/data_type_ipv6.cpp @@ -50,7 +50,7 @@ std::string DataTypeIPv6::to_string(const IColumn& column, size_t row_num) const return value.to_string(); } -std::string DataTypeIPv6::to_string(const IPv6& ipv6_val) const { +std::string DataTypeIPv6::to_string(const IPv6& ipv6_val) { auto value = IPv6Value(ipv6_val); return value.to_string(); } diff --git a/be/src/vec/data_types/data_type_ipv6.h b/be/src/vec/data_types/data_type_ipv6.h index 89a42aca862c11..9196e9c936a19f 100755 --- a/be/src/vec/data_types/data_type_ipv6.h +++ b/be/src/vec/data_types/data_type_ipv6.h @@ -63,7 +63,7 @@ class DataTypeIPv6 final : public DataTypeNumberBase { void push_number(ColumnString::Chars& chars, const IPv6& num) const; std::string to_string(const IColumn& column, size_t row_num) const override; void to_string(const IColumn& column, size_t row_num, BufferWritable& ostr) const override; - std::string to_string(const IPv6& value) const; + static std::string to_string(const IPv6& value); Status from_string(ReadBuffer& rb, IColumn* column) const override; Field get_field(const TExprNode& node) const override { diff --git a/be/src/vec/exprs/vexpr.cpp b/be/src/vec/exprs/vexpr.cpp index f16f6dc777d02f..13f67bc2eaf9aa 100644 --- a/be/src/vec/exprs/vexpr.cpp +++ b/be/src/vec/exprs/vexpr.cpp @@ -31,6 +31,7 @@ #include "common/config.h" #include "common/exception.h" #include "common/status.h" +#include "runtime/define_primitive_type.h" #include "vec/columns/column_vector.h" #include "vec/columns/columns_number.h" #include "vec/data_types/data_type_array.h" @@ -147,9 +148,17 @@ TExprNode create_texpr_node_from(const void* data, const PrimitiveType& type, in THROW_IF_ERROR(create_texpr_literal_node(data, &node)); break; } + case TYPE_IPV4: { + THROW_IF_ERROR(create_texpr_literal_node(data, &node)); + break; + } + case TYPE_IPV6: { + THROW_IF_ERROR(create_texpr_literal_node(data, &node)); + break; + } default: - DCHECK(false); - throw std::invalid_argument("Invalid type!"); + throw Exception(ErrorCode::INTERNAL_ERROR, "runtime filter meet invalid type {}", + int(type)); } return node; } diff --git a/be/src/vec/exprs/vexpr.h b/be/src/vec/exprs/vexpr.h index 4b4b387d93cd25..950f12bef47728 100644 --- a/be/src/vec/exprs/vexpr.h +++ b/be/src/vec/exprs/vexpr.h @@ -39,8 +39,10 @@ #include "vec/columns/column.h" #include "vec/core/block.h" #include "vec/core/column_with_type_and_name.h" +#include "vec/core/types.h" #include "vec/core/wide_integer.h" #include "vec/data_types/data_type.h" +#include "vec/data_types/data_type_ipv6.h" #include "vec/exprs/vexpr_fwd.h" #include "vec/functions/function.h" @@ -464,6 +466,20 @@ Status create_texpr_literal_node(const void* data, TExprNode* node, int precisio string_literal.__set_value(*origin_value); (*node).__set_string_literal(string_literal); (*node).__set_type(create_type_desc(PrimitiveType::TYPE_STRING)); + } else if constexpr (T == TYPE_IPV4) { + const auto* origin_value = reinterpret_cast(data); + (*node).__set_node_type(TExprNodeType::IPV4_LITERAL); + TIPv4Literal literal; + literal.__set_value(*origin_value); + (*node).__set_ipv4_literal(literal); + (*node).__set_type(create_type_desc(PrimitiveType::TYPE_IPV4)); + } else if constexpr (T == TYPE_IPV6) { + const auto* origin_value = reinterpret_cast(data); + (*node).__set_node_type(TExprNodeType::IPV6_LITERAL); + TIPv6Literal literal; + literal.__set_value(vectorized::DataTypeIPv6::to_string(*origin_value)); + (*node).__set_ipv6_literal(literal); + (*node).__set_type(create_type_desc(PrimitiveType::TYPE_IPV6)); } else { return Status::InvalidArgument("Invalid argument type!"); } diff --git a/be/src/vec/olap/olap_data_convertor.cpp b/be/src/vec/olap/olap_data_convertor.cpp index 5c8e09766e635f..8dcdd977d9a57f 100644 --- a/be/src/vec/olap/olap_data_convertor.cpp +++ b/be/src/vec/olap/olap_data_convertor.cpp @@ -184,10 +184,10 @@ OlapBlockDataConvertor::create_olap_column_data_convertor(const TabletColumn& co return std::make_unique>(); } case FieldType::OLAP_FIELD_TYPE_IPV4: { - return std::make_unique>(); + return std::make_unique>(); } case FieldType::OLAP_FIELD_TYPE_IPV6: { - return std::make_unique>(); + return std::make_unique>(); } case FieldType::OLAP_FIELD_TYPE_FLOAT: { return std::make_unique>(); diff --git a/be/src/vec/runtime/ipv4_value.h b/be/src/vec/runtime/ipv4_value.h index ffce9e2bb67641..76ae288e5be407 100644 --- a/be/src/vec/runtime/ipv4_value.h +++ b/be/src/vec/runtime/ipv4_value.h @@ -32,19 +32,19 @@ class IPv4Value { public: IPv4Value() = default; - explicit IPv4Value(vectorized::IPv4 ipv4) { _value = ipv4; } + explicit IPv4Value(IPv4 ipv4) { _value = ipv4; } - const vectorized::IPv4& value() const { return _value; } + const IPv4& value() const { return _value; } - vectorized::IPv4& value() { return _value; } + IPv4& value() { return _value; } - void set_value(vectorized::IPv4 ipv4) { _value = ipv4; } + void set_value(IPv4 ipv4) { _value = ipv4; } bool from_string(const std::string& ipv4_str) { return from_string(_value, ipv4_str); } std::string to_string() const { return to_string(_value); } - static bool from_string(vectorized::IPv4& value, const char* ipv4_str, size_t len) { + static bool from_string(IPv4& value, const char* ipv4_str, size_t len) { if (len == 0) { return false; } @@ -61,15 +61,15 @@ class IPv4Value { reinterpret_cast(&parse_value))) { return false; } - value = static_cast(parse_value); + value = static_cast(parse_value); return true; } - static bool from_string(vectorized::IPv4& value, const std::string& ipv4_str) { + static bool from_string(IPv4& value, const std::string& ipv4_str) { return from_string(value, ipv4_str.c_str(), ipv4_str.size()); } - static std::string to_string(vectorized::IPv4 value) { + static std::string to_string(IPv4 value) { char buf[IPV4_MAX_TEXT_LENGTH + 1]; char* start = buf; char* end = buf; @@ -97,7 +97,7 @@ class IPv4Value { } private: - vectorized::IPv4 _value; + IPv4 _value; }; } // namespace doris diff --git a/be/src/vec/runtime/ipv6_value.h b/be/src/vec/runtime/ipv6_value.h index 953b71baf84ef2..5a26af25243a0f 100644 --- a/be/src/vec/runtime/ipv6_value.h +++ b/be/src/vec/runtime/ipv6_value.h @@ -32,17 +32,17 @@ class IPv6Value { public: IPv6Value() { _value = 0; } - explicit IPv6Value(vectorized::IPv6 ipv6) { _value = ipv6; } + explicit IPv6Value(IPv6 ipv6) { _value = ipv6; } - const vectorized::IPv6& value() const { return _value; } + const IPv6& value() const { return _value; } - vectorized::IPv6& value() { return _value; } + IPv6& value() { return _value; } - void set_value(vectorized::IPv6 ipv6) { _value = ipv6; } + void set_value(IPv6 ipv6) { _value = ipv6; } bool from_string(const std::string& ipv6_str) { return from_string(_value, ipv6_str); } - static bool from_string(vectorized::IPv6& value, const char* ipv6_str, size_t len) { + static bool from_string(IPv6& value, const char* ipv6_str, size_t len) { if (len == 0) { return false; } @@ -59,13 +59,13 @@ class IPv6Value { reinterpret_cast(&value)); } - static bool from_string(vectorized::IPv6& value, const std::string& ipv6_str) { + static bool from_string(IPv6& value, const std::string& ipv6_str) { return from_string(value, ipv6_str.c_str(), ipv6_str.size()); } std::string to_string() const { return to_string(_value); } - static std::string to_string(vectorized::IPv6 value) { + static std::string to_string(IPv6 value) { char buf[IPV6_MAX_TEXT_LENGTH + 1]; char* start = buf; char* end = buf; @@ -80,7 +80,7 @@ class IPv6Value { if (len == 0 || len > IPV6_MAX_TEXT_LENGTH) { return false; } - vectorized::IPv6 value; + IPv6 value; size_t begin = 0; size_t end = len - 1; while (begin < len && std::isspace(ipv6_str[begin])) { @@ -94,7 +94,7 @@ class IPv6Value { } private: - vectorized::IPv6 _value; + IPv6 _value; }; } // namespace doris diff --git a/be/test/vec/data_types/serde/data_type_serde_mysql_test.cpp b/be/test/vec/data_types/serde/data_type_serde_mysql_test.cpp index 97e78f05c549a8..f05919e4a8f477 100644 --- a/be/test/vec/data_types/serde/data_type_serde_mysql_test.cpp +++ b/be/test/vec/data_types/serde/data_type_serde_mysql_test.cpp @@ -250,7 +250,7 @@ void serialize_and_deserialize_mysql_test() { case TYPE_IPV4: tslot.__set_slotType(type_desc.to_thrift()); { - auto column_vector_ipv4 = vectorized::ColumnVector::create(); + auto column_vector_ipv4 = vectorized::ColumnVector::create(); auto& ipv4_data = column_vector_ipv4->get_data(); for (int i = 0; i < row_num; ++i) { IPv4Value ipv4_value; @@ -267,7 +267,7 @@ void serialize_and_deserialize_mysql_test() { case TYPE_IPV6: tslot.__set_slotType(type_desc.to_thrift()); { - auto column_vector_ipv6 = vectorized::ColumnVector::create(); + auto column_vector_ipv6 = vectorized::ColumnVector::create(); auto& ipv6_data = column_vector_ipv6->get_data(); for (int i = 0; i < row_num; ++i) { IPv6Value ipv6_value; diff --git a/be/test/vec/runtime/ip_value_test.cpp b/be/test/vec/runtime/ip_value_test.cpp index cbeebcf24b67d9..f3b3b9ac9cd39f 100644 --- a/be/test/vec/runtime/ip_value_test.cpp +++ b/be/test/vec/runtime/ip_value_test.cpp @@ -49,8 +49,8 @@ static void print_bytes(T num) { TEST(IPValueTest, IPv4ValueTest) { const std::string ipv4_str1 = "192.168.103.254"; const std::string ipv4_str2 = "193.168.103.255"; - vectorized::IPv4 ipv4_val1; - vectorized::IPv4 ipv4_val2; + IPv4 ipv4_val1; + IPv4 ipv4_val2; ASSERT_TRUE(IPv4Value::from_string(ipv4_val1, ipv4_str1.c_str(), ipv4_str1.size())); ASSERT_TRUE(IPv4Value::from_string(ipv4_val2, ipv4_str2.c_str(), ipv4_str2.size())); ASSERT_TRUE(ipv4_val1 < ipv4_val2); @@ -65,8 +65,8 @@ TEST(IPValueTest, IPv4ValueTest) { TEST(IPValueTest, IPv6ValueTest) { const std::string ipv6_str1 = "2001:418:0:5000::c2d"; const std::string ipv6_str2 = "2001:428::205:171:200:230"; - vectorized::IPv6 ipv6_val1; - vectorized::IPv6 ipv6_val2; + IPv6 ipv6_val1; + IPv6 ipv6_val2; ASSERT_TRUE(IPv6Value::from_string(ipv6_val1, ipv6_str1.c_str(), ipv6_str1.size())); ASSERT_TRUE(IPv6Value::from_string(ipv6_val2, ipv6_str2.c_str(), ipv6_str2.size())); ASSERT_TRUE(ipv6_val1 < ipv6_val2); @@ -91,12 +91,12 @@ static void apply_cidr_mask(const char* __restrict src, char* __restrict dst_low TEST(IPValueTest, IPv6CIDRTest) { const std::string ipv6_str1 = "2001:0db8:0000:85a3:0000:0000:ac1f:8001"; const std::string ipv6_str2 = "2001:0db8:0000:85a3:ffff:ffff:ffff:ffff"; - vectorized::IPv6 ipv6_val1; // little-endian - vectorized::IPv6 ipv6_val2; // little-endian + IPv6 ipv6_val1; // little-endian + IPv6 ipv6_val2; // little-endian ASSERT_TRUE(IPv6Value::from_string(ipv6_val1, ipv6_str1.c_str(), ipv6_str1.size())); ASSERT_TRUE(IPv6Value::from_string(ipv6_val2, ipv6_str2.c_str(), ipv6_str2.size())); - vectorized::IPv6 min_range1, max_range1; - vectorized::IPv6 min_range2, max_range2; + IPv6 min_range1, max_range1; + IPv6 min_range2, max_range2; apply_cidr_mask(reinterpret_cast(&ipv6_val1), reinterpret_cast(&min_range1), reinterpret_cast(&max_range1), 0); apply_cidr_mask(reinterpret_cast(&ipv6_val2), reinterpret_cast(&min_range2), diff --git a/gensrc/proto/internal_service.proto b/gensrc/proto/internal_service.proto index e188001e49103f..4ac4fd24f3b7e5 100644 --- a/gensrc/proto/internal_service.proto +++ b/gensrc/proto/internal_service.proto @@ -517,16 +517,18 @@ enum PColumnType { COLUMN_TYPE_DECIMAL64 = 18; COLUMN_TYPE_DECIMAL128I = 19; COLUMN_TYPE_DECIMAL256 = 20; + COLUMN_TYPE_IPV4 = 21; + COLUMN_TYPE_IPV6 = 22; } message PMinMaxFilter { - required PColumnType column_type = 1; + required PColumnType column_type = 1; // Deprecated required PColumnValue min_val = 2; required PColumnValue max_val = 3; }; message PInFilter { - required PColumnType column_type = 1; + required PColumnType column_type = 1; // Deprecated repeated PColumnValue values = 2; optional string ignored_msg = 3; } diff --git a/regression-test/data/datatype_p0/ip/ip_rf/test_ip_rf.out b/regression-test/data/datatype_p0/ip/ip_rf/test_ip_rf.out new file mode 100644 index 00000000000000..7e8a59791f636e --- /dev/null +++ b/regression-test/data/datatype_p0/ip/ip_rf/test_ip_rf.out @@ -0,0 +1,31 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !sql -- +2 + +-- !sql -- +2 + +-- !sql -- +2 + +-- !sql -- +2 + +-- !sql -- +2 + +-- !sql -- +2 + +-- !sql -- +2 + +-- !sql -- +2 + +-- !sql -- +2 + +-- !sql -- +2 + diff --git a/regression-test/suites/datatype_p0/ip/ip_rf/test_ip_rf.groovy b/regression-test/suites/datatype_p0/ip/ip_rf/test_ip_rf.groovy new file mode 100644 index 00000000000000..c7f6c30481eb0b --- /dev/null +++ b/regression-test/suites/datatype_p0/ip/ip_rf/test_ip_rf.groovy @@ -0,0 +1,73 @@ + +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +suite("test_ip_rf") { + sql """ DROP TABLE IF EXISTS ip_test """ + sql """ DROP TABLE IF EXISTS ip_test2 """ + sql """ + CREATE TABLE ip_test ( + `id` int, + `ip_v4` ipv4, + `ip_v6` ipv6 + ) ENGINE=OLAP + DISTRIBUTED BY HASH(`id`) BUCKETS 4 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1" + ); + """ + + sql """ + CREATE TABLE ip_test2 ( + `id` int, + `ip_v4` ipv4, + `ip_v6` ipv6 + ) ENGINE=OLAP + DISTRIBUTED BY HASH(`id`) BUCKETS 4 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1" + ); + """ + + sql """ + insert into ip_test values(1, '0.0.0.0','2001:16a0:2:200a::2'),(2, '127.0.0.1','2a02:e980:83:5b09:ecb8:c669:b336:650e'); + """ + + sql """ + insert into ip_test2 values(1, '0.0.0.0','2001:16a0:2:200a::2'),(2, '127.0.0.1','2a02:e980:83:5b09:ecb8:c669:b336:650e'),(3, '59.50.185.152','2001:4888:1f:e891:161:26::'),(4, '255.255.255.255','2001:4888:1f:e891:161:26::'); + + """ + + sql "ANALYZE TABLE ip_test WITH sync;" + sql "ANALYZE TABLE ip_test2 WITH sync;" + + sql "set runtime_filter_type=0;" + qt_sql "select count(*) from ip_test a, ip_test2 b where a.ip_v4=b.ip_v4;" + qt_sql "select count(*) from ip_test a, ip_test2 b where a.ip_v6=b.ip_v6;" + sql "set runtime_filter_type=1;" + qt_sql "select count(*) from ip_test a, ip_test2 b where a.ip_v4=b.ip_v4;" + qt_sql "select count(*) from ip_test a, ip_test2 b where a.ip_v6=b.ip_v6;" + sql "set runtime_filter_type=2;" + qt_sql "select count(*) from ip_test a, ip_test2 b where a.ip_v4=b.ip_v4;" + qt_sql "select count(*) from ip_test a, ip_test2 b where a.ip_v6=b.ip_v6;" + sql "set runtime_filter_type=4;" + qt_sql "select count(*) from ip_test a, ip_test2 b where a.ip_v4=b.ip_v4;" + qt_sql "select count(*) from ip_test a, ip_test2 b where a.ip_v6=b.ip_v6;" + sql "set runtime_filter_type=8;" + qt_sql "select count(*) from ip_test a, ip_test2 b where a.ip_v4=b.ip_v4;" + qt_sql "select count(*) from ip_test a, ip_test2 b where a.ip_v6=b.ip_v6;" + +}