[Bug](runtime-filter) send ignored rf when hash join build closed early #41667
Conversation
|
run buildall |
![github-actions[bot]](https://avatars.githubusercontent.com/in/15368?s=60&v=4){kind=small}
| @@ -472,10 +472,10 @@ class RuntimePredicateWrapper { | |||
| const TExpr& probe_expr); | |||
|
|
|||
| Status merge(const RuntimePredicateWrapper* wrapper) { | |||
There was a problem hiding this comment.
warning: function 'merge' has cognitive complexity of 74 (threshold 50) [readability-function-cognitive-complexity]
Status merge(const RuntimePredicateWrapper* wrapper) {
^Additional context
be/src/exprs/runtime_filter.cpp:474: +1, including nesting penalty of 0, nesting level increased to 1
if (wrapper->is_ignored()) {
^be/src/exprs/runtime_filter.cpp:480: +1
_filter_type == RuntimeFilterType::IN_OR_BLOOM_FILTER &&
^be/src/exprs/runtime_filter.cpp:485: +1
bool can_not_merge_other = _filter_type != RuntimeFilterType::IN_OR_BLOOM_FILTER &&
^be/src/exprs/runtime_filter.cpp:488: +1
CHECK(!can_not_merge_in_or_bloom && !can_not_merge_other)
^be/src/exprs/runtime_filter.cpp:493: +1, including nesting penalty of 0, nesting level increased to 1
switch (_filter_type) {
^be/src/exprs/runtime_filter.cpp:497: +2, including nesting penalty of 1, nesting level increased to 2
if (_max_in_num >= 0 && _context->hybrid_set->size() >= _max_in_num) {
^be/src/exprs/runtime_filter.cpp:507: +2, including nesting penalty of 1, nesting level increased to 2
RETURN_IF_ERROR(_context->minmax_func->merge(wrapper->_context->minmax_func.get()));
^be/src/common/status.h:629: expanded from macro 'RETURN_IF_ERROR'
do { \
^be/src/exprs/runtime_filter.cpp:507: +3, including nesting penalty of 2, nesting level increased to 3
RETURN_IF_ERROR(_context->minmax_func->merge(wrapper->_context->minmax_func.get()));
^be/src/common/status.h:631: expanded from macro 'RETURN_IF_ERROR'
if (UNLIKELY(!_status_.ok())) { \
^be/src/exprs/runtime_filter.cpp:511: +2, including nesting penalty of 1, nesting level increased to 2
RETURN_IF_ERROR(
^be/src/common/status.h:629: expanded from macro 'RETURN_IF_ERROR'
do { \
^be/src/exprs/runtime_filter.cpp:511: +3, including nesting penalty of 2, nesting level increased to 3
RETURN_IF_ERROR(
^be/src/common/status.h:631: expanded from macro 'RETURN_IF_ERROR'
if (UNLIKELY(!_status_.ok())) { \
^be/src/exprs/runtime_filter.cpp:519: +2, including nesting penalty of 1, nesting level increased to 2
if (other_filter_type == RuntimeFilterType::IN_OR_BLOOM_FILTER) {
^be/src/exprs/runtime_filter.cpp:523: +2, including nesting penalty of 1, nesting level increased to 2
if (real_filter_type == RuntimeFilterType::IN_FILTER) {
^be/src/exprs/runtime_filter.cpp:527: +3, including nesting penalty of 2, nesting level increased to 3
if (other_filter_type == RuntimeFilterType::IN_FILTER) {
^be/src/exprs/runtime_filter.cpp:529: +4, including nesting penalty of 3, nesting level increased to 4
if (_max_in_num >= 0 && _context->hybrid_set->size() >= _max_in_num) {
^be/src/exprs/runtime_filter.cpp:531: +5, including nesting penalty of 4, nesting level increased to 5
RETURN_IF_ERROR(_context->bloom_filter_func->init_with_fixed_length());
^be/src/common/status.h:629: expanded from macro 'RETURN_IF_ERROR'
do { \
^be/src/exprs/runtime_filter.cpp:531: +6, including nesting penalty of 5, nesting level increased to 6
RETURN_IF_ERROR(_context->bloom_filter_func->init_with_fixed_length());
^be/src/common/status.h:631: expanded from macro 'RETURN_IF_ERROR'
if (UNLIKELY(!_status_.ok())) { \
^be/src/exprs/runtime_filter.cpp:532: +5, including nesting penalty of 4, nesting level increased to 5
RETURN_IF_ERROR(change_to_bloom_filter());
^be/src/common/status.h:629: expanded from macro 'RETURN_IF_ERROR'
do { \
^be/src/exprs/runtime_filter.cpp:532: +6, including nesting penalty of 5, nesting level increased to 6
RETURN_IF_ERROR(change_to_bloom_filter());
^be/src/common/status.h:631: expanded from macro 'RETURN_IF_ERROR'
if (UNLIKELY(!_status_.ok())) { \
^be/src/exprs/runtime_filter.cpp:534: +1, nesting level increased to 3
} else {
^be/src/exprs/runtime_filter.cpp:537: +4, including nesting penalty of 3, nesting level increased to 4
RETURN_IF_ERROR(change_to_bloom_filter());
^be/src/common/status.h:629: expanded from macro 'RETURN_IF_ERROR'
do { \
^be/src/exprs/runtime_filter.cpp:537: +5, including nesting penalty of 4, nesting level increased to 5
RETURN_IF_ERROR(change_to_bloom_filter());
^be/src/common/status.h:631: expanded from macro 'RETURN_IF_ERROR'
if (UNLIKELY(!_status_.ok())) { \
^be/src/exprs/runtime_filter.cpp:539: +1, nesting level increased to 2
} else {
^be/src/exprs/runtime_filter.cpp:540: +3, including nesting penalty of 2, nesting level increased to 3
if (other_filter_type == RuntimeFilterType::IN_FILTER) {
^be/src/exprs/runtime_filter.cpp:543: +1, nesting level increased to 3
} else {
^be/src/exprs/runtime_filter.cpp:545: +4, including nesting penalty of 3, nesting level increased to 4
RETURN_IF_ERROR(_context->bloom_filter_func->merge(
^be/src/common/status.h:629: expanded from macro 'RETURN_IF_ERROR'
do { \
^be/src/exprs/runtime_filter.cpp:545: +5, including nesting penalty of 4, nesting level increased to 5
RETURN_IF_ERROR(_context->bloom_filter_func->merge(
^be/src/common/status.h:631: expanded from macro 'RETURN_IF_ERROR'
if (UNLIKELY(!_status_.ok())) { \
^
|
run buildall |
|
run buildall |
|
run buildall |
| @@ -472,10 +472,10 @@ class RuntimePredicateWrapper { | |||
| const TExpr& probe_expr); | |||
|
|
|||
| Status merge(const RuntimePredicateWrapper* wrapper) { | |||
There was a problem hiding this comment.
warning: function 'merge' has cognitive complexity of 76 (threshold 50) [readability-function-cognitive-complexity]
Status merge(const RuntimePredicateWrapper* wrapper) {
^Additional context
be/src/exprs/runtime_filter.cpp:474: +1, including nesting penalty of 0, nesting level increased to 1
if (wrapper->is_ignored()) {
^be/src/exprs/runtime_filter.cpp:480: +1
_filter_type == RuntimeFilterType::IN_OR_BLOOM_FILTER &&
^be/src/exprs/runtime_filter.cpp:485: +1
bool can_not_merge_other = _filter_type != RuntimeFilterType::IN_OR_BLOOM_FILTER &&
^be/src/exprs/runtime_filter.cpp:488: +1
CHECK(!can_not_merge_in_or_bloom && !can_not_merge_other)
^be/src/exprs/runtime_filter.cpp:493: +1, including nesting penalty of 0, nesting level increased to 1
switch (_filter_type) {
^be/src/exprs/runtime_filter.cpp:496: +2, including nesting penalty of 1, nesting level increased to 2
if (!_context->hybrid_set) {
^be/src/exprs/runtime_filter.cpp:500: +2, including nesting penalty of 1, nesting level increased to 2
if (_max_in_num >= 0 && _context->hybrid_set->size() >= _max_in_num) {
^be/src/exprs/runtime_filter.cpp:510: +2, including nesting penalty of 1, nesting level increased to 2
RETURN_IF_ERROR(_context->minmax_func->merge(wrapper->_context->minmax_func.get()));
^be/src/common/status.h:629: expanded from macro 'RETURN_IF_ERROR'
do { \
^be/src/exprs/runtime_filter.cpp:510: +3, including nesting penalty of 2, nesting level increased to 3
RETURN_IF_ERROR(_context->minmax_func->merge(wrapper->_context->minmax_func.get()));
^be/src/common/status.h:631: expanded from macro 'RETURN_IF_ERROR'
if (UNLIKELY(!_status_.ok())) { \
^be/src/exprs/runtime_filter.cpp:514: +2, including nesting penalty of 1, nesting level increased to 2
RETURN_IF_ERROR(
^be/src/common/status.h:629: expanded from macro 'RETURN_IF_ERROR'
do { \
^be/src/exprs/runtime_filter.cpp:514: +3, including nesting penalty of 2, nesting level increased to 3
RETURN_IF_ERROR(
^be/src/common/status.h:631: expanded from macro 'RETURN_IF_ERROR'
if (UNLIKELY(!_status_.ok())) { \
^be/src/exprs/runtime_filter.cpp:522: +2, including nesting penalty of 1, nesting level increased to 2
if (other_filter_type == RuntimeFilterType::IN_OR_BLOOM_FILTER) {
^be/src/exprs/runtime_filter.cpp:526: +2, including nesting penalty of 1, nesting level increased to 2
if (real_filter_type == RuntimeFilterType::IN_FILTER) {
^be/src/exprs/runtime_filter.cpp:530: +3, including nesting penalty of 2, nesting level increased to 3
if (other_filter_type == RuntimeFilterType::IN_FILTER) {
^be/src/exprs/runtime_filter.cpp:532: +4, including nesting penalty of 3, nesting level increased to 4
if (_max_in_num >= 0 && _context->hybrid_set->size() >= _max_in_num) {
^be/src/exprs/runtime_filter.cpp:534: +5, including nesting penalty of 4, nesting level increased to 5
RETURN_IF_ERROR(_context->bloom_filter_func->init_with_fixed_length());
^be/src/common/status.h:629: expanded from macro 'RETURN_IF_ERROR'
do { \
^be/src/exprs/runtime_filter.cpp:534: +6, including nesting penalty of 5, nesting level increased to 6
RETURN_IF_ERROR(_context->bloom_filter_func->init_with_fixed_length());
^be/src/common/status.h:631: expanded from macro 'RETURN_IF_ERROR'
if (UNLIKELY(!_status_.ok())) { \
^be/src/exprs/runtime_filter.cpp:535: +5, including nesting penalty of 4, nesting level increased to 5
RETURN_IF_ERROR(change_to_bloom_filter());
^be/src/common/status.h:629: expanded from macro 'RETURN_IF_ERROR'
do { \
^be/src/exprs/runtime_filter.cpp:535: +6, including nesting penalty of 5, nesting level increased to 6
RETURN_IF_ERROR(change_to_bloom_filter());
^be/src/common/status.h:631: expanded from macro 'RETURN_IF_ERROR'
if (UNLIKELY(!_status_.ok())) { \
^be/src/exprs/runtime_filter.cpp:537: +1, nesting level increased to 3
} else {
^be/src/exprs/runtime_filter.cpp:540: +4, including nesting penalty of 3, nesting level increased to 4
RETURN_IF_ERROR(change_to_bloom_filter());
^be/src/common/status.h:629: expanded from macro 'RETURN_IF_ERROR'
do { \
^be/src/exprs/runtime_filter.cpp:540: +5, including nesting penalty of 4, nesting level increased to 5
RETURN_IF_ERROR(change_to_bloom_filter());
^be/src/common/status.h:631: expanded from macro 'RETURN_IF_ERROR'
if (UNLIKELY(!_status_.ok())) { \
^be/src/exprs/runtime_filter.cpp:542: +1, nesting level increased to 2
} else {
^be/src/exprs/runtime_filter.cpp:543: +3, including nesting penalty of 2, nesting level increased to 3
if (other_filter_type == RuntimeFilterType::IN_FILTER) {
^be/src/exprs/runtime_filter.cpp:546: +1, nesting level increased to 3
} else {
^be/src/exprs/runtime_filter.cpp:548: +4, including nesting penalty of 3, nesting level increased to 4
RETURN_IF_ERROR(_context->bloom_filter_func->merge(
^be/src/common/status.h:629: expanded from macro 'RETURN_IF_ERROR'
do { \
^be/src/exprs/runtime_filter.cpp:548: +5, including nesting penalty of 4, nesting level increased to 5
RETURN_IF_ERROR(_context->bloom_filter_func->merge(
^be/src/common/status.h:631: expanded from macro 'RETURN_IF_ERROR'
if (UNLIKELY(!_status_.ok())) { \
^
|
run buildall |
|
run buildall |
| @@ -472,10 +472,10 @@ class RuntimePredicateWrapper { | |||
| const TExpr& probe_expr); | |||
|
|
|||
| Status merge(const RuntimePredicateWrapper* wrapper) { | |||
There was a problem hiding this comment.
warning: function 'merge' has cognitive complexity of 76 (threshold 50) [readability-function-cognitive-complexity]
Status merge(const RuntimePredicateWrapper* wrapper) {
^Additional context
be/src/exprs/runtime_filter.cpp:474: +1, including nesting penalty of 0, nesting level increased to 1
if (wrapper->is_ignored()) {
^be/src/exprs/runtime_filter.cpp:480: +1
_filter_type == RuntimeFilterType::IN_OR_BLOOM_FILTER &&
^be/src/exprs/runtime_filter.cpp:485: +1
bool can_not_merge_other = _filter_type != RuntimeFilterType::IN_OR_BLOOM_FILTER &&
^be/src/exprs/runtime_filter.cpp:488: +1
CHECK(!can_not_merge_in_or_bloom && !can_not_merge_other)
^be/src/exprs/runtime_filter.cpp:493: +1, including nesting penalty of 0, nesting level increased to 1
switch (_filter_type) {
^be/src/exprs/runtime_filter.cpp:495: +2, including nesting penalty of 1, nesting level increased to 2
if (!_context->hybrid_set) {
^be/src/exprs/runtime_filter.cpp:500: +2, including nesting penalty of 1, nesting level increased to 2
if (_max_in_num >= 0 && _context->hybrid_set->size() >= _max_in_num) {
^be/src/exprs/runtime_filter.cpp:510: +2, including nesting penalty of 1, nesting level increased to 2
RETURN_IF_ERROR(_context->minmax_func->merge(wrapper->_context->minmax_func.get()));
^be/src/common/status.h:629: expanded from macro 'RETURN_IF_ERROR'
do { \
^be/src/exprs/runtime_filter.cpp:510: +3, including nesting penalty of 2, nesting level increased to 3
RETURN_IF_ERROR(_context->minmax_func->merge(wrapper->_context->minmax_func.get()));
^be/src/common/status.h:631: expanded from macro 'RETURN_IF_ERROR'
if (UNLIKELY(!_status_.ok())) { \
^be/src/exprs/runtime_filter.cpp:514: +2, including nesting penalty of 1, nesting level increased to 2
RETURN_IF_ERROR(
^be/src/common/status.h:629: expanded from macro 'RETURN_IF_ERROR'
do { \
^be/src/exprs/runtime_filter.cpp:514: +3, including nesting penalty of 2, nesting level increased to 3
RETURN_IF_ERROR(
^be/src/common/status.h:631: expanded from macro 'RETURN_IF_ERROR'
if (UNLIKELY(!_status_.ok())) { \
^be/src/exprs/runtime_filter.cpp:522: +2, including nesting penalty of 1, nesting level increased to 2
if (other_filter_type == RuntimeFilterType::IN_OR_BLOOM_FILTER) {
^be/src/exprs/runtime_filter.cpp:526: +2, including nesting penalty of 1, nesting level increased to 2
if (real_filter_type == RuntimeFilterType::IN_FILTER) {
^be/src/exprs/runtime_filter.cpp:530: +3, including nesting penalty of 2, nesting level increased to 3
if (other_filter_type == RuntimeFilterType::IN_FILTER) {
^be/src/exprs/runtime_filter.cpp:532: +4, including nesting penalty of 3, nesting level increased to 4
if (_max_in_num >= 0 && _context->hybrid_set->size() >= _max_in_num) {
^be/src/exprs/runtime_filter.cpp:534: +5, including nesting penalty of 4, nesting level increased to 5
RETURN_IF_ERROR(_context->bloom_filter_func->init_with_fixed_length());
^be/src/common/status.h:629: expanded from macro 'RETURN_IF_ERROR'
do { \
^be/src/exprs/runtime_filter.cpp:534: +6, including nesting penalty of 5, nesting level increased to 6
RETURN_IF_ERROR(_context->bloom_filter_func->init_with_fixed_length());
^be/src/common/status.h:631: expanded from macro 'RETURN_IF_ERROR'
if (UNLIKELY(!_status_.ok())) { \
^be/src/exprs/runtime_filter.cpp:535: +5, including nesting penalty of 4, nesting level increased to 5
RETURN_IF_ERROR(change_to_bloom_filter());
^be/src/common/status.h:629: expanded from macro 'RETURN_IF_ERROR'
do { \
^be/src/exprs/runtime_filter.cpp:535: +6, including nesting penalty of 5, nesting level increased to 6
RETURN_IF_ERROR(change_to_bloom_filter());
^be/src/common/status.h:631: expanded from macro 'RETURN_IF_ERROR'
if (UNLIKELY(!_status_.ok())) { \
^be/src/exprs/runtime_filter.cpp:537: +1, nesting level increased to 3
} else {
^be/src/exprs/runtime_filter.cpp:540: +4, including nesting penalty of 3, nesting level increased to 4
RETURN_IF_ERROR(change_to_bloom_filter());
^be/src/common/status.h:629: expanded from macro 'RETURN_IF_ERROR'
do { \
^be/src/exprs/runtime_filter.cpp:540: +5, including nesting penalty of 4, nesting level increased to 5
RETURN_IF_ERROR(change_to_bloom_filter());
^be/src/common/status.h:631: expanded from macro 'RETURN_IF_ERROR'
if (UNLIKELY(!_status_.ok())) { \
^be/src/exprs/runtime_filter.cpp:542: +1, nesting level increased to 2
} else {
^be/src/exprs/runtime_filter.cpp:543: +3, including nesting penalty of 2, nesting level increased to 3
if (other_filter_type == RuntimeFilterType::IN_FILTER) {
^be/src/exprs/runtime_filter.cpp:546: +1, nesting level increased to 3
} else {
^be/src/exprs/runtime_filter.cpp:548: +4, including nesting penalty of 3, nesting level increased to 4
RETURN_IF_ERROR(_context->bloom_filter_func->merge(
^be/src/common/status.h:629: expanded from macro 'RETURN_IF_ERROR'
do { \
^be/src/exprs/runtime_filter.cpp:548: +5, including nesting penalty of 4, nesting level increased to 5
RETURN_IF_ERROR(_context->bloom_filter_func->merge(
^be/src/common/status.h:631: expanded from macro 'RETURN_IF_ERROR'
if (UNLIKELY(!_status_.ok())) { \
^
github-actions
bot
added
the
approved
|
PR approved by at least one committer and no changes requested. |
|
PR approved by anyone and no changes requested. |
…ly (apache#41667) send ignored rf when hash join build closed early to avoid runtime filter sync/merge error Follow-up : apache#41292
…ly (apache#41667) ## Proposed changes send ignored rf when hash join build closed early to avoid runtime filter sync/merge error Follow-up : apache#41292
) In ##41667 we support ignore filter partitial, however, when sync_filter_size is turned off, the final filter may only contain part of the data because some filters are judged to be 'in'. For example: there are rf001(2 instances, in_or_bloom),rf000(2 instances, min_max), instance_1 has 1e8 row and rf001 change to bloom, the rf000 will not ignored instance_2 has 1 row and rf001 change to in, the rf000 will ignored finally, rf000 applied and make wrong result
) In ##41667 we support ignore filter partitial, however, when sync_filter_size is turned off, the final filter may only contain part of the data because some filters are judged to be 'in'. For example: there are rf001(2 instances, in_or_bloom),rf000(2 instances, min_max), instance_1 has 1e8 row and rf001 change to bloom, the rf000 will not ignored instance_2 has 1 row and rf001 change to in, the rf000 will ignored finally, rf000 applied and make wrong result
) In ##41667 we support ignore filter partitial, however, when sync_filter_size is turned off, the final filter may only contain part of the data because some filters are judged to be 'in'. For example: there are rf001(2 instances, in_or_bloom),rf000(2 instances, min_max), instance_1 has 1e8 row and rf001 change to bloom, the rf000 will not ignored instance_2 has 1 row and rf001 change to in, the rf000 will ignored finally, rf000 applied and make wrong result
Proposed changes
send ignored rf when hash join build closed early to avoid runtime filter sync/merge error
Follow-up : #41292