Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[fix](storage) low_cardinality_optimize core dump when is null predicate #9586

Merged
merged 5 commits into from
May 18, 2022
Merged

[fix](storage) low_cardinality_optimize core dump when is null predicate #9586

Changes from 2 commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
93c7510
[fix](storage) low_cardinality_optimize core dump when is null predicate
zenoyang May 16, 2022
dd6f2e0
f
zenoyang May 16, 2022
199b9f2
fix
zenoyang May 16, 2022
2aa61e8
fix2
zenoyang May 16, 2022
c9da60e
fix3
zenoyang May 17, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 22 additions & 11 deletions be/src/vec/columns/column_dictionary.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -98,12 +98,14 @@ class ColumnDictionary final : public COWHelper<IColumn, ColumnDictionary<T>> {
}

void insert_data(const char* pos, size_t /*length*/) override {
_codes.push_back(unaligned_load<T>(pos));
if (pos == nullptr) {
zenoyang marked this conversation as resolved.
Show resolved Hide resolved
_codes.push_back(_dict.get_null_code());
return;
}
LOG(FATAL) << "insert_data not supported in ColumnDictionary";
zenoyang marked this conversation as resolved.
Show resolved Hide resolved
}

void insert_data(const T value) { _codes.push_back(value); }

void insert_default() override { _codes.push_back(T()); }
void insert_default() override { _codes.push_back(_dict.get_null_code()); }

void clear() override {
_codes.clear();
Expand Down Expand Up @@ -219,13 +221,13 @@ class ColumnDictionary final : public COWHelper<IColumn, ColumnDictionary<T>> {
void insert_many_dict_data(const int32_t* data_array, size_t start_index,
const StringRef* dict_array, size_t data_num,
uint32_t dict_num) override {
if (!is_dict_inited()) {
_dict.reserve(dict_num);
if (_dict.empty()) {
_dict.reserve(dict_num + 1);
for (uint32_t i = 0; i < dict_num; ++i) {
auto value = StringValue(dict_array[i].data, dict_array[i].size);
_dict.insert_value(value);
}
_dict_inited = true;
_dict.insert_null_value(); // make the last dict value is null value
}

char* end_ptr = (char*)_codes.get_end_ptr();
Expand Down Expand Up @@ -263,8 +265,6 @@ class ColumnDictionary final : public COWHelper<IColumn, ColumnDictionary<T>> {
return _dict.find_codes(values);
}

bool is_dict_inited() const { return _dict_inited; }

bool is_dict_sorted() const { return _dict_sorted; }

bool is_dict_code_converted() const { return _dict_code_converted; }
Expand Down Expand Up @@ -296,6 +296,12 @@ class ColumnDictionary final : public COWHelper<IColumn, ColumnDictionary<T>> {
_inverted_index[value] = _inverted_index.size();
}

void insert_null_value() {
auto value = StringValue();
_dict_data.push_back_without_reserve(value);
_inverted_index[value] = _inverted_index.size();
}

int32_t find_code(const StringValue& value) const {
auto it = _inverted_index.find(value);
if (it != _inverted_index.end()) {
Expand All @@ -304,10 +310,14 @@ class ColumnDictionary final : public COWHelper<IColumn, ColumnDictionary<T>> {
return -1;
}

T get_null_code() {
return _dict_data.size() - 1; // The last dict value is null value
}
zenoyang marked this conversation as resolved.
Show resolved Hide resolved
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

So, this does not work now?


inline StringValue& get_value(T code) { return _dict_data[code]; }

inline void generate_hash_values() {
if (_hash_values.size() == 0) {
if (_hash_values.empty()) {
_hash_values.resize(_dict_data.size());
for (size_t i = 0; i < _dict_data.size(); i++) {
auto& sv = _dict_data[i];
Expand Down Expand Up @@ -380,6 +390,8 @@ class ColumnDictionary final : public COWHelper<IColumn, ColumnDictionary<T>> {

size_t byte_size() { return _dict_data.size() * sizeof(_dict_data[0]); }

bool empty() { return _dict_data.empty(); }

private:
StringValue::Comparator _comparator;
// dict code -> dict value
Expand All @@ -398,7 +410,6 @@ class ColumnDictionary final : public COWHelper<IColumn, ColumnDictionary<T>> {

private:
size_t _reserve_size;
bool _dict_inited = false;
bool _dict_sorted = false;
bool _dict_code_converted = false;
Dictionary _dict;
Expand Down