Skip to content

Commit bdaf0b3

Browse files
authored
[fix](storage) low_cardinality_optimize core dump when is null predicate (#9586)
Issue Number: close #9555 Make the last value of the dictionary null, when ColumnDict inserts a null value, add the encoding corresponding to the last value of the dictionary·
1 parent c9ab5e2 commit bdaf0b3

2 files changed

Lines changed: 18 additions & 17 deletions

File tree

‎be/src/olap/comparison_predicate.cpp‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -144,6 +144,9 @@ COMPARISON_PRED_COLUMN_BLOCK_EVALUATE(GreaterPredicate, >)
144144
COMPARISON_PRED_COLUMN_BLOCK_EVALUATE(GreaterEqualPredicate, >=)
145145

146146
// todo(zeno) define interface in IColumn to simplify code
147+
// If 1 OP 0 returns true, it means the predicate is > or >=
148+
// If 1 OP 1 returns true, it means the predicate is >= or <=
149+
// by this way, avoid redundant code
147150
#define COMPARISON_PRED_COLUMN_EVALUATE(CLASS, OP, IS_RANGE) \
148151
template <class T> \
149152
void CLASS<T>::evaluate(vectorized::IColumn& column, uint16_t* sel, uint16_t* size) const { \
@@ -161,7 +164,7 @@ COMPARISON_PRED_COLUMN_BLOCK_EVALUATE(GreaterEqualPredicate, >=)
161164
vectorized::ColumnDictionary<vectorized::Int32>>(nested_col); \
162165
auto& data_array = nested_col_ptr->get_data(); \
163166
auto dict_code = \
164-
IS_RANGE ? nested_col_ptr->find_code_by_bound(_value, 0 OP 1, 1 OP 1) \
167+
IS_RANGE ? nested_col_ptr->find_code_by_bound(_value, 1 OP 0, 1 OP 1) \
165168
: nested_col_ptr->find_code(_value); \
166169
for (uint16_t i = 0; i < *size; i++) { \
167170
uint16_t idx = sel[i]; \
@@ -190,7 +193,7 @@ COMPARISON_PRED_COLUMN_BLOCK_EVALUATE(GreaterEqualPredicate, >=)
190193
reinterpret_cast<vectorized::ColumnDictionary<vectorized::Int32>&>( \
191194
column); \
192195
auto& data_array = dict_col.get_data(); \
193-
auto dict_code = IS_RANGE ? dict_col.find_code_by_bound(_value, 0 OP 1, 1 OP 1) \
196+
auto dict_code = IS_RANGE ? dict_col.find_code_by_bound(_value, 1 OP 0, 1 OP 1) \
194197
: dict_col.find_code(_value); \
195198
for (uint16_t i = 0; i < *size; ++i) { \
196199
uint16_t idx = sel[i]; \

‎be/src/vec/columns/column_dictionary.h‎

Lines changed: 13 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -98,12 +98,10 @@ class ColumnDictionary final : public COWHelper<IColumn, ColumnDictionary<T>> {
9898
}
9999

100100
void insert_data(const char* pos, size_t /*length*/) override {
101-
_codes.push_back(unaligned_load<T>(pos));
101+
LOG(FATAL) << "insert_data not supported in ColumnDictionary";
102102
}
103103

104-
void insert_data(const T value) { _codes.push_back(value); }
105-
106-
void insert_default() override { _codes.push_back(T()); }
104+
void insert_default() override { _codes.push_back(_dict.get_null_code()); }
107105

108106
void clear() override {
109107
_codes.clear();
@@ -219,13 +217,12 @@ class ColumnDictionary final : public COWHelper<IColumn, ColumnDictionary<T>> {
219217
void insert_many_dict_data(const int32_t* data_array, size_t start_index,
220218
const StringRef* dict_array, size_t data_num,
221219
uint32_t dict_num) override {
222-
if (!is_dict_inited()) {
220+
if (_dict.empty()) {
223221
_dict.reserve(dict_num);
224222
for (uint32_t i = 0; i < dict_num; ++i) {
225223
auto value = StringValue(dict_array[i].data, dict_array[i].size);
226224
_dict.insert_value(value);
227225
}
228-
_dict_inited = true;
229226
}
230227

231228
char* end_ptr = (char*)_codes.get_end_ptr();
@@ -263,8 +260,6 @@ class ColumnDictionary final : public COWHelper<IColumn, ColumnDictionary<T>> {
263260
return _dict.find_codes(values);
264261
}
265262

266-
bool is_dict_inited() const { return _dict_inited; }
267-
268263
bool is_dict_sorted() const { return _dict_sorted; }
269264

270265
bool is_dict_code_converted() const { return _dict_code_converted; }
@@ -301,13 +296,17 @@ class ColumnDictionary final : public COWHelper<IColumn, ColumnDictionary<T>> {
301296
if (it != _inverted_index.end()) {
302297
return it->second;
303298
}
304-
return -1;
299+
return -2; // -1 is null code
305300
}
306301

307-
inline StringValue& get_value(T code) { return _dict_data[code]; }
302+
T get_null_code() { return -1; }
303+
304+
inline StringValue& get_value(T code) {
305+
return code >= _dict_data.size() ? _null_value : _dict_data[code];
306+
}
308307

309308
inline void generate_hash_values() {
310-
if (_hash_values.size() == 0) {
309+
if (_hash_values.empty()) {
311310
_hash_values.resize(_dict_data.size());
312311
for (size_t i = 0; i < _dict_data.size(); i++) {
313312
auto& sv = _dict_data[i];
@@ -380,7 +379,10 @@ class ColumnDictionary final : public COWHelper<IColumn, ColumnDictionary<T>> {
380379

381380
size_t byte_size() { return _dict_data.size() * sizeof(_dict_data[0]); }
382381

382+
bool empty() { return _dict_data.empty(); }
383+
383384
private:
385+
StringValue _null_value = StringValue();
384386
StringValue::Comparator _comparator;
385387
// dict code -> dict value
386388
DictContainer _dict_data;
@@ -398,16 +400,12 @@ class ColumnDictionary final : public COWHelper<IColumn, ColumnDictionary<T>> {
398400

399401
private:
400402
size_t _reserve_size;
401-
bool _dict_inited = false;
402403
bool _dict_sorted = false;
403404
bool _dict_code_converted = false;
404405
Dictionary _dict;
405406
Container _codes;
406407
};
407408

408-
template class ColumnDictionary<uint8_t>;
409-
template class ColumnDictionary<uint16_t>;
410-
template class ColumnDictionary<uint32_t>;
411409
template class ColumnDictionary<int32_t>;
412410

413411
using ColumnDictI32 = vectorized::ColumnDictionary<doris::vectorized::Int32>;

0 commit comments

Comments
 (0)