Skip to content

Commit 0562bab

Browse files
zenoyangmorningman
authored andcommitted
[fix](storage) low_cardinality_optimize core dump when is null predicate (#9586)
Issue Number: close #9555 Make the last value of the dictionary null, when ColumnDict inserts a null value, add the encoding corresponding to the last value of the dictionary·
1 parent 4315dc8 commit 0562bab

2 files changed

Lines changed: 19 additions & 18 deletions

File tree

‎be/src/olap/comparison_predicate.cpp‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -147,6 +147,9 @@ COMPARISON_PRED_COLUMN_BLOCK_EVALUATE(GreaterPredicate, >)
147147
COMPARISON_PRED_COLUMN_BLOCK_EVALUATE(GreaterEqualPredicate, >=)
148148

149149
// todo(zeno) define interface in IColumn to simplify code
150+
// If 1 OP 0 returns true, it means the predicate is > or >=
151+
// If 1 OP 1 returns true, it means the predicate is >= or <=
152+
// by this way, avoid redundant code
150153
#define COMPARISON_PRED_COLUMN_EVALUATE(CLASS, OP, IS_RANGE) \
151154
template <class T> \
152155
void CLASS<T>::evaluate(vectorized::IColumn& column, uint16_t* sel, uint16_t* size) const { \
@@ -164,7 +167,7 @@ COMPARISON_PRED_COLUMN_BLOCK_EVALUATE(GreaterEqualPredicate, >=)
164167
vectorized::ColumnDictionary<vectorized::Int32>>(nested_col); \
165168
auto& data_array = nested_col_ptr->get_data(); \
166169
auto dict_code = \
167-
IS_RANGE ? nested_col_ptr->find_code_by_bound(_value, 0 OP 1, 1 OP 1) \
170+
IS_RANGE ? nested_col_ptr->find_code_by_bound(_value, 1 OP 0, 1 OP 1) \
168171
: nested_col_ptr->find_code(_value); \
169172
for (uint16_t i = 0; i < *size; i++) { \
170173
uint16_t idx = sel[i]; \
@@ -192,7 +195,7 @@ COMPARISON_PRED_COLUMN_BLOCK_EVALUATE(GreaterEqualPredicate, >=)
192195
auto& dict_col = \
193196
reinterpret_cast<vectorized::ColumnDictionary<vectorized::Int32>&>(column);\
194197
auto& data_array = dict_col.get_data(); \
195-
auto dict_code = IS_RANGE ? dict_col.find_code_by_bound(_value, 0 OP 1, 1 OP 1) \
198+
auto dict_code = IS_RANGE ? dict_col.find_code_by_bound(_value, 1 OP 0, 1 OP 1) \
196199
: dict_col.find_code(_value); \
197200
for (uint16_t i = 0; i < *size; ++i) { \
198201
uint16_t idx = sel[i]; \

‎be/src/vec/columns/column_dictionary.h‎

Lines changed: 14 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -97,12 +97,10 @@ class ColumnDictionary final : public COWHelper<IColumn, ColumnDictionary<T>> {
9797
}
9898

9999
void insert_data(const char* pos, size_t /*length*/) override {
100-
_codes.push_back(unaligned_load<T>(pos));
100+
LOG(FATAL) << "insert_data not supported in ColumnDictionary";
101101
}
102102

103-
void insert_data(const T value) { _codes.push_back(value); }
104-
105-
void insert_default() override { _codes.push_back(T()); }
103+
void insert_default() override { _codes.push_back(_dict.get_null_code()); }
106104

107105
void clear() override {
108106
_codes.clear();
@@ -218,13 +216,12 @@ class ColumnDictionary final : public COWHelper<IColumn, ColumnDictionary<T>> {
218216
void insert_many_dict_data(const int32_t* data_array, size_t start_index,
219217
const StringRef* dict_array, size_t data_num,
220218
uint32_t dict_num) override {
221-
if (!is_dict_inited()) {
219+
if (_dict.empty()) {
222220
_dict.reserve(dict_num);
223221
for (uint32_t i = 0; i < dict_num; ++i) {
224222
auto value = StringValue(dict_array[i].data, dict_array[i].size);
225223
_dict.insert_value(value);
226224
}
227-
_dict_inited = true;
228225
}
229226

230227
char* end_ptr = (char*)_codes.get_end_ptr();
@@ -266,8 +263,6 @@ class ColumnDictionary final : public COWHelper<IColumn, ColumnDictionary<T>> {
266263
return _dict.find_codes(values);
267264
}
268265

269-
bool is_dict_inited() const { return _dict_inited; }
270-
271266
bool is_dict_sorted() const { return _dict_sorted; }
272267

273268
bool is_dict_code_converted() const { return _dict_code_converted; }
@@ -304,13 +299,17 @@ class ColumnDictionary final : public COWHelper<IColumn, ColumnDictionary<T>> {
304299
if (it != _inverted_index.end()) {
305300
return it->second;
306301
}
307-
return -1;
302+
return -2; // -1 is null code
303+
}
304+
305+
T get_null_code() { return -1; }
306+
307+
inline StringValue& get_value(T code) {
308+
return code >= _dict_data.size() ? _null_value : _dict_data[code];
308309
}
309310

310-
inline StringValue& get_value(T code) { return _dict_data[code]; }
311-
312311
inline void generate_hash_values() {
313-
if (_hash_values.size() == 0) {
312+
if (_hash_values.empty()) {
314313
_hash_values.resize(_dict_data.size());
315314
for (size_t i = 0; i < _dict_data.size(); i++) {
316315
auto& sv = _dict_data[i];
@@ -387,7 +386,10 @@ class ColumnDictionary final : public COWHelper<IColumn, ColumnDictionary<T>> {
387386

388387
size_t byte_size() { return _dict_data.size() * sizeof(_dict_data[0]); }
389388

389+
bool empty() { return _dict_data.empty(); }
390+
390391
private:
392+
StringValue _null_value = StringValue();
391393
StringValue::Comparator _comparator;
392394
// dict code -> dict value
393395
DictContainer _dict_data;
@@ -405,16 +407,12 @@ class ColumnDictionary final : public COWHelper<IColumn, ColumnDictionary<T>> {
405407

406408
private:
407409
size_t _reserve_size;
408-
bool _dict_inited = false;
409410
bool _dict_sorted = false;
410411
bool _dict_code_converted = false;
411412
Dictionary _dict;
412413
Container _codes;
413414
};
414415

415-
template class ColumnDictionary<uint8_t>;
416-
template class ColumnDictionary<uint16_t>;
417-
template class ColumnDictionary<uint32_t>;
418416
template class ColumnDictionary<int32_t>;
419417

420418
using ColumnDictI32 = vectorized::ColumnDictionary<doris::vectorized::Int32>;

0 commit comments

Comments
 (0)