Skip to content

Commit 7898c81

Browse files
authored
Revert "[improvement]Do not lazily read dict encoded columns (#10420)" (#10466)
Reason: 1. Some queries performance degradation 2. Coredump bug: #10419 This reverts commit 904e757.
1 parent 17eb8c0 commit 7898c81

2 files changed

Lines changed: 64 additions & 71 deletions

File tree

‎be/src/olap/rowset/segment_v2/segment_iterator.cpp‎

Lines changed: 62 additions & 65 deletions
Original file line numberDiff line numberDiff line change
@@ -643,18 +643,13 @@ Status SegmentIterator::next_batch(RowBlockV2* block) {
643643

644644
// todo(wb) need a UT here
645645
void SegmentIterator::_vec_init_lazy_materialization() {
646-
_is_first_read_column.resize(_schema.columns().size(), false);
646+
_is_pred_column.resize(_schema.columns().size(), false);
647647

648648
// including short/vec/delete pred
649649
std::set<ColumnId> pred_column_ids;
650-
651-
// Other columns(not predicate columns) can read firstly together with predicate columns
652-
std::set<ColumnId> other_first_read_column_ids;
653-
654650
_lazy_materialization_read = false;
655651

656652
std::set<ColumnId> del_cond_id_set;
657-
std::set<ColumnId> lazy_read_column_ids;
658653
_opts.delete_condition_predicates->get_all_column_ids(del_cond_id_set);
659654

660655
if (!_col_predicates.empty() || !del_cond_id_set.empty()) {
@@ -663,7 +658,7 @@ void SegmentIterator::_vec_init_lazy_materialization() {
663658

664659
for (auto predicate : _col_predicates) {
665660
auto cid = predicate->column_id();
666-
_is_first_read_column[cid] = true;
661+
_is_pred_column[cid] = true;
667662
pred_column_ids.insert(cid);
668663

669664
// Step1: check pred using short eval or vec eval
@@ -686,7 +681,7 @@ void SegmentIterator::_vec_init_lazy_materialization() {
686681
pred_column_ids.insert(del_cond_id_set.begin(), del_cond_id_set.end());
687682

688683
for (auto cid : del_cond_id_set) {
689-
_is_first_read_column[cid] = true;
684+
_is_pred_column[cid] = true;
690685
}
691686
}
692687

@@ -703,58 +698,65 @@ void SegmentIterator::_vec_init_lazy_materialization() {
703698
}
704699

705700
// Step 2: check non-predicate read costs to determine whether need lazy materialization
701+
// fill _non_predicate_columns.
706702
// note(wb) For block schema, query layer and storage layer may have some diff
707703
// query layer block schema not contains delete column, but storage layer appends delete column to end of block schema
708704
// When output block to query layer, delete column can be skipped.
709705
// _schema.column_ids() stands for storage layer block schema, so it contains delete columnid
710706
// we just regard delete column as common pred column here.
711-
for (size_t i = 0; i < _schema.num_column_ids(); ++i) {
712-
auto cid = _schema.column_id(i);
713-
FieldType type = _schema.column(cid)->type();
714-
if (!_is_first_read_column[cid]) {
715-
switch (type) {
716-
case OLAP_FIELD_TYPE_VARCHAR:
717-
case OLAP_FIELD_TYPE_CHAR:
718-
case OLAP_FIELD_TYPE_STRING: {
719-
// if a string column is all dict encoding in one segment, it's almost same as
720-
// an int32_t column, it can be read together with predicate columns.
721-
if (config::enable_low_cardinality_optimize &&
722-
_column_iterators[cid]->is_all_dict_encoding()) {
723-
other_first_read_column_ids.insert(cid);
724-
_is_first_read_column[cid] = true;
725-
} else {
726-
lazy_read_column_ids.insert(cid);
707+
if (_schema.column_ids().size() > pred_column_ids.size()) {
708+
for (auto cid : _schema.column_ids()) {
709+
if (!_is_pred_column[cid]) {
710+
_non_predicate_columns.push_back(cid);
711+
FieldType type = _schema.column(cid)->type();
712+
713+
// todo(wb) maybe we can make read char type faster
714+
// todo(wb) support map/array type
715+
// todo(wb) consider multiple integer columns cost, such as 1000 columns, maybe lazy materialization faster
716+
if (!_lazy_materialization_read &&
717+
(_is_need_vec_eval ||
718+
_is_need_short_eval) && // only when pred exists, we need to consider lazy materialization
719+
(type == OLAP_FIELD_TYPE_HLL || type == OLAP_FIELD_TYPE_OBJECT ||
720+
type == OLAP_FIELD_TYPE_VARCHAR || type == OLAP_FIELD_TYPE_CHAR ||
721+
type == OLAP_FIELD_TYPE_STRING || type == OLAP_FIELD_TYPE_BOOL ||
722+
type == OLAP_FIELD_TYPE_DATE || type == OLAP_FIELD_TYPE_DATETIME ||
723+
type == OLAP_FIELD_TYPE_DECIMAL)) {
724+
_lazy_materialization_read = true;
727725
}
728-
break;
729-
}
730-
case OLAP_FIELD_TYPE_HLL:
731-
case OLAP_FIELD_TYPE_OBJECT:
732-
case OLAP_FIELD_TYPE_BOOL:
733-
case OLAP_FIELD_TYPE_DATE:
734-
case OLAP_FIELD_TYPE_DATETIME:
735-
case OLAP_FIELD_TYPE_DECIMAL:
736-
lazy_read_column_ids.insert(cid);
737-
break;
738-
default:
739-
other_first_read_column_ids.insert(cid);
740-
_is_first_read_column[cid] = true;
741-
break;
742726
}
743727
}
744728
}
745729

746-
_first_read_column_ids.assign(pred_column_ids.begin(), pred_column_ids.end());
747-
_first_read_column_ids.insert(_first_read_column_ids.end(), other_first_read_column_ids.begin(),
748-
other_first_read_column_ids.end());
749-
if (_is_need_vec_eval || _is_need_short_eval) {
750-
_lazy_materialization_read = !lazy_read_column_ids.empty();
751-
_lazy_read_column_ids.assign(lazy_read_column_ids.begin(), lazy_read_column_ids.end());
752-
} else {
753-
_lazy_materialization_read = false;
754-
755-
// No need to lazy read, all columns should be read firstly.
756-
_first_read_column_ids.insert(_first_read_column_ids.end(), lazy_read_column_ids.begin(),
757-
lazy_read_column_ids.end());
730+
// Step 3: fill column ids for read and output
731+
if (_lazy_materialization_read) {
732+
// insert pred cid to first_read_columns
733+
for (auto cid : pred_column_ids) {
734+
_first_read_column_ids.push_back(cid);
735+
}
736+
} else if (!_is_need_vec_eval &&
737+
!_is_need_short_eval) { // no pred exists, just read and output column
738+
for (int i = 0; i < _schema.num_column_ids(); i++) {
739+
auto cid = _schema.column_id(i);
740+
_first_read_column_ids.push_back(cid);
741+
}
742+
} else { // pred exits, but we can eliminate lazy materialization
743+
// insert pred/non-pred cid to first read columns
744+
std::set<ColumnId> pred_id_set;
745+
pred_id_set.insert(_short_cir_pred_column_ids.begin(), _short_cir_pred_column_ids.end());
746+
pred_id_set.insert(_vec_pred_column_ids.begin(), _vec_pred_column_ids.end());
747+
std::set<ColumnId> non_pred_set(_non_predicate_columns.begin(),
748+
_non_predicate_columns.end());
749+
750+
for (int i = 0; i < _schema.num_column_ids(); i++) {
751+
auto cid = _schema.column_id(i);
752+
if (pred_id_set.find(cid) != pred_id_set.end()) {
753+
_first_read_column_ids.push_back(cid);
754+
} else if (non_pred_set.find(cid) != non_pred_set.end()) {
755+
_first_read_column_ids.push_back(cid);
756+
// when _lazy_materialization_read = false, non-predicate column should also be filtered by sel idx, so we regard it as pred columns
757+
_is_pred_column[cid] = true;
758+
}
759+
}
758760
}
759761

760762
// make _schema_block_id_map
@@ -820,7 +822,7 @@ void SegmentIterator::_init_current_block(
820822
auto column_desc = _schema.column(cid);
821823

822824
// the column in block must clear() here to insert new data
823-
if (_is_first_read_column[cid] ||
825+
if (_is_pred_column[cid] ||
824826
i >= block->columns()) { //todo(wb) maybe we can release it after output block
825827
current_columns[cid]->clear();
826828
} else { // non-predicate column
@@ -839,14 +841,9 @@ void SegmentIterator::_init_current_block(
839841
}
840842
}
841843

842-
void SegmentIterator::_output_lazy_read_columns(vectorized::Block* block) {
843-
_output_columns_by_relace(block, _lazy_read_column_ids);
844-
}
845-
846-
void SegmentIterator::_output_columns_by_relace(vectorized::Block* block,
847-
const std::vector<ColumnId>& column_ids) {
844+
void SegmentIterator::_output_non_pred_columns(vectorized::Block* block) {
848845
SCOPED_RAW_TIMER(&_opts.stats->output_col_ns);
849-
for (auto cid : column_ids) {
846+
for (auto cid : _non_predicate_columns) {
850847
auto loc = _schema_block_id_map[cid];
851848
// if loc < block->block->columns() means the column is delete column and should
852849
// not output by block, so just skip the column.
@@ -1007,7 +1004,7 @@ Status SegmentIterator::next_batch(vectorized::Block* block) {
10071004
for (size_t i = 0; i < _schema.num_column_ids(); i++) {
10081005
auto cid = _schema.column_id(i);
10091006
auto column_desc = _schema.column(cid);
1010-
if (_is_first_read_column[cid]) {
1007+
if (_is_pred_column[cid]) {
10111008
_current_return_columns[cid] = Schema::get_predicate_column_nullable_ptr(
10121009
column_desc->type(), column_desc->is_nullable());
10131010
_current_return_columns[cid]->reserve(_opts.block_row_max);
@@ -1039,7 +1036,7 @@ Status SegmentIterator::next_batch(vectorized::Block* block) {
10391036
for (int i = 0; i < block->columns(); i++) {
10401037
auto cid = _schema.column_id(i);
10411038
// todo(wb) abstract make column where
1042-
if (!_is_first_read_column[cid]) { // non-predicate
1039+
if (!_is_pred_column[cid]) { // non-predicate
10431040
block->replace_by_position(i, std::move(_current_return_columns[cid]));
10441041
}
10451042
}
@@ -1048,7 +1045,7 @@ Status SegmentIterator::next_batch(vectorized::Block* block) {
10481045
}
10491046

10501047
if (!_is_need_vec_eval && !_is_need_short_eval) {
1051-
_output_columns_by_relace(block, _first_read_column_ids);
1048+
_output_non_pred_columns(block);
10521049
} else {
10531050
uint16_t selected_size = nrows_read;
10541051
uint16_t sel_rowid_idx[selected_size];
@@ -1073,13 +1070,13 @@ Status SegmentIterator::next_batch(vectorized::Block* block) {
10731070
return ret;
10741071
}
10751072

1076-
// step3: read lazy_read_column_ids column
1077-
_read_columns_by_rowids(_lazy_read_column_ids, _block_rowids, sel_rowid_idx, selected_size,
1073+
// step3: read non_predicate column
1074+
_read_columns_by_rowids(_non_predicate_columns, _block_rowids, sel_rowid_idx, selected_size,
10781075
&_current_return_columns);
10791076

10801077
// step4: output columns
1081-
// 4.1 output lazy_read_column_ids column
1082-
_output_lazy_read_columns(block);
1078+
// 4.1 output non-predicate column
1079+
_output_non_pred_columns(block);
10831080

10841081
// 4.3 output short circuit and predicate column
10851082
// when lazy materialization enables, _first_read_column_ids = distinct(_short_cir_pred_column_ids + _vec_pred_column_ids)

‎be/src/olap/rowset/segment_v2/segment_iterator.h‎

Lines changed: 2 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -101,9 +101,7 @@ class SegmentIterator : public RowwiseIterator {
101101
std::vector<vectorized::MutableColumnPtr>& non_pred_vector);
102102
uint16_t _evaluate_vectorization_predicate(uint16_t* sel_rowid_idx, uint16_t selected_size);
103103
uint16_t _evaluate_short_circuit_predicate(uint16_t* sel_rowid_idx, uint16_t selected_size);
104-
void _output_lazy_read_columns(vectorized::Block* block);
105-
void _output_columns_by_relace(vectorized::Block* block,
106-
const std::vector<ColumnId>& column_ids);
104+
void _output_non_pred_columns(vectorized::Block* block);
107105
void _read_columns_by_rowids(std::vector<ColumnId>& read_column_ids,
108106
std::vector<rowid_t>& rowid_vector, uint16_t* sel_rowid_idx,
109107
size_t select_size, vectorized::MutableColumns* mutable_columns);
@@ -158,7 +156,7 @@ class SegmentIterator : public RowwiseIterator {
158156
_vec_pred_column_ids; // keep columnId of columns for vectorized predicate evaluation
159157
std::vector<ColumnId>
160158
_short_cir_pred_column_ids; // keep columnId of columns for short circuit predicate evaluation
161-
std::vector<bool> _is_first_read_column; // columns hold by segmentIter
159+
std::vector<bool> _is_pred_column; // columns hold by segmentIter
162160
vectorized::MutableColumns _current_return_columns;
163161
std::unique_ptr<AndBlockColumnPredicate> _pre_eval_block_predicate;
164162
std::vector<ColumnPredicate*> _short_cir_eval_predicate;
@@ -167,8 +165,6 @@ class SegmentIterator : public RowwiseIterator {
167165
// second, read non-predicate columns
168166
// so we need a field to stand for columns first time to read
169167
std::vector<ColumnId> _first_read_column_ids;
170-
std::vector<ColumnId> _lazy_read_column_ids;
171-
172168
std::vector<int> _schema_block_id_map; // map from schema column id to column idx in Block
173169

174170
// the actual init process is delayed to the first call to next_batch()

0 commit comments

Comments
 (0)