@@ -643,18 +643,13 @@ Status SegmentIterator::next_batch(RowBlockV2* block) {
643643
644644// todo(wb) need a UT here
645645void SegmentIterator::_vec_init_lazy_materialization () {
646- _is_first_read_column .resize (_schema.columns ().size (), false );
646+ _is_pred_column .resize (_schema.columns ().size (), false );
647647
648648 // including short/vec/delete pred
649649 std::set<ColumnId> pred_column_ids;
650-
651- // Other columns(not predicate columns) can read firstly together with predicate columns
652- std::set<ColumnId> other_first_read_column_ids;
653-
654650 _lazy_materialization_read = false ;
655651
656652 std::set<ColumnId> del_cond_id_set;
657- std::set<ColumnId> lazy_read_column_ids;
658653 _opts.delete_condition_predicates ->get_all_column_ids (del_cond_id_set);
659654
660655 if (!_col_predicates.empty () || !del_cond_id_set.empty ()) {
@@ -663,7 +658,7 @@ void SegmentIterator::_vec_init_lazy_materialization() {
663658
664659 for (auto predicate : _col_predicates) {
665660 auto cid = predicate->column_id ();
666- _is_first_read_column [cid] = true ;
661+ _is_pred_column [cid] = true ;
667662 pred_column_ids.insert (cid);
668663
669664 // Step1: check pred using short eval or vec eval
@@ -686,7 +681,7 @@ void SegmentIterator::_vec_init_lazy_materialization() {
686681 pred_column_ids.insert (del_cond_id_set.begin (), del_cond_id_set.end ());
687682
688683 for (auto cid : del_cond_id_set) {
689- _is_first_read_column [cid] = true ;
684+ _is_pred_column [cid] = true ;
690685 }
691686 }
692687
@@ -703,58 +698,65 @@ void SegmentIterator::_vec_init_lazy_materialization() {
703698 }
704699
705700 // Step 2: check non-predicate read costs to determine whether need lazy materialization
701+ // fill _non_predicate_columns.
706702 // note(wb) For block schema, query layer and storage layer may have some diff
707703 // query layer block schema not contains delete column, but storage layer appends delete column to end of block schema
708704 // When output block to query layer, delete column can be skipped.
709705 // _schema.column_ids() stands for storage layer block schema, so it contains delete columnid
710706 // we just regard delete column as common pred column here.
711- for (size_t i = 0 ; i < _schema.num_column_ids (); ++i) {
712- auto cid = _schema.column_id (i);
713- FieldType type = _schema.column (cid)->type ();
714- if (!_is_first_read_column[cid]) {
715- switch (type) {
716- case OLAP_FIELD_TYPE_VARCHAR :
717- case OLAP_FIELD_TYPE_CHAR :
718- case OLAP_FIELD_TYPE_STRING : {
719- // if a string column is all dict encoding in one segment, it's almost same as
720- // an int32_t column, it can be read together with predicate columns.
721- if (config::enable_low_cardinality_optimize &&
722- _column_iterators[cid]->is_all_dict_encoding ()) {
723- other_first_read_column_ids.insert (cid);
724- _is_first_read_column[cid] = true ;
725- } else {
726- lazy_read_column_ids.insert (cid);
707+ if (_schema.column_ids ().size () > pred_column_ids.size ()) {
708+ for (auto cid : _schema.column_ids ()) {
709+ if (!_is_pred_column[cid]) {
710+ _non_predicate_columns.push_back (cid);
711+ FieldType type = _schema.column (cid)->type ();
712+
713+ // todo(wb) maybe we can make read char type faster
714+ // todo(wb) support map/array type
715+ // todo(wb) consider multiple integer columns cost, such as 1000 columns, maybe lazy materialization faster
716+ if (!_lazy_materialization_read &&
717+ (_is_need_vec_eval ||
718+ _is_need_short_eval) && // only when pred exists, we need to consider lazy materialization
719+ (type == OLAP_FIELD_TYPE_HLL || type == OLAP_FIELD_TYPE_OBJECT ||
720+ type == OLAP_FIELD_TYPE_VARCHAR || type == OLAP_FIELD_TYPE_CHAR ||
721+ type == OLAP_FIELD_TYPE_STRING || type == OLAP_FIELD_TYPE_BOOL ||
722+ type == OLAP_FIELD_TYPE_DATE || type == OLAP_FIELD_TYPE_DATETIME ||
723+ type == OLAP_FIELD_TYPE_DECIMAL )) {
724+ _lazy_materialization_read = true ;
727725 }
728- break ;
729- }
730- case OLAP_FIELD_TYPE_HLL :
731- case OLAP_FIELD_TYPE_OBJECT :
732- case OLAP_FIELD_TYPE_BOOL :
733- case OLAP_FIELD_TYPE_DATE :
734- case OLAP_FIELD_TYPE_DATETIME :
735- case OLAP_FIELD_TYPE_DECIMAL :
736- lazy_read_column_ids.insert (cid);
737- break ;
738- default :
739- other_first_read_column_ids.insert (cid);
740- _is_first_read_column[cid] = true ;
741- break ;
742726 }
743727 }
744728 }
745729
746- _first_read_column_ids.assign (pred_column_ids.begin (), pred_column_ids.end ());
747- _first_read_column_ids.insert (_first_read_column_ids.end (), other_first_read_column_ids.begin (),
748- other_first_read_column_ids.end ());
749- if (_is_need_vec_eval || _is_need_short_eval) {
750- _lazy_materialization_read = !lazy_read_column_ids.empty ();
751- _lazy_read_column_ids.assign (lazy_read_column_ids.begin (), lazy_read_column_ids.end ());
752- } else {
753- _lazy_materialization_read = false ;
754-
755- // No need to lazy read, all columns should be read firstly.
756- _first_read_column_ids.insert (_first_read_column_ids.end (), lazy_read_column_ids.begin (),
757- lazy_read_column_ids.end ());
730+ // Step 3: fill column ids for read and output
731+ if (_lazy_materialization_read) {
732+ // insert pred cid to first_read_columns
733+ for (auto cid : pred_column_ids) {
734+ _first_read_column_ids.push_back (cid);
735+ }
736+ } else if (!_is_need_vec_eval &&
737+ !_is_need_short_eval) { // no pred exists, just read and output column
738+ for (int i = 0 ; i < _schema.num_column_ids (); i++) {
739+ auto cid = _schema.column_id (i);
740+ _first_read_column_ids.push_back (cid);
741+ }
742+ } else { // pred exits, but we can eliminate lazy materialization
743+ // insert pred/non-pred cid to first read columns
744+ std::set<ColumnId> pred_id_set;
745+ pred_id_set.insert (_short_cir_pred_column_ids.begin (), _short_cir_pred_column_ids.end ());
746+ pred_id_set.insert (_vec_pred_column_ids.begin (), _vec_pred_column_ids.end ());
747+ std::set<ColumnId> non_pred_set (_non_predicate_columns.begin (),
748+ _non_predicate_columns.end ());
749+
750+ for (int i = 0 ; i < _schema.num_column_ids (); i++) {
751+ auto cid = _schema.column_id (i);
752+ if (pred_id_set.find (cid) != pred_id_set.end ()) {
753+ _first_read_column_ids.push_back (cid);
754+ } else if (non_pred_set.find (cid) != non_pred_set.end ()) {
755+ _first_read_column_ids.push_back (cid);
756+ // when _lazy_materialization_read = false, non-predicate column should also be filtered by sel idx, so we regard it as pred columns
757+ _is_pred_column[cid] = true ;
758+ }
759+ }
758760 }
759761
760762 // make _schema_block_id_map
@@ -820,7 +822,7 @@ void SegmentIterator::_init_current_block(
820822 auto column_desc = _schema.column (cid);
821823
822824 // the column in block must clear() here to insert new data
823- if (_is_first_read_column [cid] ||
825+ if (_is_pred_column [cid] ||
824826 i >= block->columns ()) { // todo(wb) maybe we can release it after output block
825827 current_columns[cid]->clear ();
826828 } else { // non-predicate column
@@ -839,14 +841,9 @@ void SegmentIterator::_init_current_block(
839841 }
840842}
841843
842- void SegmentIterator::_output_lazy_read_columns (vectorized::Block* block) {
843- _output_columns_by_relace (block, _lazy_read_column_ids);
844- }
845-
846- void SegmentIterator::_output_columns_by_relace (vectorized::Block* block,
847- const std::vector<ColumnId>& column_ids) {
844+ void SegmentIterator::_output_non_pred_columns (vectorized::Block* block) {
848845 SCOPED_RAW_TIMER (&_opts.stats ->output_col_ns );
849- for (auto cid : column_ids ) {
846+ for (auto cid : _non_predicate_columns ) {
850847 auto loc = _schema_block_id_map[cid];
851848 // if loc < block->block->columns() means the column is delete column and should
852849 // not output by block, so just skip the column.
@@ -1007,7 +1004,7 @@ Status SegmentIterator::next_batch(vectorized::Block* block) {
10071004 for (size_t i = 0 ; i < _schema.num_column_ids (); i++) {
10081005 auto cid = _schema.column_id (i);
10091006 auto column_desc = _schema.column (cid);
1010- if (_is_first_read_column [cid]) {
1007+ if (_is_pred_column [cid]) {
10111008 _current_return_columns[cid] = Schema::get_predicate_column_nullable_ptr (
10121009 column_desc->type (), column_desc->is_nullable ());
10131010 _current_return_columns[cid]->reserve (_opts.block_row_max );
@@ -1039,7 +1036,7 @@ Status SegmentIterator::next_batch(vectorized::Block* block) {
10391036 for (int i = 0 ; i < block->columns (); i++) {
10401037 auto cid = _schema.column_id (i);
10411038 // todo(wb) abstract make column where
1042- if (!_is_first_read_column [cid]) { // non-predicate
1039+ if (!_is_pred_column [cid]) { // non-predicate
10431040 block->replace_by_position (i, std::move (_current_return_columns[cid]));
10441041 }
10451042 }
@@ -1048,7 +1045,7 @@ Status SegmentIterator::next_batch(vectorized::Block* block) {
10481045 }
10491046
10501047 if (!_is_need_vec_eval && !_is_need_short_eval) {
1051- _output_columns_by_relace (block, _first_read_column_ids );
1048+ _output_non_pred_columns (block);
10521049 } else {
10531050 uint16_t selected_size = nrows_read;
10541051 uint16_t sel_rowid_idx[selected_size];
@@ -1073,13 +1070,13 @@ Status SegmentIterator::next_batch(vectorized::Block* block) {
10731070 return ret;
10741071 }
10751072
1076- // step3: read lazy_read_column_ids column
1077- _read_columns_by_rowids (_lazy_read_column_ids , _block_rowids, sel_rowid_idx, selected_size,
1073+ // step3: read non_predicate column
1074+ _read_columns_by_rowids (_non_predicate_columns , _block_rowids, sel_rowid_idx, selected_size,
10781075 &_current_return_columns);
10791076
10801077 // step4: output columns
1081- // 4.1 output lazy_read_column_ids column
1082- _output_lazy_read_columns (block);
1078+ // 4.1 output non-predicate column
1079+ _output_non_pred_columns (block);
10831080
10841081 // 4.3 output short circuit and predicate column
10851082 // when lazy materialization enables, _first_read_column_ids = distinct(_short_cir_pred_column_ids + _vec_pred_column_ids)
0 commit comments