Skip to content

Commit 3e5c8d9

Browse files
authored
[fix](read) remove logic of estimating count of rows to read in segment iterator to avoid wrong result of unique key. (apache#29109)
1 parent 17f9215 commit 3e5c8d9

2 files changed

Lines changed: 0 additions & 30 deletions

File tree

‎be/src/olap/rowset/segment_v2/segment_iterator.cpp‎

Lines changed: 0 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -263,8 +263,6 @@ SegmentIterator::SegmentIterator(std::shared_ptr<Segment> segment, SchemaSPtr sc
263263
_lazy_materialization_read(false),
264264
_lazy_inited(false),
265265
_inited(false),
266-
_estimate_row_size(true),
267-
_wait_times_estimate_row_size(10),
268266
_pool(new ObjectPool) {}
269267

270268
Status SegmentIterator::init(const StorageReadOptions& opts) {
@@ -2167,13 +2165,6 @@ Status SegmentIterator::_next_batch_internal(vectorized::Block* block) {
21672165

21682166
_current_batch_rows_read = 0;
21692167
uint32_t nrows_read_limit = _opts.block_row_max;
2170-
if (_wait_times_estimate_row_size > 0) {
2171-
// first time, read 100 rows to estimate average row size, to avoid oom caused by a single batch being too large.
2172-
// If no valid data is read for the first time, block_row_max is read each time thereafter.
2173-
// Avoid low performance when valid data cannot be read all the time
2174-
nrows_read_limit = std::min(nrows_read_limit, (uint32_t)100);
2175-
_wait_times_estimate_row_size--;
2176-
}
21772168
RETURN_IF_ERROR(_read_columns_by_index(
21782169
nrows_read_limit, _current_batch_rows_read,
21792170
_lazy_materialization_read || _opts.record_rowids || _is_need_expr_eval));
@@ -2320,9 +2311,6 @@ Status SegmentIterator::_next_batch_internal(vectorized::Block* block) {
23202311
// shrink char_type suffix zero data
23212312
block->shrink_char_type_column_suffix_zero(_char_type_idx);
23222313

2323-
if (UNLIKELY(_estimate_row_size) && block->rows() > 0) {
2324-
_update_max_row(block);
2325-
}
23262314
return Status::OK();
23272315
}
23282316
// step4: read non_predicate column
@@ -2358,10 +2346,6 @@ Status SegmentIterator::_next_batch_internal(vectorized::Block* block) {
23582346
}
23592347
#endif
23602348

2361-
if (UNLIKELY(_estimate_row_size) && block->rows() > 0) {
2362-
_update_max_row(block);
2363-
}
2364-
23652349
// reverse block row order
23662350
if (_opts.read_orderby_key_reverse) {
23672351
size_t num_rows = block->rows();
@@ -2515,15 +2499,6 @@ void SegmentIterator::_convert_dict_code_for_predicate_if_necessary_impl(
25152499
}
25162500
}
25172501

2518-
void SegmentIterator::_update_max_row(const vectorized::Block* block) {
2519-
_estimate_row_size = false;
2520-
auto avg_row_size = block->bytes() / block->rows();
2521-
if (avg_row_size > 0) {
2522-
int block_row_max = config::doris_scan_block_max_mb / avg_row_size;
2523-
_opts.block_row_max = std::min(block_row_max, _opts.block_row_max);
2524-
}
2525-
}
2526-
25272502
Status SegmentIterator::current_block_row_locations(std::vector<RowLocation>* block_row_locations) {
25282503
DCHECK(_opts.record_rowids);
25292504
DCHECK_GE(_block_rowids.size(), _current_batch_rows_read);

‎be/src/olap/rowset/segment_v2/segment_iterator.h‎

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -286,8 +286,6 @@ class SegmentIterator : public RowwiseIterator {
286286

287287
void _convert_dict_code_for_predicate_if_necessary_impl(ColumnPredicate* predicate);
288288

289-
void _update_max_row(const vectorized::Block* block);
290-
291289
bool _check_apply_by_bitmap_index(ColumnPredicate* pred);
292290
bool _check_apply_by_inverted_index(ColumnPredicate* pred, bool pred_in_compound = false);
293291

@@ -442,9 +440,6 @@ class SegmentIterator : public RowwiseIterator {
442440
// the actual init process is delayed to the first call to next_batch()
443441
bool _lazy_inited;
444442
bool _inited;
445-
bool _estimate_row_size;
446-
// Read up to 100 rows at a time while waiting for the estimated row size.
447-
int _wait_times_estimate_row_size;
448443

449444
StorageReadOptions _opts;
450445
// make a copy of `_opts.column_predicates` in order to make local changes

0 commit comments

Comments
 (0)