Skip to content

Commit 74b5ef0

Browse files
authored
[fix](read) remove logic of estimating count of rows to read in segment iterator to avoid wrong result of unique key (apache#29109) (apache#29110)
1 parent 3edac2f commit 74b5ef0

2 files changed

Lines changed: 0 additions & 29 deletions

File tree

‎be/src/olap/rowset/segment_v2/segment_iterator.cpp‎

Lines changed: 0 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -259,8 +259,6 @@ SegmentIterator::SegmentIterator(std::shared_ptr<Segment> segment, SchemaSPtr sc
259259
_lazy_materialization_read(false),
260260
_lazy_inited(false),
261261
_inited(false),
262-
_estimate_row_size(true),
263-
_wait_times_estimate_row_size(10),
264262
_pool(new ObjectPool) {}
265263

266264
Status SegmentIterator::init(const StorageReadOptions& opts) {
@@ -2034,13 +2032,6 @@ Status SegmentIterator::_next_batch_internal(vectorized::Block* block) {
20342032

20352033
_current_batch_rows_read = 0;
20362034
uint32_t nrows_read_limit = _opts.block_row_max;
2037-
if (_wait_times_estimate_row_size > 0) {
2038-
// first time, read 100 rows to estimate average row size, to avoid oom caused by a single batch being too large.
2039-
// If no valid data is read for the first time, block_row_max is read each time thereafter.
2040-
// Avoid low performance when valid data cannot be read all the time
2041-
nrows_read_limit = std::min(nrows_read_limit, (uint32_t)100);
2042-
_wait_times_estimate_row_size--;
2043-
}
20442035
RETURN_IF_ERROR(_read_columns_by_index(
20452036
nrows_read_limit, _current_batch_rows_read,
20462037
_lazy_materialization_read || _opts.record_rowids || _is_need_expr_eval));
@@ -2180,9 +2171,6 @@ Status SegmentIterator::_next_batch_internal(vectorized::Block* block) {
21802171
// shrink char_type suffix zero data
21812172
block->shrink_char_type_column_suffix_zero(_char_type_idx);
21822173

2183-
if (UNLIKELY(_estimate_row_size) && block->rows() > 0) {
2184-
_update_max_row(block);
2185-
}
21862174
return Status::OK();
21872175
}
21882176

@@ -2208,10 +2196,6 @@ Status SegmentIterator::_next_batch_internal(vectorized::Block* block) {
22082196
// shrink char_type suffix zero data
22092197
block->shrink_char_type_column_suffix_zero(_char_type_idx);
22102198

2211-
if (UNLIKELY(_estimate_row_size) && block->rows() > 0) {
2212-
_update_max_row(block);
2213-
}
2214-
22152199
// reverse block row order
22162200
if (_opts.read_orderby_key_reverse) {
22172201
size_t num_rows = block->rows();
@@ -2362,14 +2346,6 @@ void SegmentIterator::_convert_dict_code_for_predicate_if_necessary_impl(
23622346
}
23632347
}
23642348

2365-
void SegmentIterator::_update_max_row(const vectorized::Block* block) {
2366-
_estimate_row_size = false;
2367-
auto avg_row_size = block->bytes() / block->rows();
2368-
2369-
int block_row_max = config::doris_scan_block_max_mb / avg_row_size;
2370-
_opts.block_row_max = std::min(block_row_max, _opts.block_row_max);
2371-
}
2372-
23732349
Status SegmentIterator::current_block_row_locations(std::vector<RowLocation>* block_row_locations) {
23742350
DCHECK(_opts.record_rowids);
23752351
DCHECK_GE(_block_rowids.size(), _current_batch_rows_read);

‎be/src/olap/rowset/segment_v2/segment_iterator.h‎

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -251,8 +251,6 @@ class SegmentIterator : public RowwiseIterator {
251251

252252
void _convert_dict_code_for_predicate_if_necessary_impl(ColumnPredicate* predicate);
253253

254-
void _update_max_row(const vectorized::Block* block);
255-
256254
bool _check_apply_by_bitmap_index(ColumnPredicate* pred);
257255
bool _check_apply_by_inverted_index(ColumnPredicate* pred, bool pred_in_compound = false);
258256

@@ -385,9 +383,6 @@ class SegmentIterator : public RowwiseIterator {
385383
// the actual init process is delayed to the first call to next_batch()
386384
bool _lazy_inited;
387385
bool _inited;
388-
bool _estimate_row_size;
389-
// Read up to 100 rows at a time while waiting for the estimated row size.
390-
int _wait_times_estimate_row_size;
391386

392387
StorageReadOptions _opts;
393388
// make a copy of `_opts.column_predicates` in order to make local changes

0 commit comments

Comments
 (0)