Skip to content

Commit 47a59c7

Browse files
authored
[fix](OlapScanner)fix bitmap or hll's OOM when loading too many unqualified data (apache#9205)
1 parent a20cf1e commit 47a59c7

3 files changed

Lines changed: 25 additions & 1 deletion

File tree

‎be/src/common/config.h‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -732,6 +732,10 @@ CONF_mInt32(string_type_length_soft_limit_bytes, "1048576");
732732
CONF_Validator(string_type_length_soft_limit_bytes,
733733
[](const int config) -> bool { return config > 0 && config <= 2147483643; });
734734

735+
// used for olap scanner to save memory, when the size of unused_object_pool
736+
// is greater than object_pool_buffer_size, release the object in the unused_object_pool.
737+
CONF_Int32(object_pool_buffer_size, "100");
738+
735739
} // namespace config
736740

737741
} // namespace doris

‎be/src/common/object_pool.h‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,11 @@ class ObjectPool {
6060
src->_objects.clear();
6161
}
6262

63+
uint64_t size() {
64+
std::lock_guard<SpinLock> l(_lock);
65+
return _objects.size();
66+
}
67+
6368
private:
6469
ObjectPool(const ObjectPool&) = delete;
6570
void operator=(const ObjectPool&) = delete;

‎be/src/exec/olap_scanner.cpp‎

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -287,6 +287,11 @@ Status OlapScanner::get_batch(RuntimeState* state, RowBatch* batch, bool* eof) {
287287
int64_t raw_bytes_threshold = config::doris_scanner_row_bytes;
288288
{
289289
SCOPED_TIMER(_parent->_scan_timer);
290+
// store the object which may can't pass the conjuncts temporarily.
291+
// otherwise, pushed all objects into agg_object_pool directly may lead to OOM.
292+
ObjectPool tmp_object_pool;
293+
// release the memory of the object which can't pass the conjuncts.
294+
ObjectPool unused_object_pool;
290295
while (true) {
291296
// Batch is full or reach raw_rows_threshold or raw_bytes_threshold, break
292297
if (batch->is_full() ||
@@ -295,9 +300,18 @@ Status OlapScanner::get_batch(RuntimeState* state, RowBatch* batch, bool* eof) {
295300
_update_realtime_counter();
296301
break;
297302
}
303+
304+
if (tmp_object_pool.size() > 0) {
305+
unused_object_pool.acquire_data(&tmp_object_pool);
306+
}
307+
308+
if (unused_object_pool.size() >= config::object_pool_buffer_size) {
309+
unused_object_pool.clear();
310+
}
311+
298312
// Read one row from reader
299313
auto res = _tablet_reader->next_row_with_aggregation(&_read_row_cursor, mem_pool.get(),
300-
batch->agg_object_pool(), eof);
314+
&tmp_object_pool, eof);
301315
if (!res.ok()) {
302316
std::stringstream ss;
303317
ss << "Internal Error: read storage fail. res=" << res
@@ -396,6 +410,7 @@ Status OlapScanner::get_batch(RuntimeState* state, RowBatch* batch, bool* eof) {
396410

397411
// check direct && pushdown conjuncts success then commit tuple
398412
batch->commit_last_row();
413+
batch->agg_object_pool()->acquire_data(&tmp_object_pool);
399414
char* new_tuple = reinterpret_cast<char*>(tuple);
400415
new_tuple += _tuple_desc->byte_size();
401416
tuple = reinterpret_cast<Tuple*>(new_tuple);

0 commit comments

Comments
 (0)