Skip to content

Commit d90ac33

Browse files
authored
[Improvement](scan) Update scanner limit controller (#61617) (#61962)
pick from #61617
1 parent fe86029 commit d90ac33

6 files changed

Lines changed: 96 additions & 34 deletions

File tree

‎be/src/exec/operator/scan_operator.cpp‎

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1026,14 +1026,14 @@ template <typename Derived>
10261026
Status ScanLocalState<Derived>::_start_scanners(
10271027
const std::list<std::shared_ptr<ScannerDelegate>>& scanners) {
10281028
auto& p = _parent->cast<typename Derived::Parent>();
1029-
_scanner_ctx = ScannerContext::create_shared(state(), this, p._output_tuple_desc,
1030-
p.output_row_descriptor(), scanners, p.limit(),
1031-
_scan_dependency
1029+
_scanner_ctx.store(ScannerContext::create_shared(state(), this, p._output_tuple_desc,
1030+
p.output_row_descriptor(), scanners, p.limit(),
1031+
_scan_dependency, &p._shared_scan_limit
10321032
#ifdef BE_TEST
1033-
,
1034-
max_scanners_concurrency(state())
1033+
,
1034+
max_scanners_concurrency(state())
10351035
#endif
1036-
);
1036+
));
10371037
return Status::OK();
10381038
}
10391039

@@ -1180,6 +1180,7 @@ ScanOperatorX<LocalStateType>::ScanOperatorX(ObjectPool* pool, const TPlanNode&
11801180
if (tnode.__isset.push_down_count) {
11811181
_push_down_count = tnode.push_down_count;
11821182
}
1183+
_shared_scan_limit.store(this->_limit, std::memory_order_relaxed);
11831184
}
11841185

11851186
template <typename LocalStateType>

‎be/src/exec/operator/scan_operator.h‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -417,6 +417,10 @@ class ScanOperatorX : public OperatorX<LocalStateType> {
417417
// If sort info is set, push limit to each scanner;
418418
int64_t _limit_per_scanner = -1;
419419

420+
// Shared remaining limit across all parallel instances and their scanners.
421+
// Initialized to _limit (SQL LIMIT); -1 means no limit.
422+
std::atomic<int64_t> _shared_scan_limit {-1};
423+
420424
std::vector<TRuntimeFilterDesc> _runtime_filter_descs;
421425

422426
TPushAggOp::type _push_down_agg_type;

‎be/src/exec/scan/scanner_context.cpp‎

Lines changed: 39 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,8 @@ ScannerContext::ScannerContext(RuntimeState* state, ScanLocalStateBase* local_st
5656
const TupleDescriptor* output_tuple_desc,
5757
const RowDescriptor* output_row_descriptor,
5858
const std::list<std::shared_ptr<ScannerDelegate>>& scanners,
59-
int64_t limit_, std::shared_ptr<Dependency> dependency
59+
int64_t limit_, std::shared_ptr<Dependency> dependency,
60+
std::atomic<int64_t>* shared_scan_limit
6061
#ifdef BE_TEST
6162
,
6263
int num_parallel_instances
@@ -71,6 +72,7 @@ ScannerContext::ScannerContext(RuntimeState* state, ScanLocalStateBase* local_st
7172
_output_row_descriptor(output_row_descriptor),
7273
_batch_size(state->batch_size()),
7374
limit(limit_),
75+
_shared_scan_limit(shared_scan_limit),
7476
_all_scanners(scanners.begin(), scanners.end()),
7577
#ifndef BE_TEST
7678
_scanner_scheduler(local_state->scan_scheduler(state)),
@@ -100,6 +102,27 @@ ScannerContext::ScannerContext(RuntimeState* state, ScanLocalStateBase* local_st
100102
DorisMetrics::instance()->scanner_ctx_cnt->increment(1);
101103
}
102104

105+
int64_t ScannerContext::acquire_limit_quota(int64_t desired) {
106+
DCHECK(desired > 0);
107+
int64_t remaining = _shared_scan_limit->load(std::memory_order_acquire);
108+
while (true) {
109+
if (remaining < 0) {
110+
// No limit set, grant all desired rows.
111+
return desired;
112+
}
113+
if (remaining == 0) {
114+
return 0;
115+
}
116+
int64_t granted = std::min(desired, remaining);
117+
if (_shared_scan_limit->compare_exchange_weak(remaining, remaining - granted,
118+
std::memory_order_acq_rel,
119+
std::memory_order_acquire)) {
120+
return granted;
121+
}
122+
// CAS failed, `remaining` is updated to current value, retry.
123+
}
124+
}
125+
103126
// After init function call, should not access _parent
104127
Status ScannerContext::init() {
105128
#ifndef BE_TEST
@@ -325,7 +348,12 @@ Status ScannerContext::get_block_from_queue(RuntimeState* state, Block* block, b
325348
}
326349
}
327350

328-
if (_num_finished_scanners == _all_scanners.size() && _tasks_queue.empty()) {
351+
// Mark finished when either:
352+
// (1) all scanners completed normally, or
353+
// (2) shared limit exhausted and no scanners are still running.
354+
if (_tasks_queue.empty() && (_num_finished_scanners == _all_scanners.size() ||
355+
(_shared_scan_limit->load(std::memory_order_acquire) == 0 &&
356+
_num_scheduled_scanners == 0))) {
329357
_set_scanner_done();
330358
_is_finished = true;
331359
}
@@ -434,11 +462,12 @@ std::string ScannerContext::debug_string() {
434462
return fmt::format(
435463
"id: {}, total scanners: {}, pending tasks: {},"
436464
" _should_stop: {}, _is_finished: {}, free blocks: {},"
437-
" limit: {}, _num_running_scanners: {}, _max_thread_num: {},"
465+
" limit: {}, remaining_limit: {}, _num_running_scanners: {}, _max_thread_num: {},"
438466
" _max_bytes_in_queue: {}, query_id: {}",
439467
ctx_id, _all_scanners.size(), _tasks_queue.size(), _should_stop, _is_finished,
440-
_free_blocks.size_approx(), limit, _num_scheduled_scanners, _max_scan_concurrency,
441-
_max_bytes_in_queue, print_id(_query_id));
468+
_free_blocks.size_approx(), limit, _shared_scan_limit->load(std::memory_order_relaxed),
469+
_num_scheduled_scanners, _max_scan_concurrency, _max_bytes_in_queue,
470+
print_id(_query_id));
442471
}
443472

444473
void ScannerContext::_set_scanner_done() {
@@ -601,6 +630,11 @@ std::shared_ptr<ScanTask> ScannerContext::_pull_next_scan_task(
601630
}
602631

603632
if (!_pending_scanners.empty()) {
633+
// If shared limit quota is exhausted, do not submit new scanners from pending queue.
634+
int64_t remaining = _shared_scan_limit->load(std::memory_order_acquire);
635+
if (remaining == 0) {
636+
return nullptr;
637+
}
604638
std::shared_ptr<ScanTask> next_scan_task;
605639
next_scan_task = _pending_scanners.top();
606640
_pending_scanners.pop();

‎be/src/exec/scan/scanner_context.h‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -115,7 +115,7 @@ class ScannerContext : public std::enable_shared_from_this<ScannerContext>,
115115
const TupleDescriptor* output_tuple_desc,
116116
const RowDescriptor* output_row_descriptor,
117117
const std::list<std::shared_ptr<ScannerDelegate>>& scanners, int64_t limit_,
118-
std::shared_ptr<Dependency> dependency
118+
std::shared_ptr<Dependency> dependency, std::atomic<int64_t>* shared_scan_limit
119119
#ifdef BE_TEST
120120
,
121121
int num_parallel_instances
@@ -221,6 +221,12 @@ class ScannerContext : public std::enable_shared_from_this<ScannerContext>,
221221
int _batch_size;
222222
// The limit from SQL's limit clause
223223
int64_t limit;
224+
// Points to the shared remaining limit on ScanOperatorX, shared across all
225+
// parallel instances and their scanners. -1 means no limit.
226+
std::atomic<int64_t>* _shared_scan_limit = nullptr;
227+
// Atomically acquire up to `desired` rows. Returns actual granted count (0 = exhausted).
228+
int64_t acquire_limit_quota(int64_t desired);
229+
int64_t remaining_limit() const { return _shared_scan_limit->load(std::memory_order_acquire); }
224230

225231
int64_t _max_bytes_in_queue = 0;
226232
// Using stack so that we can resubmit scanner in a LIFO order, maybe more cache friendly

‎be/src/exec/scan/scanner_scheduler.cpp‎

Lines changed: 24 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -232,6 +232,11 @@ void ScannerScheduler::_scanner_scan(std::shared_ptr<ScannerContext> ctx,
232232
eos = true;
233233
break;
234234
}
235+
// If shared limit quota is exhausted, stop scanning.
236+
if (ctx->remaining_limit() == 0) {
237+
eos = true;
238+
break;
239+
}
235240
if (max_run_time_watch.elapsed_time() >
236241
config::doris_scanner_max_run_time_ms * 1e6) {
237242
break;
@@ -268,6 +273,23 @@ void ScannerScheduler::_scanner_scan(std::shared_ptr<ScannerContext> ctx,
268273
// Check column type only after block is read successfully.
269274
// Or it may cause a crash when the block is not normal.
270275
_make_sure_virtual_col_is_materialized(scanner, free_block.get());
276+
277+
// Shared limit quota: acquire rows from the context's shared pool.
278+
// Discard or truncate the block if quota is exhausted.
279+
if (free_block->rows() > 0) {
280+
int64_t block_rows = free_block->rows();
281+
int64_t granted = ctx->acquire_limit_quota(block_rows);
282+
if (granted == 0) {
283+
// No quota remaining, discard this block and mark eos.
284+
ctx->return_free_block(std::move(free_block));
285+
eos = true;
286+
break;
287+
} else if (granted < block_rows) {
288+
// Partial quota: truncate block to granted rows and mark eos.
289+
free_block->set_num_rows(granted);
290+
eos = true;
291+
}
292+
}
271293
// Projection will truncate useless columns, makes block size change.
272294
auto free_block_bytes = free_block->allocated_bytes();
273295
raw_bytes_read += free_block_bytes;
@@ -298,15 +320,9 @@ void ScannerScheduler::_scanner_scan(std::shared_ptr<ScannerContext> ctx,
298320
scan_task->cached_blocks.emplace_back(std::move(free_block), free_block_bytes);
299321
}
300322

323+
// Per-scanner small-limit optimization: if limit is small (< batch_size),
324+
// return immediately instead of accumulating to raw_bytes_threshold.
301325
if (limit > 0 && limit < ctx->batch_size()) {
302-
// If this scanner has limit, and less than batch size,
303-
// return immediately and no need to wait raw_bytes_threshold.
304-
// This can save time that each scanner may only return a small number of rows,
305-
// but rows are enough from all scanners.
306-
// If not break, the query like "select * from tbl where id=1 limit 10"
307-
// may scan a lot data when the "id=1"'s filter ratio is high.
308-
// If limit is larger than batch size, this rule is skipped,
309-
// to avoid user specify a large limit and causing too much small blocks.
310326
break;
311327
}
312328

‎be/test/exec/scan/scanner_context_test.cpp‎

Lines changed: 15 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -122,6 +122,7 @@ class ScannerContextTest : public testing::Test {
122122
std::shared_ptr<CgroupCpuCtl> cgroup_cpu_ctl = std::make_shared<CgroupV2CpuCtl>(1);
123123
std::unique_ptr<ScannerScheduler> scan_scheduler =
124124
std::make_unique<ThreadPoolSimplifiedScanScheduler>("ForTest", cgroup_cpu_ctl);
125+
std::atomic<int64_t> shared_limit {-1};
125126
};
126127

127128
TEST_F(ScannerContextTest, test_init) {
@@ -150,7 +151,7 @@ TEST_F(ScannerContextTest, test_init) {
150151

151152
std::shared_ptr<ScannerContext> scanner_context = ScannerContext::create_shared(
152153
state.get(), olap_scan_local_state.get(), output_tuple_desc, output_row_descriptor,
153-
scanners, limit, scan_dependency, parallel_tasks);
154+
scanners, limit, scan_dependency, &shared_limit, parallel_tasks);
154155

155156
scan_operator->_should_run_serial = false;
156157

@@ -210,7 +211,7 @@ TEST_F(ScannerContextTest, test_serial_run) {
210211

211212
std::shared_ptr<ScannerContext> scanner_context = ScannerContext::create_shared(
212213
state.get(), olap_scan_local_state.get(), output_tuple_desc, output_row_descriptor,
213-
scanners, limit, scan_dependency, parallel_tasks);
214+
scanners, limit, scan_dependency, &shared_limit, parallel_tasks);
214215

215216
scan_operator->_should_run_serial = true;
216217

@@ -268,7 +269,7 @@ TEST_F(ScannerContextTest, test_max_column_reader_num) {
268269

269270
std::shared_ptr<ScannerContext> scanner_context = ScannerContext::create_shared(
270271
state.get(), olap_scan_local_state.get(), output_tuple_desc, output_row_descriptor,
271-
scanners, limit, scan_dependency, parallel_tasks);
272+
scanners, limit, scan_dependency, &shared_limit, parallel_tasks);
272273

273274
scan_operator->_should_run_serial = false;
274275

@@ -318,7 +319,7 @@ TEST_F(ScannerContextTest, test_push_back_scan_task) {
318319

319320
std::shared_ptr<ScannerContext> scanner_context = ScannerContext::create_shared(
320321
state.get(), olap_scan_local_state.get(), output_tuple_desc, output_row_descriptor,
321-
scanners, limit, scan_dependency, parallel_tasks);
322+
scanners, limit, scan_dependency, &shared_limit, parallel_tasks);
322323

323324
scanner_context->_num_scheduled_scanners = 11;
324325

@@ -355,7 +356,7 @@ TEST_F(ScannerContextTest, get_margin) {
355356

356357
std::shared_ptr<ScannerContext> scanner_context = ScannerContext::create_shared(
357358
state.get(), olap_scan_local_state.get(), output_tuple_desc, output_row_descriptor,
358-
scanners, limit, scan_dependency, parallel_tasks);
359+
scanners, limit, scan_dependency, &shared_limit, parallel_tasks);
359360

360361
std::mutex transfer_mutex;
361362
std::unique_lock<std::mutex> transfer_lock(transfer_mutex);
@@ -451,7 +452,7 @@ TEST_F(ScannerContextTest, pull_next_scan_task) {
451452

452453
std::shared_ptr<ScannerContext> scanner_context = ScannerContext::create_shared(
453454
state.get(), olap_scan_local_state.get(), output_tuple_desc, output_row_descriptor,
454-
scanners, limit, scan_dependency, parallel_tasks);
455+
scanners, limit, scan_dependency, &shared_limit, parallel_tasks);
455456

456457
std::mutex transfer_mutex;
457458
std::unique_lock<std::mutex> transfer_lock(transfer_mutex);
@@ -527,7 +528,7 @@ TEST_F(ScannerContextTest, schedule_scan_task) {
527528

528529
std::shared_ptr<ScannerContext> scanner_context = ScannerContext::create_shared(
529530
state.get(), olap_scan_local_state.get(), output_tuple_desc, output_row_descriptor,
530-
scanners, limit, scan_dependency, parallel_tasks);
531+
scanners, limit, scan_dependency, &shared_limit, parallel_tasks);
531532

532533
std::mutex transfer_mutex;
533534
std::unique_lock<std::mutex> transfer_lock(transfer_mutex);
@@ -559,7 +560,7 @@ TEST_F(ScannerContextTest, schedule_scan_task) {
559560

560561
scanner_context = ScannerContext::create_shared(
561562
state.get(), olap_scan_local_state.get(), output_tuple_desc, output_row_descriptor,
562-
scanners, limit, scan_dependency, parallel_tasks);
563+
scanners, limit, scan_dependency, &shared_limit, parallel_tasks);
563564

564565
scanner_context->_scanner_scheduler = scheduler.get();
565566

@@ -580,7 +581,7 @@ TEST_F(ScannerContextTest, schedule_scan_task) {
580581

581582
scanner_context = ScannerContext::create_shared(
582583
state.get(), olap_scan_local_state.get(), output_tuple_desc, output_row_descriptor,
583-
scanners, limit, scan_dependency, parallel_tasks);
584+
scanners, limit, scan_dependency, &shared_limit, parallel_tasks);
584585

585586
scanner_context->_scanner_scheduler = scheduler.get();
586587

@@ -596,7 +597,7 @@ TEST_F(ScannerContextTest, schedule_scan_task) {
596597

597598
scanner_context = ScannerContext::create_shared(
598599
state.get(), olap_scan_local_state.get(), output_tuple_desc, output_row_descriptor,
599-
scanners, limit, scan_dependency, parallel_tasks);
600+
scanners, limit, scan_dependency, &shared_limit, parallel_tasks);
600601

601602
scanner_context->_scanner_scheduler = scheduler.get();
602603

@@ -649,7 +650,7 @@ TEST_F(ScannerContextTest, scan_queue_mem_limit) {
649650

650651
std::shared_ptr<ScannerContext> scanner_context = ScannerContext::create_shared(
651652
state.get(), olap_scan_local_state.get(), output_tuple_desc, output_row_descriptor,
652-
scanners, limit, scan_dependency, parallel_tasks);
653+
scanners, limit, scan_dependency, &shared_limit, parallel_tasks);
653654

654655
std::unique_ptr<MockSimplifiedScanScheduler> scheduler =
655656
std::make_unique<MockSimplifiedScanScheduler>(cgroup_cpu_ctl);
@@ -689,7 +690,7 @@ TEST_F(ScannerContextTest, get_free_block) {
689690

690691
std::shared_ptr<ScannerContext> scanner_context = ScannerContext::create_shared(
691692
state.get(), olap_scan_local_state.get(), output_tuple_desc, output_row_descriptor,
692-
scanners, limit, scan_dependency, parallel_tasks);
693+
scanners, limit, scan_dependency, &shared_limit, parallel_tasks);
693694
scanner_context->_newly_create_free_blocks_num = newly_create_free_blocks_num.get();
694695
scanner_context->_newly_create_free_blocks_num->set(0L);
695696
scanner_context->_scanner_memory_used_counter = scanner_memory_used_counter.get();
@@ -742,7 +743,7 @@ TEST_F(ScannerContextTest, return_free_block) {
742743

743744
std::shared_ptr<ScannerContext> scanner_context = ScannerContext::create_shared(
744745
state.get(), olap_scan_local_state.get(), output_tuple_desc, output_row_descriptor,
745-
scanners, limit, scan_dependency, parallel_tasks);
746+
scanners, limit, scan_dependency, &shared_limit, parallel_tasks);
746747
scanner_context->_newly_create_free_blocks_num = newly_create_free_blocks_num.get();
747748
scanner_context->_scanner_memory_used_counter = scanner_memory_used_counter.get();
748749
scanner_context->_max_bytes_in_queue = 200;
@@ -786,7 +787,7 @@ TEST_F(ScannerContextTest, get_block_from_queue) {
786787

787788
std::shared_ptr<ScannerContext> scanner_context = ScannerContext::create_shared(
788789
state.get(), olap_scan_local_state.get(), output_tuple_desc, output_row_descriptor,
789-
scanners, limit, scan_dependency, parallel_tasks);
790+
scanners, limit, scan_dependency, &shared_limit, parallel_tasks);
790791
scanner_context->_newly_create_free_blocks_num = newly_create_free_blocks_num.get();
791792
scanner_context->_scanner_memory_used_counter = scanner_memory_used_counter.get();
792793
scanner_context->_max_bytes_in_queue = 200;

0 commit comments

Comments
 (0)