Skip to content

Commit 0491437

Browse files
authored
[Opt](scanner-scheduler) Optimize BlockingQueue, BlockingPriorityQueue and change remote scan thread pool. (apache#26784)
## Proposed changes - Optimize `BlockingQueue`, `BlockingPriorityQueue` by swapping `notify` and `unlock` to reduce lock competition. Ref: https://www.boost.org/doc/libs/1_54_0/boost/thread/sync_bounded_queue.hpp - Change remote scan thread pool to `PriorityQueue`. ### Test result Before: ``` mysql> select sum(lo_partkey) from lineorder; +-----------------+ | sum(lo_partkey) | +-----------------+ | 300021444265405 | +-----------------+ 1 row in set (1.11 sec) ``` After: ``` mysql> select sum(lo_partkey) from lineorder; +-----------------+ | sum(lo_partkey) | +-----------------+ | 300021444265405 | +-----------------+ 1 row in set (0.80 sec) ```
1 parent c1c8376 commit 0491437

4 files changed

Lines changed: 82 additions & 32 deletions

File tree

‎be/src/util/blocking_priority_queue.hpp‎

Lines changed: 43 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,9 @@ class BlockingPriorityQueue {
4242
_max_element(max_elements),
4343
_upgrade_counter(0),
4444
_total_get_wait_time(0),
45-
_total_put_wait_time(0) {}
45+
_total_put_wait_time(0),
46+
_get_waiting(0),
47+
_put_waiting(0) {}
4648

4749
// Get an element from the queue, waiting indefinitely (or until timeout) for one to become available.
4850
// Returns false if we were shut down prior to getting the element, and there
@@ -55,17 +57,29 @@ class BlockingPriorityQueue {
5557
bool wait_successful = false;
5658
#if !defined(USE_BTHREAD_SCANNER)
5759
if (timeout_ms > 0) {
58-
wait_successful = _get_cv.wait_for(unique_lock, std::chrono::milliseconds(timeout_ms),
59-
[this] { return _shutdown || !_queue.empty(); });
60+
while (!(_shutdown || !_queue.empty())) {
61+
++_get_waiting;
62+
if (_get_cv.wait_for(unique_lock, std::chrono::milliseconds(timeout_ms)) ==
63+
std::cv_status::timeout) {
64+
// timeout
65+
wait_successful = _shutdown || !_queue.empty();
66+
break;
67+
}
68+
}
6069
} else {
61-
_get_cv.wait(unique_lock, [this] { return _shutdown || !_queue.empty(); });
70+
while (!(_shutdown || !_queue.empty())) {
71+
++_get_waiting;
72+
_get_cv.wait(unique_lock);
73+
}
6274
wait_successful = true;
6375
}
6476
#else
6577
if (timeout_ms > 0) {
6678
wait_successful = true;
6779
while (!(_shutdown || !_queue.empty())) {
68-
if (_get_cv.wait_for(unique_lock, timeout_ms * 1000) != 0) {
80+
++_get_waiting;
81+
if (_get_cv.wait_for(unique_lock, std::chrono::milliseconds(timeout_ms)) ==
82+
std::cv_status::timeout) {
6983
// timeout
7084
wait_successful = _shutdown || !_queue.empty();
7185
break;
@@ -95,7 +109,11 @@ class BlockingPriorityQueue {
95109
*out = _queue.top();
96110
_queue.pop();
97111
++_upgrade_counter;
98-
_put_cv.notify_one();
112+
if (_put_waiting > 0) {
113+
--_put_waiting;
114+
unique_lock.unlock();
115+
_put_cv.notify_one();
116+
}
99117
return true;
100118
} else {
101119
assert(_shutdown);
@@ -131,7 +149,11 @@ class BlockingPriorityQueue {
131149
_queue.pop();
132150
++_upgrade_counter;
133151
_total_get_wait_time += timer.elapsed_time();
134-
_put_cv.notify_one();
152+
if (_put_waiting > 0) {
153+
--_put_waiting;
154+
unique_lock.unlock();
155+
_put_cv.notify_one();
156+
}
135157
return true;
136158
}
137159

@@ -145,6 +167,7 @@ class BlockingPriorityQueue {
145167
timer.start();
146168
std::unique_lock unique_lock(_lock);
147169
while (!(_shutdown || _queue.size() < _max_element)) {
170+
++_put_waiting;
148171
_put_cv.wait(unique_lock);
149172
}
150173
_total_put_wait_time += timer.elapsed_time();
@@ -154,7 +177,11 @@ class BlockingPriorityQueue {
154177
}
155178

156179
_queue.push(val);
157-
_get_cv.notify_one();
180+
if (_get_waiting > 0) {
181+
--_get_waiting;
182+
unique_lock.unlock();
183+
_get_cv.notify_one();
184+
}
158185
return true;
159186
}
160187

@@ -163,8 +190,11 @@ class BlockingPriorityQueue {
163190
std::unique_lock unique_lock(_lock);
164191
if (_queue.size() < _max_element && !_shutdown) {
165192
_queue.push(val);
166-
unique_lock.unlock();
167-
_get_cv.notify_one();
193+
if (_get_waiting > 0) {
194+
--_get_waiting;
195+
unique_lock.unlock();
196+
_get_cv.notify_one();
197+
}
168198
return true;
169199
}
170200
return false;
@@ -204,6 +234,8 @@ class BlockingPriorityQueue {
204234
int _upgrade_counter;
205235
std::atomic<uint64_t> _total_get_wait_time;
206236
std::atomic<uint64_t> _total_put_wait_time;
237+
size_t _get_waiting;
238+
size_t _put_waiting;
207239
};
208240

209-
} // namespace doris
241+
} // namespace doris

‎be/src/util/blocking_queue.hpp‎

Lines changed: 28 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,9 @@ class BlockingQueue {
4141
: _shutdown(false),
4242
_max_elements(max_elements),
4343
_total_get_wait_time(0),
44-
_total_put_wait_time(0) {}
44+
_total_put_wait_time(0),
45+
_get_waiting(0),
46+
_put_waiting(0) {}
4547

4648
// Get an element from the queue, waiting indefinitely for one to become available.
4749
// Returns false if we were shut down prior to getting the element, and there
@@ -50,13 +52,20 @@ class BlockingQueue {
5052
MonotonicStopWatch timer;
5153
timer.start();
5254
std::unique_lock<std::mutex> unique_lock(_lock);
53-
_get_cv.wait(unique_lock, [this] { return _shutdown || !_list.empty(); });
55+
while (!(_shutdown || !_list.empty())) {
56+
++_get_waiting;
57+
_get_cv.wait(unique_lock);
58+
}
5459
_total_get_wait_time += timer.elapsed_time();
5560

5661
if (!_list.empty()) {
5762
*out = _list.front();
5863
_list.pop_front();
59-
_put_cv.notify_one();
64+
if (_put_waiting > 0) {
65+
--_put_waiting;
66+
unique_lock.unlock();
67+
_put_cv.notify_one();
68+
}
6069
return true;
6170
} else {
6271
assert(_shutdown);
@@ -70,15 +79,22 @@ class BlockingQueue {
7079
MonotonicStopWatch timer;
7180
timer.start();
7281
std::unique_lock<std::mutex> unique_lock(_lock);
73-
_put_cv.wait(unique_lock, [this] { return _shutdown || _list.size() < _max_elements; });
82+
while (!(_shutdown || _list.size() < _max_elements)) {
83+
++_put_waiting;
84+
_put_cv.wait(unique_lock);
85+
}
7486
_total_put_wait_time += timer.elapsed_time();
7587

7688
if (_shutdown) {
7789
return false;
7890
}
7991

8092
_list.push_back(val);
81-
_get_cv.notify_one();
93+
if (_get_waiting > 0) {
94+
--_get_waiting;
95+
unique_lock.unlock();
96+
_get_cv.notify_one();
97+
}
8298
return true;
8399
}
84100

@@ -98,7 +114,11 @@ class BlockingQueue {
98114
}
99115

100116
_list.push_back(val);
101-
_get_cv.notify_one();
117+
if (_get_waiting > 0) {
118+
--_get_waiting;
119+
unique_lock.unlock();
120+
_get_cv.notify_one();
121+
}
102122
return true;
103123
}
104124

@@ -136,6 +156,8 @@ class BlockingQueue {
136156
std::list<T> _list;
137157
std::atomic<uint64_t> _total_get_wait_time;
138158
std::atomic<uint64_t> _total_put_wait_time;
159+
size_t _get_waiting;
160+
size_t _put_waiting;
139161
};
140162

141163
} // namespace doris

‎be/src/vec/exec/scan/scanner_scheduler.cpp‎

Lines changed: 10 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -97,7 +97,7 @@ void ScannerScheduler::stop() {
9797

9898
_scheduler_pool->wait();
9999
_local_scan_thread_pool->join();
100-
_remote_scan_thread_pool->wait();
100+
_remote_scan_thread_pool->join();
101101
_limited_scan_thread_pool->wait();
102102
_group_local_scan_thread_pool->wait();
103103

@@ -123,15 +123,9 @@ Status ScannerScheduler::init(ExecEnv* env) {
123123
config::doris_scanner_thread_pool_queue_size, "local_scan");
124124

125125
// 3. remote scan thread pool
126-
_remote_thread_pool_max_size = config::doris_max_remote_scanner_thread_pool_thread_num != -1
127-
? config::doris_max_remote_scanner_thread_pool_thread_num
128-
: std::max(512, CpuInfo::num_cores() * 10);
129-
static_cast<void>(
130-
ThreadPoolBuilder("RemoteScanThreadPool")
131-
.set_min_threads(config::doris_scanner_thread_pool_thread_num) // 48 default
132-
.set_max_threads(_remote_thread_pool_max_size)
133-
.set_max_queue_size(config::doris_scanner_thread_pool_queue_size)
134-
.build(&_remote_scan_thread_pool));
126+
_remote_scan_thread_pool = std::make_unique<PriorityThreadPool>(
127+
config::doris_remote_scanner_thread_pool_thread_num,
128+
config::doris_remote_scanner_thread_pool_queue_size, "RemoteScanThreadPool");
135129

136130
// 4. limited scan thread pool
137131
static_cast<void>(ThreadPoolBuilder("LimitedScanThreadPool")
@@ -273,9 +267,12 @@ void ScannerScheduler::_schedule_scanners(ScannerContext* ctx) {
273267
ret = _local_scan_thread_pool->offer(task);
274268
}
275269
} else {
276-
ret = _remote_scan_thread_pool->submit_func([this, scanner = *iter, ctx] {
270+
PriorityThreadPool::Task task;
271+
task.work_function = [this, scanner = *iter, ctx] {
277272
this->_scanner_scan(this, ctx, scanner);
278-
});
273+
};
274+
task.priority = nice;
275+
ret = _remote_scan_thread_pool->offer(task);
279276
}
280277
if (ret) {
281278
this_run.erase(iter++);
@@ -486,7 +483,7 @@ void ScannerScheduler::_register_metrics() {
486483
REGISTER_HOOK_METRIC(remote_scan_thread_pool_queue_size,
487484
[this]() { return _remote_scan_thread_pool->get_queue_size(); });
488485
REGISTER_HOOK_METRIC(remote_scan_thread_pool_thread_num,
489-
[this]() { return _remote_scan_thread_pool->num_threads(); });
486+
[this]() { return _remote_scan_thread_pool->get_active_threads(); });
490487
REGISTER_HOOK_METRIC(limited_scan_thread_pool_queue_size,
491488
[this]() { return _limited_scan_thread_pool->get_queue_size(); });
492489
REGISTER_HOOK_METRIC(limited_scan_thread_pool_thread_num,
@@ -507,5 +504,4 @@ void ScannerScheduler::_deregister_metrics() {
507504
DEREGISTER_HOOK_METRIC(group_local_scan_thread_pool_queue_size);
508505
DEREGISTER_HOOK_METRIC(group_local_scan_thread_pool_thread_num);
509506
}
510-
511507
} // namespace doris::vectorized

‎be/src/vec/exec/scan/scanner_scheduler.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -111,7 +111,7 @@ class ScannerScheduler {
111111
// _remote_scan_thread_pool is for remote scan task(cold data on s3, hdfs, etc.)
112112
// _limited_scan_thread_pool is a special pool for queries with resource limit
113113
std::unique_ptr<PriorityThreadPool> _local_scan_thread_pool;
114-
std::unique_ptr<ThreadPool> _remote_scan_thread_pool;
114+
std::unique_ptr<PriorityThreadPool> _remote_scan_thread_pool;
115115
std::unique_ptr<ThreadPool> _limited_scan_thread_pool;
116116

117117
std::unique_ptr<taskgroup::ScanTaskTaskGroupQueue> _task_group_local_scan_queue;

0 commit comments

Comments
 (0)