Skip to content

Commit d783450

Browse files
committed
[fix](be) Serialize thread-pool scan budget check and bound wait credit
- Hold a scheduler-wide lock from the pool budget check in ThreadPoolSimplifiedScanScheduler::schedule_scan_task() through the Context submission. Otherwise two Contexts can both see the last free pool slot and the loser fails its query with TOO_MANY_TASKS instead of deferring, which the removed scheduler lock used to prevent. - Read the pool queue size before its active thread count when admitting. The two reads take the pool lock separately while a worker moves a task from queued to active atomically, so this order makes a torn read over-count (defer) instead of under-count (overbook the last slot). - Credit a Context runnable's queue wait to the admitted scanner only for the part during which that scanner was pending. With LIFO re-admission the runnable can be queued before the scanner's previous attempt ran, and crediting the whole queue age counted that execution as ScannerWorkerWaitTime.
1 parent 1b3cae9 commit d783450

5 files changed

Lines changed: 207 additions & 18 deletions

File tree

‎be/src/exec/scan/scanner_context.cpp‎

Lines changed: 17 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
#include <glog/logging.h>
2323
#include <zconf.h>
2424

25+
#include <algorithm>
2526
#include <cstdint>
2627
#include <ctime>
2728
#include <memory>
@@ -492,6 +493,7 @@ void ScannerContext::push_pending_scan_task(std::shared_ptr<ScanTask> scan_task,
492493
DORIS_CHECK(scan_task != nullptr);
493494
DORIS_CHECK(scan_task->cached_blocks.empty());
494495
DORIS_CHECK(!scan_task->is_eos());
496+
scan_task->pending_since_ns = MonotonicNanos();
495497
_pending_scanners.push(std::move(scan_task));
496498
}
497499

@@ -525,9 +527,13 @@ bool ScannerContext::can_admit_scan_task(const std::unique_lock<std::mutex>& tra
525527
// _transfer_lock exactly as the TaskExecutor path reads them. A worker admitting the task it
526528
// runs itself is already counted as active; like the task _get_margin() is about to submit, it
527529
// must not count against the budget, otherwise the pool would stop one slot short of it.
528-
const int32_t busy_scan_slots = _scanner_scheduler->get_active_threads() +
529-
_scanner_scheduler->get_queue_size() -
530-
(admitting_on_worker ? 1 : 0);
530+
// The two counters are read under separate pool locks, and a worker moves a task from queued
531+
// to active atomically. Reading the queue first makes a concurrent dequeue count that task
532+
// twice rather than not at all, so a torn read defers instead of overbooking the last slot.
533+
const int32_t queued_scan_tasks = _scanner_scheduler->get_queue_size();
534+
const int32_t active_scan_threads = _scanner_scheduler->get_active_threads();
535+
const int32_t busy_scan_slots =
536+
active_scan_threads + queued_scan_tasks - (admitting_on_worker ? 1 : 0);
531537
if (busy_scan_slots < _min_scan_concurrency_of_scan_scheduler) {
532538
return true;
533539
}
@@ -537,7 +543,8 @@ bool ScannerContext::can_admit_scan_task(const std::unique_lock<std::mutex>& tra
537543
}
538544

539545
std::shared_ptr<ScanTask> ScannerContext::try_get_next_scan_task(
540-
const std::unique_lock<std::mutex>& transfer_lock, int64_t context_queue_wait_ns) {
546+
const std::unique_lock<std::mutex>& transfer_lock, int64_t context_submit_time_ns,
547+
int64_t context_start_time_ns) {
541548
if (!can_admit_scan_task(transfer_lock, true)) {
542549
VLOG_DEBUG << fmt::format(
543550
"[{}|{}] refuse admission, pending: {}, task queue: {}, scheduled: {}, done: {}",
@@ -553,10 +560,14 @@ std::shared_ptr<ScanTask> ScannerContext::try_get_next_scan_task(
553560
// ThreadPool admission bypasses ScannerScheduler::submit(); restart the per-scanner wait
554561
// timer here so it measures admission-to-execution instead of everything since the previous
555562
// attempt paused, which would include time the cached blocks waited for the operator. The
556-
// Context runnable waited for a worker on behalf of this scanner, so credit that wait too.
563+
// Context runnable waited for a worker on behalf of this scanner only while the scanner was
564+
// pending: with LIFO re-admission the runnable may have been queued before the scanner's
565+
// previous attempt ran, so do not credit the part of the runnable's wait before it was pending.
557566
if (auto scanner_delegate = scan_task->scanner.lock()) {
558567
scanner_delegate->_scanner->start_wait_worker_timer();
559-
scanner_delegate->_scanner->add_wait_worker_time(context_queue_wait_ns);
568+
const int64_t wait_start_ns = std::max(context_submit_time_ns, scan_task->pending_since_ns);
569+
scanner_delegate->_scanner->add_wait_worker_time(
570+
std::max<int64_t>(0, context_start_time_ns - wait_start_ns));
560571
}
561572
++_num_scheduled_scanners;
562573
VLOG_DEBUG << fmt::format("[{}|{}] admit scanner, pending: {}, task queue: {}, scheduled: {}",

‎be/src/exec/scan/scanner_context.h‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -79,6 +79,9 @@ class ScanTask {
7979
std::weak_ptr<ScannerDelegate> scanner;
8080
std::list<std::pair<BlockUPtr, size_t>> cached_blocks;
8181
bool is_first_schedule = true;
82+
// MonotonicNanos() at which the ThreadPool scheduler returned this task to _pending_scanners,
83+
// or 0 if it has been pending since the Context was created. Protected by _transfer_lock.
84+
int64_t pending_since_ns = 0;
8285
// Use weak_ptr to avoid circular references and potential memory leaks with SplitRunner.
8386
// ScannerContext only needs to observe the lifetime of SplitRunner without owning it.
8487
// When SplitRunner is destroyed, split_runner.lock() will return nullptr, ensuring safe access.
@@ -225,10 +228,11 @@ class ScannerContext : public std::enable_shared_from_this<ScannerContext>,
225228

226229
// Atomically check whether this context can start another scan task, move one task from
227230
// pending to scheduled, and return it. It is called by the pool worker that will run the task;
228-
// `context_queue_wait_ns` is how long that worker's Context runnable waited in the pool queue.
229-
// The caller must hold _transfer_lock.
231+
// the worker's Context runnable was submitted at `context_submit_time_ns` and started at
232+
// `context_start_time_ns` (both MonotonicNanos()). The caller must hold _transfer_lock.
230233
std::shared_ptr<ScanTask> try_get_next_scan_task(
231-
const std::unique_lock<std::mutex>& transfer_lock, int64_t context_queue_wait_ns);
234+
const std::unique_lock<std::mutex>& transfer_lock, int64_t context_submit_time_ns,
235+
int64_t context_start_time_ns);
232236

233237
protected:
234238
/// Four criteria to determine whether to increase the parallelism of the scanners

‎be/src/exec/scan/scanner_scheduler.h‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919

2020
#include <atomic>
2121
#include <memory>
22+
#include <mutex>
2223

2324
#include "common/be_mock_util.h"
2425
#include "common/status.h"
@@ -252,6 +253,9 @@ class ThreadPoolSimplifiedScanScheduler MOCK_REMOVE(final) : public ScannerSched
252253
std::weak_ptr<CgroupCpuCtl> _cgroup_cpu_ctl;
253254
std::string _sched_name;
254255
std::string _workload_group;
256+
// Serializes the scheduler-wide budget check and the Context submission across Contexts.
257+
// Acquired while holding a Context's _transfer_lock; never the other way around.
258+
std::mutex _submit_lock;
255259
};
256260

257261
class TaskExecutorSimplifiedScanScheduler final : public ScannerScheduler {

‎be/src/exec/scan/simplified_scan_scheduler.cpp‎

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,10 @@ Status ThreadPoolSimplifiedScanScheduler::schedule_scan_task(
5656
// cleared; fail below instead of waiting for that runnable forever.
5757
return Status::OK();
5858
}
59+
// The pool budget read by can_admit_scan_task() is shared by every Context. Hold the
60+
// scheduler-wide lock from that check through submission, otherwise two Contexts can both see
61+
// the last free slot and the loser fails its query on a full pool instead of deferring.
62+
std::lock_guard<std::mutex> submit_lock(_submit_lock);
5963
if (!context_queued && !scanner_ctx->can_admit_scan_task(transfer_lock, false)) {
6064
// No runnable is needed when the Context has no pending scanner or its concurrency slots
6165
// are occupied. A completion only wakes the operator; the operator's next consumption in
@@ -69,6 +73,8 @@ Status ThreadPoolSimplifiedScanScheduler::schedule_scan_task(
6973
return failure;
7074
}
7175

76+
DBUG_EXECUTE_IF("ThreadPoolSimplifiedScanScheduler.schedule_scan_task.before_submit",
77+
DBUG_RUN_CALLBACK());
7278
// ThreadPool::submit_func() may return an error after retaining the runnable. Set the marker
7379
// before submission so either outcome is safe: a retained callback clears it, while a truly
7480
// rejected callback leaves a terminal Context that no longer needs rescheduling.
@@ -91,7 +97,7 @@ Status ThreadPoolSimplifiedScanScheduler::schedule_scan_task(
9197

9298
void ThreadPoolSimplifiedScanScheduler::_run_context(std::shared_ptr<ScannerContext> scanner_ctx,
9399
int64_t submit_time_ns) {
94-
const int64_t context_queue_wait_ns = MonotonicNanos() - submit_time_ns;
100+
const int64_t start_time_ns = MonotonicNanos();
95101
std::shared_ptr<ScanTask> scan_task;
96102
Status admission_status = [&]() -> Status {
97103
std::unique_lock<std::mutex> transfer_lock(scanner_ctx->transfer_lock());
@@ -111,8 +117,8 @@ void ThreadPoolSimplifiedScanScheduler::_run_context(std::shared_ptr<ScannerCont
111117
RETURN_IF_CATCH_EXCEPTION({
112118
// Admission checks cached results, scheduled tasks, and pool slack while holding
113119
// transfer_lock.
114-
scan_task =
115-
scanner_ctx->try_get_next_scan_task(transfer_lock, context_queue_wait_ns);
120+
scan_task = scanner_ctx->try_get_next_scan_task(transfer_lock, submit_time_ns,
121+
start_time_ns);
116122
if (scan_task != nullptr) {
117123
DBUG_EXECUTE_IF("ThreadPoolSimplifiedScanScheduler._run_context.inject_failure",
118124
{

‎be/test/exec/scan/scanner_context_test.cpp‎

Lines changed: 170 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -25,11 +25,13 @@
2525

2626
#include <atomic>
2727
#include <chrono>
28+
#include <functional>
2829
#include <list>
2930
#include <memory>
3031
#include <mutex>
3132
#include <thread>
3233
#include <tuple>
34+
#include <vector>
3335

3436
#include "common/config.h"
3537
#include "common/object_pool.h"
@@ -967,19 +969,34 @@ TEST_F(ScannerContextTest, thread_pool_admission_state) {
967969
EXPECT_TRUE(scanner_context->is_context_queued(context_transfer_lock));
968970
scanner_context->set_context_queued(false, context_transfer_lock);
969971

970-
// The Context admits exactly one scanner and counts it as scheduled. The time its runnable
971-
// waited in the pool queue is credited to the admitted scanner's wait-worker time.
972-
const int64_t wait_worker_time = scanner->get_scanner_wait_worker_timer();
973-
auto admitted_task = scanner_context->try_get_next_scan_task(context_transfer_lock, 1000);
972+
// The Context admits exactly one scanner and counts it as scheduled. Its runnable was queued
973+
// (at 1000) before the scanner was returned to pending (at 2500), as with LIFO re-admission
974+
// of a scanner whose previous attempt ran while the runnable waited. Only the part of the
975+
// runnable's wait during which the scanner was pending is its wait-worker time.
976+
EXPECT_GT(consumed_task->pending_since_ns, 0);
977+
consumed_task->pending_since_ns = 2500;
978+
int64_t wait_worker_time = scanner->get_scanner_wait_worker_timer();
979+
auto admitted_task = scanner_context->try_get_next_scan_task(context_transfer_lock, 1000, 3000);
974980
EXPECT_EQ(admitted_task, consumed_task);
975-
EXPECT_EQ(scanner->get_scanner_wait_worker_timer(), wait_worker_time + 1000);
981+
EXPECT_EQ(scanner->get_scanner_wait_worker_timer(), wait_worker_time + 500);
976982
EXPECT_EQ(scanner_context->_num_scheduled_scanners, 1);
977983
EXPECT_TRUE(scanner_context->_pending_scanners.empty());
978984

985+
// A scanner already pending when the runnable was queued waited for the whole queue time.
986+
scanner_context->_num_scheduled_scanners = 0;
987+
auto early_task = std::make_shared<ScanTask>(scanners.front());
988+
scanner_context->push_pending_scan_task(early_task, context_transfer_lock);
989+
early_task->pending_since_ns = 500;
990+
wait_worker_time = scanner->get_scanner_wait_worker_timer();
991+
EXPECT_EQ(scanner_context->try_get_next_scan_task(context_transfer_lock, 1000, 3000),
992+
early_task);
993+
EXPECT_EQ(scanner->get_scanner_wait_worker_timer(), wait_worker_time + 2000);
994+
EXPECT_EQ(scanner_context->_num_scheduled_scanners, 1);
995+
979996
auto blocked_task = std::make_shared<ScanTask>(std::make_shared<ScannerDelegate>(scanner));
980997
scanner_context->push_pending_scan_task(blocked_task, context_transfer_lock);
981998
EXPECT_FALSE(scanner_context->can_admit_scan_task(context_transfer_lock, false));
982-
EXPECT_EQ(scanner_context->try_get_next_scan_task(context_transfer_lock, 0), nullptr);
999+
EXPECT_EQ(scanner_context->try_get_next_scan_task(context_transfer_lock, 0, 0), nullptr);
9831000

9841001
// A stopped Context admits nothing even when nothing is progressing.
9851002
scanner_context->_num_scheduled_scanners = 0;
@@ -988,6 +1005,51 @@ TEST_F(ScannerContextTest, thread_pool_admission_state) {
9881005
EXPECT_FALSE(scanner_context->can_admit_scan_task(context_transfer_lock, false));
9891006
}
9901007

1008+
TEST_F(ScannerContextTest, thread_pool_admission_reads_queue_before_active_threads) {
1009+
const int parallel_tasks = 2;
1010+
auto scan_operator = std::make_unique<OlapScanOperatorX>(obj_pool.get(), tnode, 0, *descs,
1011+
parallel_tasks, TQueryCacheParam {});
1012+
auto olap_scan_local_state =
1013+
OlapScanLocalState::create_unique(state.get(), scan_operator.get());
1014+
1015+
OlapScanner::Params scanner_params;
1016+
scanner_params.state = state.get();
1017+
scanner_params.profile = profile.get();
1018+
scanner_params.limit = -1;
1019+
scanner_params.key_ranges = std::vector<OlapScanRange*>();
1020+
std::shared_ptr<Scanner> scanner =
1021+
OlapScanner::create_shared(olap_scan_local_state.get(), std::move(scanner_params));
1022+
std::list<std::shared_ptr<ScannerDelegate>> scanners;
1023+
for (int i = 0; i < 2; ++i) {
1024+
scanners.push_back(std::make_shared<ScannerDelegate>(scanner));
1025+
}
1026+
auto scanner_context = ScannerContext::create_shared(
1027+
state.get(), olap_scan_local_state.get(), output_tuple_desc, output_row_descriptor,
1028+
scanners, -1, scan_dependency, parallel_tasks);
1029+
1030+
// A worker moves a task from queued to active under one pool lock, while admission reads the
1031+
// two counters under separate locks. Reading the queue first means a dequeue in between is
1032+
// counted twice (defer) rather than missed (overbook the last slot and fail the submission).
1033+
std::unique_ptr<MockSimplifiedScanScheduler> scheduler =
1034+
std::make_unique<MockSimplifiedScanScheduler>(cgroup_cpu_ctl);
1035+
{
1036+
testing::InSequence sequence;
1037+
EXPECT_CALL(*scheduler, get_queue_size()).WillOnce(testing::Return(1));
1038+
EXPECT_CALL(*scheduler, get_active_threads()).WillOnce(testing::Return(3));
1039+
}
1040+
scanner_context->_scanner_scheduler = scheduler.get();
1041+
scanner_context->_min_scan_concurrency_of_scan_scheduler = 4;
1042+
scanner_context->_max_scan_concurrency = parallel_tasks;
1043+
scanner_context->_min_scan_concurrency = 1;
1044+
scanner_context->_scan_starving = false;
1045+
scanner_context->_num_scheduled_scanners = 1;
1046+
1047+
std::unique_lock<std::mutex> transfer_lock(scanner_context->transfer_lock());
1048+
ASSERT_FALSE(scanner_context->_pending_scanners.empty());
1049+
// The over-counted pool looks full, so the Context stays at its minimum concurrency.
1050+
EXPECT_FALSE(scanner_context->can_admit_scan_task(transfer_lock, false));
1051+
}
1052+
9911053
TEST_F(ScannerContextTest, thread_pool_admission_follows_margin_limits) {
9921054
const int parallel_tasks = 4;
9931055
auto scan_operator = std::make_unique<OlapScanOperatorX>(obj_pool.get(), tnode, 0, *descs,
@@ -1174,6 +1236,108 @@ TEST_F(ScannerContextTest, thread_pool_submit_failure_policy) {
11741236
EXPECT_TRUE(scanner_context->is_context_queued(transfer_lock));
11751237
}
11761238

1239+
TEST_F(ScannerContextTest, thread_pool_budget_check_and_submit_are_atomic_across_contexts) {
1240+
// Three workers and no queue capacity. Two workers are parked, as if each ran a scanner of
1241+
// another Context, so the pool can accept exactly one more runnable.
1242+
ThreadPoolSimplifiedScanScheduler scheduler("budget_race_test", cgroup_cpu_ctl);
1243+
ASSERT_TRUE(scheduler.start(3, 3, 0, 3).ok());
1244+
CountDownLatch tasks_started(2);
1245+
CountDownLatch release_tasks(1);
1246+
Defer cleanup = [&] {
1247+
release_tasks.count_down();
1248+
scheduler.stop();
1249+
};
1250+
for (int i = 0; i < 2; ++i) {
1251+
ASSERT_TRUE(scheduler
1252+
.submit_scan_task(SimplifiedScanTask(
1253+
[&] {
1254+
tasks_started.count_down();
1255+
release_tasks.wait();
1256+
return true;
1257+
},
1258+
nullptr, nullptr))
1259+
.ok());
1260+
}
1261+
ASSERT_TRUE(tasks_started.wait_for(std::chrono::seconds(5)));
1262+
ASSERT_EQ(scheduler.get_active_threads(), 2);
1263+
1264+
const int parallel_tasks = 4;
1265+
auto scan_operator = std::make_unique<OlapScanOperatorX>(obj_pool.get(), tnode, 0, *descs,
1266+
parallel_tasks, TQueryCacheParam {});
1267+
auto olap_scan_local_state =
1268+
OlapScanLocalState::create_unique(state.get(), scan_operator.get());
1269+
OlapScanner::Params scanner_params;
1270+
scanner_params.state = state.get();
1271+
scanner_params.profile = profile.get();
1272+
scanner_params.limit = -1;
1273+
scanner_params.key_ranges = std::vector<OlapScanRange*>();
1274+
std::shared_ptr<Scanner> scanner =
1275+
OlapScanner::create_shared(olap_scan_local_state.get(), std::move(scanner_params));
1276+
1277+
// Each Context has one scanner running and another pending, and has reached its minimum
1278+
// concurrency. It may only ramp up while the pool has slack.
1279+
std::vector<std::shared_ptr<ScannerContext>> contexts;
1280+
for (int i = 0; i < 2; ++i) {
1281+
std::list<std::shared_ptr<ScannerDelegate>> scanners;
1282+
for (int j = 0; j < 2; ++j) {
1283+
scanners.push_back(std::make_shared<ScannerDelegate>(scanner));
1284+
}
1285+
auto scanner_context = ScannerContext::create_shared(
1286+
state.get(), olap_scan_local_state.get(), output_tuple_desc, output_row_descriptor,
1287+
scanners, -1, scan_dependency, parallel_tasks);
1288+
scanner_context->_scanner_scheduler = &scheduler;
1289+
scanner_context->_min_scan_concurrency_of_scan_scheduler = 3;
1290+
scanner_context->_max_scan_concurrency = parallel_tasks;
1291+
scanner_context->_min_scan_concurrency = 1;
1292+
scanner_context->_scan_starving = false;
1293+
scanner_context->_num_scheduled_scanners = 1;
1294+
ASSERT_FALSE(scanner_context->_pending_scanners.empty());
1295+
contexts.push_back(std::move(scanner_context));
1296+
}
1297+
1298+
// Park the first Context between its budget check and its submission until the second one
1299+
// reaches the same point, or for a bounded time if the scheduler serializes them. Without a
1300+
// scheduler-wide lock both pass the check on the last free slot and one submission fails.
1301+
std::atomic<int> entered {0};
1302+
CountDownLatch second_entered(1);
1303+
const bool old_enable_debug_points = config::enable_debug_points;
1304+
config::enable_debug_points = true;
1305+
DebugPoints::instance()->add_with_callback(
1306+
"ThreadPoolSimplifiedScanScheduler.schedule_scan_task.before_submit",
1307+
std::function<void()>([&] {
1308+
if (entered.fetch_add(1) == 0) {
1309+
static_cast<void>(second_entered.wait_for(std::chrono::milliseconds(500)));
1310+
} else {
1311+
second_entered.count_down();
1312+
}
1313+
}));
1314+
Defer cleanup_debug_point = [&] {
1315+
DebugPoints::instance()->remove(
1316+
"ThreadPoolSimplifiedScanScheduler.schedule_scan_task.before_submit");
1317+
config::enable_debug_points = old_enable_debug_points;
1318+
};
1319+
1320+
Status statuses[2];
1321+
std::vector<std::thread> threads;
1322+
for (int i = 0; i < 2; ++i) {
1323+
threads.emplace_back([&, i] {
1324+
std::unique_lock<std::mutex> transfer_lock(contexts[i]->transfer_lock());
1325+
statuses[i] = scheduler.schedule_scan_task(contexts[i], nullptr, transfer_lock);
1326+
});
1327+
}
1328+
for (auto& thread : threads) {
1329+
thread.join();
1330+
}
1331+
1332+
// The Context that lost the last slot defers instead of failing its query.
1333+
for (int i = 0; i < 2; ++i) {
1334+
EXPECT_TRUE(statuses[i].ok()) << statuses[i].to_string();
1335+
EXPECT_FALSE(contexts[i]->done());
1336+
std::unique_lock<std::mutex> transfer_lock(contexts[i]->transfer_lock());
1337+
EXPECT_TRUE(contexts[i]->_process_status.ok());
1338+
}
1339+
}
1340+
11771341
TEST_F(ScannerContextTest, run_context_publishes_admission_failure) {
11781342
const bool old_enable_debug_points = config::enable_debug_points;
11791343
config::enable_debug_points = true;

0 commit comments

Comments
 (0)