Skip to content

Commit e7b7f1d

Browse files
authored
[opt](cloud) Add warm up job count metric on BE (#64734)
## Proposed changes Add a `file_cache_warm_up_job_num` bvar metric that tracks the number of warm up jobs currently held in a BE's memory. This gives operators per-BE visibility into how many warm up jobs each backend is currently holding. ### What changed In `be/src/cloud/cloud_warm_up_manager.cpp`: - New `bvar::Adder<int64_t> g_file_cache_warm_up_job_num("file_cache_warm_up_job_num")`. - **+1** when FE dispatches a new job to this BE: - `check_and_set_job_id` — regular (cluster/table) warm up `SET_JOB`, on `_cur_job_id` transition `0 -> job_id`. - `check_and_set_batch_id` — defensive, same `0 -> job_id` transition (e.g. if a `SET_BATCH` were to arrive first). - `set_event` — event-driven `SET_JOB`, when a new `job_id` is inserted into `_tablet_replica_cache`. - **-1** when the job is cleared: - `clear_job` — regular `CLEAR_JOB`, only when a live job actually existed. - `set_event` (clear) — event-driven `CLEAR_JOB`, only when `_tablet_replica_cache.erase()` actually removed an entry.
1 parent efa3748 commit e7b7f1d

1 file changed

Lines changed: 20 additions & 2 deletions

File tree

‎be/src/cloud/cloud_warm_up_manager.cpp‎

Lines changed: 20 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -109,6 +109,10 @@ bvar::Status<int64_t> g_file_cache_warm_up_rowset_last_call_unix_ts(
109109
"file_cache_warm_up_rowset_last_call_unix_ts", 0);
110110
bvar::Adder<uint64_t> file_cache_warm_up_failed_task_num("file_cache_warm_up", "failed_task_num");
111111
bvar::Adder<int64_t> g_balance_tablet_be_mapping_size("balance_tablet_be_mapping_size");
112+
// Number of warm up jobs currently held in this BE's memory.
113+
// Incremented when FE dispatches a new job to this BE (SET_JOB / SET_BATCH / event SET_JOB),
114+
// decremented when the job is cleared (CLEAR_JOB / event CLEAR_JOB).
115+
bvar::Adder<int64_t> g_file_cache_warm_up_job_num("file_cache_warm_up_job_num");
112116

113117
bvar::LatencyRecorder g_file_cache_warm_up_rowset_wait_for_compaction_latency(
114118
"file_cache_warm_up_rowset_wait_for_compaction_latency");
@@ -440,6 +444,7 @@ Status CloudWarmUpManager::check_and_set_job_id(int64_t job_id) {
440444
std::lock_guard lock(_mtx);
441445
if (_cur_job_id == 0) {
442446
_cur_job_id = job_id;
447+
g_file_cache_warm_up_job_num << 1;
443448
}
444449
Status st = Status::OK();
445450
if (_cur_job_id != job_id) {
@@ -458,6 +463,7 @@ Status CloudWarmUpManager::check_and_set_batch_id(int64_t job_id, int64_t batch_
458463
}
459464
if (_cur_job_id == 0) {
460465
_cur_job_id = job_id;
466+
g_file_cache_warm_up_job_num << 1;
461467
}
462468
if (_cur_batch_id == batch_id) {
463469
*retry = true;
@@ -503,6 +509,9 @@ Status CloudWarmUpManager::clear_job(int64_t job_id) {
503509
std::lock_guard lock(_mtx);
504510
Status st = Status::OK();
505511
if (job_id == _cur_job_id) {
512+
if (_cur_job_id != 0) {
513+
g_file_cache_warm_up_job_num << -1;
514+
}
506515
_cur_job_id = 0;
507516
_cur_batch_id = -1;
508517
_pending_job_metas.clear();
@@ -525,11 +534,14 @@ Status CloudWarmUpManager::set_event(int64_t job_id, TWarmUpEventType::type even
525534
Status st = Status::OK();
526535
if (event == TWarmUpEventType::type::LOAD) {
527536
if (clear) {
528-
_tablet_replica_cache.erase(job_id);
537+
if (_tablet_replica_cache.erase(job_id) > 0) {
538+
g_file_cache_warm_up_job_num << -1;
539+
}
529540
_event_driven_filters.erase(job_id);
530541
LOG(INFO) << "Clear event driven sync, job_id=" << job_id << ", event=" << event;
531542
} else if (!_tablet_replica_cache.contains(job_id)) {
532543
static_cast<void>(_tablet_replica_cache[job_id]);
544+
g_file_cache_warm_up_job_num << 1;
533545
if (table_ids != nullptr) {
534546
// table-level filter: set to the given table_id set (may be empty,
535547
// meaning all matched tables were deleted — warm up nothing)
@@ -663,7 +675,13 @@ std::vector<JobReplicaInfo> CloudWarmUpManager::get_replica_info(int64_t tablet_
663675
}
664676
for (auto job_id : cancelled_jobs) {
665677
LOG(INFO) << "get_replica_info: erasing cancelled job, job_id=" << job_id;
666-
_tablet_replica_cache.erase(job_id);
678+
// Lazy cleanup path: FE reported the warm up job as CANCELLED, so the job is
679+
// no longer held in memory. Keep the job-count metric and the event filter
680+
// consistent with the explicit CLEAR_JOB path in set_event().
681+
if (_tablet_replica_cache.erase(job_id) > 0) {
682+
g_file_cache_warm_up_job_num << -1;
683+
}
684+
_event_driven_filters.erase(job_id);
667685
}
668686
VLOG_DEBUG << "get_replica_info: return " << replicas.size()
669687
<< " replicas, tablet id=" << tablet_id;

0 commit comments

Comments
 (0)