Skip to content

Commit 30406d6

Browse files
[enhancement](filecache) use rocksdb to persist cache block meta (#57072)
1 parent a77e7c5 commit 30406d6

24 files changed

Lines changed: 2637 additions & 418 deletions

‎be/src/common/config.cpp‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1152,6 +1152,8 @@ DEFINE_mInt64(cache_lock_held_long_tail_threshold_us, "30000000");
11521152
DEFINE_mBool(enable_file_cache_keep_base_compaction_output, "false");
11531153
DEFINE_mBool(enable_file_cache_adaptive_write, "true");
11541154
DEFINE_mDouble(file_cache_keep_base_compaction_output_min_hit_ratio, "0.7");
1155+
// if difference below this threshold, we consider cache's progressive upgrading (2.0->3.0) successful
1156+
DEFINE_mDouble(file_cache_meta_store_vs_file_system_diff_num_threshold, "0.3");
11551157

11561158
DEFINE_mInt64(file_cache_remove_block_qps_limit, "1000");
11571159
DEFINE_mInt64(file_cache_background_gc_interval_ms, "100");

‎be/src/common/config.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1175,6 +1175,7 @@ DECLARE_mInt64(cache_lock_held_long_tail_threshold_us);
11751175
DECLARE_mBool(enable_file_cache_keep_base_compaction_output);
11761176
DECLARE_mBool(enable_file_cache_adaptive_write);
11771177
DECLARE_mDouble(file_cache_keep_base_compaction_output_min_hit_ratio);
1178+
DECLARE_mDouble(file_cache_meta_store_vs_file_system_diff_num_threshold);
11781179
DECLARE_mInt64(file_cache_remove_block_qps_limit);
11791180
DECLARE_mInt64(file_cache_background_gc_interval_ms);
11801181
DECLARE_mInt64(file_cache_background_block_lru_update_interval_ms);

‎be/src/io/cache/block_file_cache.cpp‎

Lines changed: 26 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -240,6 +240,8 @@ BlockFileCache::BlockFileCache(const std::string& cache_base_path,
240240
_cache_base_path.c_str(), "file_cache_disk_limit_mode", 0);
241241
_need_evict_cache_in_advance_metrics = std::make_shared<bvar::Status<size_t>>(
242242
_cache_base_path.c_str(), "file_cache_need_evict_cache_in_advance", 0);
243+
_meta_store_write_queue_size_metrics = std::make_shared<bvar::Status<size_t>>(
244+
_cache_base_path.c_str(), "file_cache_meta_store_write_queue_size", 0);
243245

244246
_cache_lock_wait_time_us = std::make_shared<bvar::LatencyRecorder>(
245247
_cache_base_path.c_str(), "file_cache_cache_lock_wait_time_us");
@@ -460,6 +462,7 @@ FileBlocks BlockFileCache::get_impl(const UInt128Wrapper& hash, const CacheConte
460462
key.hash = hash;
461463
key.meta.type = context.cache_type;
462464
key.meta.expiration_time = context.expiration_time;
465+
key.meta.tablet_id = context.tablet_id;
463466
_storage->load_blocks_directly_unlocked(this, key, cache_lock);
464467

465468
it = _files.find(hash);
@@ -489,7 +492,7 @@ FileBlocks BlockFileCache::get_impl(const UInt128Wrapper& hash, const CacheConte
489492

490493
FileCacheType origin_type = cell.file_block->cache_type();
491494
if (origin_type == FileCacheType::TTL) continue;
492-
st = cell.file_block->change_cache_type_between_ttl_and_others(FileCacheType::TTL);
495+
st = cell.file_block->change_cache_type_lock(FileCacheType::TTL, cache_lock);
493496
if (st.ok()) {
494497
auto& queue = get_queue(origin_type);
495498
queue.remove(cell.queue_iterator.value(), cache_lock);
@@ -533,8 +536,8 @@ FileBlocks BlockFileCache::get_impl(const UInt128Wrapper& hash, const CacheConte
533536
for (auto& [_, cell] : file_blocks) {
534537
auto cache_type = cell.file_block->cache_type();
535538
if (cache_type != FileCacheType::TTL) continue;
536-
auto st = cell.file_block->change_cache_type_between_ttl_and_others(
537-
FileCacheType::NORMAL);
539+
auto st =
540+
cell.file_block->change_cache_type_lock(FileCacheType::NORMAL, cache_lock);
538541
if (st.ok()) {
539542
if (cell.queue_iterator) {
540543
auto& ttl_queue = get_queue(FileCacheType::TTL);
@@ -702,6 +705,7 @@ FileBlocks BlockFileCache::split_range_into_cells(const UInt128Wrapper& hash,
702705
key.offset = current_pos;
703706
key.meta.type = context.cache_type;
704707
key.meta.expiration_time = context.expiration_time;
708+
key.meta.tablet_id = context.tablet_id;
705709
auto file_block = std::make_shared<FileBlock>(key, current_size, this,
706710
FileBlock::State::SKIP_CACHE);
707711
file_blocks.push_back(std::move(file_block));
@@ -861,12 +865,13 @@ FileBlockCell* BlockFileCache::add_cell(const UInt128Wrapper& hash, const CacheC
861865
key.offset = offset;
862866
key.meta.type = context.cache_type;
863867
key.meta.expiration_time = context.expiration_time;
868+
key.meta.tablet_id = context.tablet_id;
864869
FileBlockCell cell(std::make_shared<FileBlock>(key, size, this, state), cache_lock);
865870
Status st;
866871
if (context.expiration_time == 0 && context.cache_type == FileCacheType::TTL) {
867-
st = cell.file_block->change_cache_type_between_ttl_and_others(FileCacheType::NORMAL);
872+
st = cell.file_block->change_cache_type_lock(FileCacheType::NORMAL, cache_lock);
868873
} else if (context.cache_type != FileCacheType::TTL && context.expiration_time != 0) {
869-
st = cell.file_block->change_cache_type_between_ttl_and_others(FileCacheType::TTL);
874+
st = cell.file_block->change_cache_type_lock(FileCacheType::TTL, cache_lock);
870875
}
871876
if (!st.ok()) {
872877
LOG(WARNING) << "Cannot change cache type. expiration_time=" << context.expiration_time
@@ -1157,8 +1162,7 @@ bool BlockFileCache::remove_if_ttl_file_blocks(const UInt128Wrapper& file_key, b
11571162
}
11581163

11591164
if (cell.file_block->cache_type() == FileCacheType::NORMAL) continue;
1160-
st = cell.file_block->change_cache_type_between_ttl_and_others(
1161-
FileCacheType::NORMAL);
1165+
st = cell.file_block->change_cache_type_lock(FileCacheType::NORMAL, cache_lock);
11621166
if (st.ok()) {
11631167
if (cell.queue_iterator) {
11641168
ttl_queue.remove(cell.queue_iterator.value(), cache_lock);
@@ -1457,6 +1461,7 @@ void BlockFileCache::remove(FileBlockSPtr file_block, T& cache_lock, U& block_lo
14571461
auto offset = file_block->offset();
14581462
auto type = file_block->cache_type();
14591463
auto expiration_time = file_block->expiration_time();
1464+
auto tablet_id = file_block->tablet_id();
14601465
auto* cell = get_cell(hash, offset, cache_lock);
14611466
file_block->cell = nullptr;
14621467
DCHECK(cell);
@@ -1477,6 +1482,7 @@ void BlockFileCache::remove(FileBlockSPtr file_block, T& cache_lock, U& block_lo
14771482
key.offset = offset;
14781483
key.meta.type = type;
14791484
key.meta.expiration_time = expiration_time;
1485+
key.meta.tablet_id = tablet_id;
14801486
if (sync) {
14811487
int64_t duration_ns = 0;
14821488
Status st;
@@ -2009,6 +2015,18 @@ void BlockFileCache::run_background_monitor() {
20092015
_cur_disposable_queue_element_count_metrics->set_value(
20102016
_disposable_queue.get_elements_num(cache_lock));
20112017

2018+
// Update meta store write queue size if storage is FSFileCacheStorage
2019+
if (_storage->get_type() == FileCacheStorageType::DISK) {
2020+
auto* fs_storage = dynamic_cast<FSFileCacheStorage*>(_storage.get());
2021+
if (fs_storage != nullptr) {
2022+
auto* meta_store = fs_storage->get_meta_store();
2023+
if (meta_store != nullptr) {
2024+
_meta_store_write_queue_size_metrics->set_value(
2025+
meta_store->get_write_queue_size());
2026+
}
2027+
}
2028+
}
2029+
20122030
if (_num_read_blocks->get_value() > 0) {
20132031
_hit_ratio->set_value((double)_num_hit_blocks->get_value() /
20142032
(double)_num_read_blocks->get_value());
@@ -2214,7 +2232,7 @@ void BlockFileCache::modify_expiration_time(const UInt128Wrapper& hash,
22142232

22152233
FileCacheType origin_type = cell.file_block->cache_type();
22162234
if (origin_type == FileCacheType::TTL) continue;
2217-
st = cell.file_block->change_cache_type_between_ttl_and_others(FileCacheType::TTL);
2235+
st = cell.file_block->change_cache_type_lock(FileCacheType::TTL, cache_lock);
22182236
if (st.ok()) {
22192237
auto& queue = get_queue(origin_type);
22202238
queue.remove(cell.queue_iterator.value(), cache_lock);

‎be/src/io/cache/block_file_cache.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -100,6 +100,7 @@ struct FileBlockCell {
100100

101101
size_t size() const { return file_block->_block_range.size(); }
102102

103+
FileBlockCell() = default;
103104
FileBlockCell(FileBlockSPtr file_block, std::lock_guard<std::mutex>& cache_lock);
104105
FileBlockCell(FileBlockCell&& other) noexcept
105106
: file_block(std::move(other.file_block)),
@@ -553,6 +554,7 @@ class BlockFileCache {
553554
std::shared_ptr<bvar::Status<double>> _no_warmup_hit_ratio_1h;
554555
std::shared_ptr<bvar::Status<size_t>> _disk_limit_mode_metrics;
555556
std::shared_ptr<bvar::Status<size_t>> _need_evict_cache_in_advance_metrics;
557+
std::shared_ptr<bvar::Status<size_t>> _meta_store_write_queue_size_metrics;
556558

557559
std::shared_ptr<bvar::LatencyRecorder> _cache_lock_wait_time_us;
558560
std::shared_ptr<bvar::LatencyRecorder> _get_or_set_latency_us;

0 commit comments

Comments
 (0)