Skip to content

Commit 0918952

Browse files
[refactor](filecache) ttl management refactoring (#57922)
This commit refactors the file cache TTL management by: - introducing a dedicated BlockFileCacheTtlMgr into BlockFileCache to manage TTL updates - removing the ad‑hoc TTL bookkeeping, GC thread, and expiration helpers from BlockFileCache - changing TTL management grain from block level to tablet level Signed-off-by: zhengyu <[email protected]> Signed-off-by: freemandealer <[email protected]>
1 parent 86f0b3a commit 0918952

46 files changed

Lines changed: 1408 additions & 1212 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎be/src/cloud/cloud_internal_service.cpp‎

Lines changed: 1 addition & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -425,13 +425,7 @@ void CloudInternalServiceImpl::warm_up_rowset(google::protobuf::RpcController* c
425425
<< " us, tablet_id: " << rs_meta.tablet_id()
426426
<< ", rowset_id: " << rowset_id.to_string();
427427
}
428-
int64_t expiration_time =
429-
tablet_meta->ttl_seconds() == 0 || rs_meta.newest_write_timestamp() <= 0
430-
? 0
431-
: rs_meta.newest_write_timestamp() + tablet_meta->ttl_seconds();
432-
if (expiration_time <= UnixSeconds()) {
433-
expiration_time = 0;
434-
}
428+
int64_t expiration_time = tablet_meta->ttl_seconds();
435429

436430
if (!tablet->add_rowset_warmup_state(rs_meta, WarmUpTriggerSource::EVENT_DRIVEN)) {
437431
LOG(INFO) << "found duplicate warmup task for rowset " << rowset_id.to_string()

‎be/src/cloud/cloud_storage_engine.cpp‎

Lines changed: 27 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -280,11 +280,36 @@ bool CloudStorageEngine::stopped() {
280280

281281
Result<BaseTabletSPtr> CloudStorageEngine::get_tablet(int64_t tablet_id,
282282
SyncRowsetStats* sync_stats,
283-
bool force_use_only_cached) {
284-
return _tablet_mgr->get_tablet(tablet_id, false, true, sync_stats, force_use_only_cached)
283+
bool force_use_only_cached,
284+
bool cache_on_miss) {
285+
return _tablet_mgr
286+
->get_tablet(tablet_id, false, true, sync_stats, force_use_only_cached, cache_on_miss)
285287
.transform([](auto&& t) { return static_pointer_cast<BaseTablet>(std::move(t)); });
286288
}
287289

290+
Status CloudStorageEngine::get_tablet_meta(int64_t tablet_id, TabletMetaSharedPtr* tablet_meta,
291+
bool force_use_only_cached) {
292+
if (tablet_meta == nullptr) {
293+
return Status::InvalidArgument("tablet_meta output is null");
294+
}
295+
296+
#if 0
297+
if (_tablet_mgr && _tablet_mgr->peek_tablet_meta(tablet_id, tablet_meta)) {
298+
return Status::OK();
299+
}
300+
301+
if (force_use_only_cached) {
302+
return Status::NotFound("tablet meta {} not found in cache", tablet_id);
303+
}
304+
#endif
305+
306+
if (_meta_mgr == nullptr) {
307+
return Status::InternalError("cloud meta manager is not initialized");
308+
}
309+
310+
return _meta_mgr->get_tablet_meta(tablet_id, tablet_meta);
311+
}
312+
288313
Status CloudStorageEngine::start_bg_threads(std::shared_ptr<WorkloadGroup> wg_sptr) {
289314
RETURN_IF_ERROR(Thread::create(
290315
"CloudStorageEngine", "refresh_s3_info_thread",

‎be/src/cloud/cloud_storage_engine.h‎

Lines changed: 18 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -62,8 +62,25 @@ class CloudStorageEngine final : public BaseStorageEngine {
6262
void stop() override;
6363
bool stopped() override;
6464

65+
/* Parameters:
66+
* - tablet_id: the id of tablet to get
67+
* - sync_stats: the stats of sync rowset
68+
* - force_use_only_cached: whether only use cached tablet meta
69+
* - cache_on_miss: whether cache the tablet meta when missing in cache
70+
*/
6571
Result<BaseTabletSPtr> get_tablet(int64_t tablet_id, SyncRowsetStats* sync_stats = nullptr,
66-
bool force_use_only_cached = false) override;
72+
bool force_use_only_cached = false,
73+
bool cache_on_miss = true) override;
74+
75+
/*
76+
* Get the tablet meta for a specific tablet
77+
* Parameters:
78+
* - tablet_id: the id of tablet to get meta for
79+
* - tablet_meta: output TabletMeta shared pointer
80+
* - force_use_only_cached: whether only use cached tablet meta (return NotFound on miss)
81+
*/
82+
Status get_tablet_meta(int64_t tablet_id, TabletMetaSharedPtr* tablet_meta,
83+
bool force_use_only_cached = false) override;
6784

6885
Status start_bg_threads(std::shared_ptr<WorkloadGroup> wg_sptr = nullptr) override;
6986

‎be/src/cloud/cloud_tablet.cpp‎

Lines changed: 1 addition & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -407,12 +407,7 @@ void CloudTablet::add_rowsets(std::vector<RowsetSharedPtr> to_add, bool version_
407407
continue;
408408
}
409409

410-
int64_t expiration_time =
411-
_tablet_meta->ttl_seconds() == 0 ||
412-
rowset_meta->newest_write_timestamp() <= 0
413-
? 0
414-
: rowset_meta->newest_write_timestamp() +
415-
_tablet_meta->ttl_seconds();
410+
int64_t expiration_time = _tablet_meta->ttl_seconds();
416411
g_file_cache_cloud_tablet_submitted_segment_num << 1;
417412
if (rs->rowset_meta()->segment_file_size(seg_id) > 0) {
418413
g_file_cache_cloud_tablet_submitted_segment_size
@@ -1535,23 +1530,6 @@ Status CloudTablet::sync_meta() {
15351530
return st;
15361531
}
15371532

1538-
auto new_ttl_seconds = tablet_meta->ttl_seconds();
1539-
if (_tablet_meta->ttl_seconds() != new_ttl_seconds) {
1540-
_tablet_meta->set_ttl_seconds(new_ttl_seconds);
1541-
int64_t cur_time = UnixSeconds();
1542-
std::shared_lock rlock(_meta_lock);
1543-
for (auto& [_, rs] : _rs_version_map) {
1544-
for (int seg_id = 0; seg_id < rs->num_segments(); ++seg_id) {
1545-
int64_t new_expiration_time =
1546-
new_ttl_seconds + rs->rowset_meta()->newest_write_timestamp();
1547-
new_expiration_time = new_expiration_time > cur_time ? new_expiration_time : 0;
1548-
auto file_key = Segment::file_cache_key(rs->rowset_id().to_string(), seg_id);
1549-
auto* file_cache = io::FileCacheFactory::instance()->get_by_path(file_key);
1550-
file_cache->modify_expiration_time(file_key, new_expiration_time);
1551-
}
1552-
}
1553-
}
1554-
15551533
auto new_compaction_policy = tablet_meta->compaction_policy();
15561534
if (_tablet_meta->compaction_policy() != new_compaction_policy) {
15571535
_tablet_meta->set_compaction_policy(new_compaction_policy);

‎be/src/cloud/cloud_tablet_mgr.cpp‎

Lines changed: 32 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -161,7 +161,8 @@ void set_tablet_access_time_ms(CloudTablet* tablet) {
161161
Result<std::shared_ptr<CloudTablet>> CloudTabletMgr::get_tablet(int64_t tablet_id, bool warmup_data,
162162
bool sync_delete_bitmap,
163163
SyncRowsetStats* sync_stats,
164-
bool force_use_only_cached) {
164+
bool force_use_only_cached,
165+
bool cache_on_miss) {
165166
// LRU value type. `Value`'s lifetime MUST NOT be longer than `CloudTabletMgr`
166167
class Value : public LRUCacheValueBase {
167168
public:
@@ -196,7 +197,7 @@ Result<std::shared_ptr<CloudTablet>> CloudTabletMgr::get_tablet(int64_t tablet_i
196197
if (sync_stats) {
197198
++sync_stats->tablet_meta_cache_miss;
198199
}
199-
auto load_tablet = [this, &key, warmup_data, sync_delete_bitmap,
200+
auto load_tablet = [this, warmup_data, sync_delete_bitmap,
200201
sync_stats](int64_t tablet_id) -> Result<std::shared_ptr<CloudTablet>> {
201202
TabletMetaSharedPtr tablet_meta;
202203
auto start = std::chrono::steady_clock::now();
@@ -212,7 +213,6 @@ Result<std::shared_ptr<CloudTablet>> CloudTabletMgr::get_tablet(int64_t tablet_i
212213
}
213214

214215
auto tablet = std::make_shared<CloudTablet>(_engine, std::move(tablet_meta));
215-
auto value = std::make_unique<Value>(tablet, *_tablet_map);
216216
// MUST sync stats to let compaction scheduler work correctly
217217
SyncOptions options;
218218
options.warmup_delta_data = warmup_data;
@@ -222,16 +222,7 @@ Result<std::shared_ptr<CloudTablet>> CloudTabletMgr::get_tablet(int64_t tablet_i
222222
LOG(WARNING) << "failed to sync tablet " << tablet_id << ": " << st;
223223
return ResultError(st);
224224
}
225-
226-
auto* handle = _cache->insert(key, value.release(), 1, sizeof(CloudTablet),
227-
CachePriority::NORMAL);
228-
auto ret =
229-
std::shared_ptr<CloudTablet>(tablet.get(), [this, handle](CloudTablet* tablet) {
230-
set_tablet_access_time_ms(tablet);
231-
_cache->release(handle);
232-
});
233-
_tablet_map->put(std::move(tablet));
234-
return ret;
225+
return tablet;
235226
};
236227

237228
auto load_result = s_singleflight_load_tablet.load(tablet_id, std::move(load_tablet));
@@ -240,8 +231,22 @@ Result<std::shared_ptr<CloudTablet>> CloudTabletMgr::get_tablet(int64_t tablet_i
240231
load_result.error()));
241232
}
242233
auto tablet = load_result.value();
243-
set_tablet_access_time_ms(tablet.get());
244-
return tablet;
234+
if (!cache_on_miss) {
235+
set_tablet_access_time_ms(tablet.get());
236+
return tablet;
237+
}
238+
239+
auto value = std::make_unique<Value>(tablet, *_tablet_map);
240+
auto* insert_handle =
241+
_cache->insert(key, value.release(), 1, sizeof(CloudTablet), CachePriority::NORMAL);
242+
auto ret = std::shared_ptr<CloudTablet>(tablet.get(),
243+
[this, insert_handle](CloudTablet* tablet_ptr) {
244+
set_tablet_access_time_ms(tablet_ptr);
245+
_cache->release(insert_handle);
246+
});
247+
_tablet_map->put(std::move(tablet));
248+
set_tablet_access_time_ms(ret.get());
249+
return ret;
245250
}
246251
if (sync_stats) {
247252
++sync_stats->tablet_meta_cache_hit;
@@ -255,6 +260,18 @@ Result<std::shared_ptr<CloudTablet>> CloudTabletMgr::get_tablet(int64_t tablet_i
255260
return tablet;
256261
}
257262

263+
bool CloudTabletMgr::peek_tablet_meta(int64_t tablet_id, TabletMetaSharedPtr* tablet_meta) {
264+
if (tablet_meta == nullptr) {
265+
return false;
266+
}
267+
auto tablet = _tablet_map->get(tablet_id);
268+
if (!tablet) {
269+
return false;
270+
}
271+
*tablet_meta = tablet->tablet_meta();
272+
return true;
273+
}
274+
258275
void CloudTabletMgr::erase_tablet(int64_t tablet_id) {
259276
auto tablet_id_str = std::to_string(tablet_id);
260277
CacheKey key(tablet_id_str.data(), tablet_id_str.size());

‎be/src/cloud/cloud_tablet_mgr.h‎

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@
2626

2727
#include "common/status.h"
2828
#include "olap/olap_common.h"
29+
#include "olap/tablet_fwd.h"
2930

3031
namespace doris {
3132

@@ -44,10 +45,22 @@ class CloudTabletMgr {
4445

4546
// If the tablet is in cache, return this tablet directly; otherwise will get tablet meta first,
4647
// sync rowsets after, and download segment data in background if `warmup_data` is true.
48+
/* Parameters:
49+
* - tablet_id: the id of tablet to get
50+
* - warmup_data: whether warmup tablet data in background
51+
* - sync_delete_bitmap: whether sync delete bitmap when getting tablet
52+
* - sync_stats: the stats of sync rowset
53+
* - force_use_only_cached: whether only use cached tablet meta
54+
* - cache_on_miss: whether cache the tablet meta when missing in cache
55+
*/
4756
Result<std::shared_ptr<CloudTablet>> get_tablet(int64_t tablet_id, bool warmup_data = false,
4857
bool sync_delete_bitmap = true,
4958
SyncRowsetStats* sync_stats = nullptr,
50-
bool local_only = false);
59+
bool force_use_only_cached = false,
60+
bool cache_on_miss = true);
61+
62+
// Return true if cached tablet meta is found (without triggering RPC) and filled.
63+
bool peek_tablet_meta(int64_t tablet_id, TabletMetaSharedPtr* tablet_meta);
5164

5265
void erase_tablet(int64_t tablet_id);
5366

‎be/src/cloud/cloud_warm_up_manager.cpp‎

Lines changed: 1 addition & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -236,13 +236,7 @@ void CloudWarmUpManager::handle_jobs() {
236236
continue;
237237
}
238238

239-
int64_t expiration_time =
240-
tablet_meta->ttl_seconds() == 0 || rs->newest_write_timestamp() <= 0
241-
? 0
242-
: rs->newest_write_timestamp() + tablet_meta->ttl_seconds();
243-
if (expiration_time <= UnixSeconds()) {
244-
expiration_time = 0;
245-
}
239+
int64_t expiration_time = tablet_meta->ttl_seconds();
246240
if (!tablet->add_rowset_warmup_state(*rs, WarmUpTriggerSource::JOB)) {
247241
LOG(INFO) << "found duplicate warmup task for rowset " << rs->rowset_id()
248242
<< ", skip it";

‎be/src/common/config.cpp‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1151,7 +1151,9 @@ DEFINE_mInt64(file_cache_background_block_lru_update_interval_ms, "5000");
11511151
DEFINE_mInt64(file_cache_background_block_lru_update_qps_limit, "1000");
11521152
DEFINE_mBool(enable_reader_dryrun_when_download_file_cache, "true");
11531153
DEFINE_mInt64(file_cache_background_monitor_interval_ms, "5000");
1154-
DEFINE_mInt64(file_cache_background_ttl_gc_interval_ms, "3000");
1154+
DEFINE_mInt64(file_cache_background_ttl_gc_interval_ms, "180000");
1155+
DEFINE_mInt64(file_cache_background_ttl_info_update_interval_ms, "180000");
1156+
DEFINE_mInt64(file_cache_background_tablet_id_flush_interval_ms, "1000");
11551157
DEFINE_mInt64(file_cache_background_ttl_gc_batch, "1000");
11561158
DEFINE_mInt64(file_cache_background_lru_dump_interval_ms, "60000");
11571159
// dump queue only if the queue update specific times through several dump intervals

‎be/src/common/config.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1190,6 +1190,8 @@ DECLARE_mInt64(file_cache_background_block_lru_update_qps_limit);
11901190
DECLARE_mBool(enable_reader_dryrun_when_download_file_cache);
11911191
DECLARE_mInt64(file_cache_background_monitor_interval_ms);
11921192
DECLARE_mInt64(file_cache_background_ttl_gc_interval_ms);
1193+
DECLARE_mInt64(file_cache_background_ttl_info_update_interval_ms);
1194+
DECLARE_mInt64(file_cache_background_tablet_id_flush_interval_ms);
11931195
DECLARE_mInt64(file_cache_background_ttl_gc_batch);
11941196
DECLARE_Int32(file_cache_downloader_thread_num_min);
11951197
DECLARE_Int32(file_cache_downloader_thread_num_max);

0 commit comments

Comments
 (0)