Skip to content

Commit f8f8121

Browse files
committed
[fix](be) Restore file-level cache metadata loading during startup
Related PR: #57072, #65658 Problem Summary: During BE startup, get_or_set, the fully-covered lookup, and probe attempt to restore an absent file before background metadata loading completes. Their FileCacheKey leaves offset uninitialized, while the storage implementation now performs an exact RocksDB lookup for one block. A file with multiple persisted blocks can therefore miss existing cache data or restore only one block. Query metadata by tablet and file hash prefix, restore all blocks, and assert that the file hash is absent when entering synchronous restoration. Keep caller trigger conditions, key encoding, and existing tablet-range queries unchanged. Fix reuse of persistent file cache blocks while BE startup metadata loading is in progress. - Test: Unit Test via run-be-ut.sh with ASAN and GLIBC_COMPATIBILITY=OFF: 38 related tests passed; 9 focused tests passed again after final cleanup. The 5 startup regression cases failed before the fix and passed afterward. clang-format v16, build hygiene, git diff checks, and clang-tidy on changed lines passed. - Behavior changed: Yes, startup lookups restore all persisted blocks for an absent file instead of querying one uninitialized offset. - Does this need documentation: No
1 parent e4e4af3 commit f8f8121

10 files changed

Lines changed: 278 additions & 68 deletions

‎be/src/io/cache/block_file_cache.cpp‎

Lines changed: 3 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -701,12 +701,7 @@ FileBlocks BlockFileCache::get_impl(const UInt128Wrapper& hash, const CacheConte
701701
if (_async_open_done) {
702702
return {};
703703
}
704-
FileCacheKey key;
705-
key.hash = hash;
706-
key.meta.type = context.cache_type;
707-
key.meta.expiration_time = context.expiration_time;
708-
key.meta.tablet_id = context.tablet_id;
709-
_storage->load_blocks_directly_unlocked(this, key, cache_lock);
704+
_storage->load_file_blocks_directly_unlocked(this, hash, context.tablet_id, cache_lock);
710705

711706
it = _files.find(hash);
712707
if (it == _files.end()) [[unlikely]] {
@@ -812,12 +807,7 @@ Status BlockFileCache::get_downloaded_blocks_if_fully_covered(const UInt128Wrapp
812807
if (_async_open_done) {
813808
return Status::OK();
814809
}
815-
FileCacheKey key;
816-
key.hash = hash;
817-
key.meta.type = context.cache_type;
818-
key.meta.expiration_time = context.expiration_time;
819-
key.meta.tablet_id = context.tablet_id;
820-
_storage->load_blocks_directly_unlocked(this, key, cache_lock);
810+
_storage->load_file_blocks_directly_unlocked(this, hash, context.tablet_id, cache_lock);
821811

822812
it = _files.find(hash);
823813
if (it == _files.end()) {
@@ -893,12 +883,7 @@ FileBlocksProbeResult BlockFileCache::probe(const UInt128Wrapper& hash, size_t o
893883

894884
auto file_iterator = _files.find(hash);
895885
if (file_iterator == _files.end() && !_async_open_done) {
896-
FileCacheKey key;
897-
key.hash = hash;
898-
key.meta.type = context.cache_type;
899-
key.meta.expiration_time = context.expiration_time;
900-
key.meta.tablet_id = context.tablet_id;
901-
_storage->load_blocks_directly_unlocked(this, key, cache_lock);
886+
_storage->load_file_blocks_directly_unlocked(this, hash, context.tablet_id, cache_lock);
902887
file_iterator = _files.find(hash);
903888
}
904889

‎be/src/io/cache/cache_block_meta_store.cpp‎

Lines changed: 32 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,26 @@
3838
#include "util/threadpool.h"
3939

4040
namespace doris::io {
41+
namespace {
42+
43+
std::string serialize_key_prefix(int64_t tablet_id) {
44+
std::string prefix;
45+
prefix.push_back(0x1); // version byte
46+
get_key_coder(FieldType::OLAP_FIELD_TYPE_BIGINT)->full_encode_ascending(&tablet_id, &prefix);
47+
return prefix;
48+
}
49+
50+
std::string serialize_key_prefix(int64_t tablet_id, const UInt128Wrapper& hash) {
51+
auto prefix = serialize_key_prefix(tablet_id);
52+
const auto* coder = get_key_coder(FieldType::OLAP_FIELD_TYPE_BIGINT);
53+
uint64_t hash_high = hash.high();
54+
uint64_t hash_low = hash.low();
55+
coder->full_encode_ascending(&hash_high, &prefix);
56+
coder->full_encode_ascending(&hash_low, &prefix);
57+
return prefix;
58+
}
59+
60+
} // namespace
4161

4262
const std::string FILE_CACHE_META_COLUMN_FAMILY = "file_cache_meta";
4363

@@ -170,12 +190,16 @@ std::optional<BlockMeta> CacheBlockMetaStore::get(const BlockMetaKey& key) {
170190
}
171191

172192
std::unique_ptr<BlockMetaIterator> CacheBlockMetaStore::range_get(int64_t tablet_id) {
173-
// Generate prefix using new serialization format
174-
std::string prefix;
175-
prefix.push_back(0x1); // version byte
176-
auto* tablet_id_coder = get_key_coder(FieldType::OLAP_FIELD_TYPE_BIGINT);
177-
tablet_id_coder->full_encode_ascending(&tablet_id, &prefix);
193+
return range_get_by_prefix(serialize_key_prefix(tablet_id));
194+
}
178195

196+
std::unique_ptr<BlockMetaIterator> CacheBlockMetaStore::range_get(int64_t tablet_id,
197+
const UInt128Wrapper& hash) {
198+
return range_get_by_prefix(serialize_key_prefix(tablet_id, hash));
199+
}
200+
201+
std::unique_ptr<BlockMetaIterator> CacheBlockMetaStore::range_get_by_prefix(
202+
const std::string& prefix) {
179203
class RocksDBIterator : public BlockMetaIterator {
180204
public:
181205
RocksDBIterator(rocksdb::Iterator* iter, const std::string& prefix)
@@ -459,22 +483,8 @@ void CacheBlockMetaStore::async_write_worker() {
459483
}
460484

461485
std::string serialize_key(const BlockMetaKey& key) {
462-
std::string result;
463-
// Add version byte
464-
result.push_back(0x1);
465-
466-
// Encode tablet_id using KeyCoderTraits
467-
auto* tablet_id_coder = get_key_coder(FieldType::OLAP_FIELD_TYPE_BIGINT);
468-
tablet_id_coder->full_encode_ascending(&key.tablet_id, &result);
469-
470-
// Encode hash high and low parts
471-
uint64_t hash_high = key.hash.high();
472-
uint64_t hash_low = key.hash.low();
473-
tablet_id_coder->full_encode_ascending(&hash_high, &result);
474-
tablet_id_coder->full_encode_ascending(&hash_low, &result);
475-
476-
// Encode offset
477-
tablet_id_coder->full_encode_ascending(&key.offset, &result);
486+
auto result = serialize_key_prefix(key.tablet_id, key.hash);
487+
get_key_coder(FieldType::OLAP_FIELD_TYPE_BIGINT)->full_encode_ascending(&key.offset, &result);
478488

479489
return result;
480490
}
@@ -609,4 +619,4 @@ std::optional<BlockMeta> deserialize_value(std::string_view value_view, Status*
609619
return std::nullopt;
610620
}
611621

612-
} // namespace doris::io
622+
} // namespace doris::io

‎be/src/io/cache/cache_block_meta_store.h‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -99,6 +99,9 @@ class CacheBlockMetaStore {
9999
// Range query all BlockMeta for specified tablet_id
100100
std::unique_ptr<BlockMetaIterator> range_get(int64_t tablet_id);
101101

102+
// Range query all BlockMeta for a file in the specified tablet.
103+
std::unique_ptr<BlockMetaIterator> range_get(int64_t tablet_id, const UInt128Wrapper& hash);
104+
102105
// Get iterator for all BlockMeta records
103106
std::unique_ptr<BlockMetaIterator> get_all();
104107

@@ -116,6 +119,7 @@ class CacheBlockMetaStore {
116119

117120
private:
118121
void async_write_worker();
122+
std::unique_ptr<BlockMetaIterator> range_get_by_prefix(const std::string& prefix);
119123

120124
std::string _db_path;
121125
std::unique_ptr<rocksdb::DB> _db;

‎be/src/io/cache/file_cache_storage.h‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -79,9 +79,10 @@ class FileCacheStorage {
7979
// change the block meta
8080
virtual Status change_key_meta_type(const FileCacheKey& key, const FileCacheType type,
8181
const size_t size) = 0;
82-
// use when lazy load cache
83-
virtual void load_blocks_directly_unlocked(BlockFileCache* _mgr, const FileCacheKey& key,
84-
std::lock_guard<std::mutex>& cache_lock) {}
82+
// Restore a file while lazy loading, with its hash absent from the cache.
83+
virtual void load_file_blocks_directly_unlocked(BlockFileCache* mgr, const UInt128Wrapper& hash,
84+
int64_t tablet_id,
85+
std::lock_guard<std::mutex>& cache_lock) {}
8586
// force clear all current data in the cache
8687
virtual Status clear(std::string& msg) = 0;
8788
virtual FileCacheStorageType get_type() = 0;

‎be/src/io/cache/fs_file_cache_storage.cpp‎

Lines changed: 25 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@
4242
#include <unordered_set>
4343
#include <vector>
4444

45+
#include "common/check.h"
4546
#include "common/config.h"
4647
#include "common/logging.h"
4748
#include "common/status.h"
@@ -1157,26 +1158,34 @@ void FSFileCacheStorage::load_cache_info_into_memory(BlockFileCache* mgr) const
11571158
}
11581159
}
11591160

1160-
void FSFileCacheStorage::load_blocks_directly_unlocked(BlockFileCache* mgr, const FileCacheKey& key,
1161-
std::lock_guard<std::mutex>& cache_lock) {
1162-
BlockMetaKey mkey(key.meta.tablet_id, UInt128Wrapper(key.hash), key.offset);
1163-
auto block_meta = _meta_store->get(mkey);
1164-
if (!block_meta.has_value()) {
1165-
// cache miss
1161+
void FSFileCacheStorage::load_file_blocks_directly_unlocked(
1162+
BlockFileCache* mgr, const UInt128Wrapper& hash, int64_t tablet_id,
1163+
std::lock_guard<std::mutex>& cache_lock) {
1164+
DORIS_CHECK(!mgr->_files.contains(hash));
1165+
auto iterator = _meta_store->range_get(tablet_id, hash);
1166+
// Meta-store initialization failure leaves no database to query.
1167+
if (!iterator) {
1168+
LOG(WARNING) << "Failed to create iterator for meta store";
11661169
return;
11671170
}
11681171

1169-
CacheContext context_original;
1170-
context_original.query_id = TUniqueId();
1171-
context_original.expiration_time = block_meta->ttl;
1172-
context_original.cache_type = block_meta->type;
1173-
context_original.tablet_id = key.meta.tablet_id;
1172+
for (; iterator->valid(); iterator->next()) {
1173+
const auto meta_key = iterator->key();
1174+
const auto block_meta = iterator->value();
1175+
// Malformed persisted records cannot describe a downloaded cache block.
1176+
if (!iterator->get_last_key_error().ok() || !iterator->get_last_value_error().ok()) {
1177+
LOG(WARNING) << "Failed to deserialize cache block metadata: "
1178+
<< "key_error=" << iterator->get_last_key_error().to_string()
1179+
<< ", value_error=" << iterator->get_last_value_error().to_string();
1180+
continue;
1181+
}
11741182

1175-
if (handle_already_loaded_block(mgr, key.hash, key.offset, block_meta->size, key.meta.tablet_id,
1176-
cache_lock)) {
1177-
return;
1178-
} else {
1179-
mgr->add_cell(key.hash, context_original, key.offset, block_meta->size,
1183+
CacheContext context_original;
1184+
context_original.query_id = TUniqueId();
1185+
context_original.expiration_time = block_meta.ttl;
1186+
context_original.cache_type = block_meta.type;
1187+
context_original.tablet_id = tablet_id;
1188+
mgr->add_cell(hash, context_original, meta_key.offset, block_meta.size,
11801189
FileBlock::State::DOWNLOADED, cache_lock);
11811190
}
11821191
}

‎be/src/io/cache/fs_file_cache_storage.h‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -87,8 +87,9 @@ class FSFileCacheStorage : public FileCacheStorage {
8787
Status remove(const FileCacheKey& key) override;
8888
Status change_key_meta_type(const FileCacheKey& key, const FileCacheType type,
8989
const size_t size) override;
90-
void load_blocks_directly_unlocked(BlockFileCache* _mgr, const FileCacheKey& key,
91-
std::lock_guard<std::mutex>& cache_lock) override;
90+
void load_file_blocks_directly_unlocked(BlockFileCache* mgr, const UInt128Wrapper& hash,
91+
int64_t tablet_id,
92+
std::lock_guard<std::mutex>& cache_lock) override;
9293
Status clear(std::string& msg) override;
9394
std::string get_local_file(const FileCacheKey& key) override;
9495

‎be/src/io/cache/mem_file_cache_storage.cpp‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -106,9 +106,9 @@ Status MemFileCacheStorage::change_key_meta_type(const FileCacheKey& key, const
106106
return Status::OK();
107107
}
108108

109-
void MemFileCacheStorage::load_blocks_directly_unlocked(BlockFileCache* _mgr,
110-
const FileCacheKey& key,
111-
std::lock_guard<std::mutex>& cache_lock) {
109+
void MemFileCacheStorage::load_file_blocks_directly_unlocked(
110+
BlockFileCache* mgr, const UInt128Wrapper& hash, int64_t tablet_id,
111+
std::lock_guard<std::mutex>& cache_lock) {
112112
// load nothing for in memory cache coz nothing is persisted
113113
}
114114

‎be/src/io/cache/mem_file_cache_storage.h‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -43,8 +43,9 @@ class MemFileCacheStorage : public FileCacheStorage {
4343
Status remove(const FileCacheKey& key) override;
4444
Status change_key_meta_type(const FileCacheKey& key, const FileCacheType type,
4545
const size_t size) override;
46-
void load_blocks_directly_unlocked(BlockFileCache* _mgr, const FileCacheKey& key,
47-
std::lock_guard<std::mutex>& cache_lock) override;
46+
void load_file_blocks_directly_unlocked(BlockFileCache* mgr, const UInt128Wrapper& hash,
47+
int64_t tablet_id,
48+
std::lock_guard<std::mutex>& cache_lock) override;
4849
Status clear(std::string& msg) override;
4950
std::string get_local_file(const FileCacheKey& key) override;
5051

0 commit comments

Comments
 (0)