Skip to content

Commit b562478

Browse files
committed
[fix](be) Pass peer tablet IDs to startup cache restoration
### What problem does this PR solve? Issue Number: None Problem Summary: Peer cache requests carried a tablet ID in rowset_meta, but the server left CacheContext.tablet_id at zero. During startup, lazy cache restoration therefore queried the wrong metadata prefix and missed persisted blocks. Pass the supplied tablet ID to get_or_set while preserving the zero default for older clients that omit rowset_meta. Cover both request versions with a restart test, and verify that background restoration preserves cache accounting and the block objects already restored synchronously. ### Release note Peer reads can restore persisted cache blocks under the requested tablet before background cache loading finishes. ### Check List (For Author) - Test: Unit Test - Reproduced the peer startup miss before the fix; the legacy request passed. - Passed 65 related ASAN unit tests using run-be-ut.sh, including peer reads, cache metadata, all five lazy-restore cases, and both peer startup request versions. - Passed clang-format v16, check-format, build hygiene, and git diff --check. - clang-tidy: no diagnostics on modified source lines; test checks passed. The existing peer handler cognitive-complexity warning remains (58 > 50). - Used GLIBC_COMPATIBILITY=OFF for the local unit-test link, as in the preceding cache restoration validation; no production build options changed. - Behavior changed: Yes; peer requests use their supplied tablet ID for cache restoration. - Does this need documentation: No
1 parent f8f8121 commit b562478

3 files changed

Lines changed: 101 additions & 1 deletion

File tree

‎be/src/cloud/cloud_internal_service.cpp‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -724,6 +724,10 @@ Status handle_peer_file_cache_block_request(const PFetchPeerDataRequest* request
724724
io::CacheContext ctx {};
725725
io::ReadStatistics local_stats;
726726
ctx.stats = &local_stats;
727+
// Startup restore needs the caller's tablet; older clients omit rowset_meta.
728+
if (request->has_rowset_meta()) {
729+
ctx.tablet_id = request->rowset_meta().tablet_id();
730+
}
727731
const size_t file_size =
728732
request->has_file_size()
729733
? static_cast<size_t>(std::max<int64_t>(0, request->file_size()))

‎be/test/io/cache/block_file_cache_test_meta_store.cpp‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -892,6 +892,20 @@ TEST_P(FileCacheLazyRestoreTest, RestoreAllFileBlocksBeforeBackgroundLoad) {
892892
expected_offset += block_size;
893893
}
894894
EXPECT_EQ(cache->_cur_cache_size, 2 * block_size);
895+
896+
{
897+
std::lock_guard lock(mutex);
898+
resume_background = true;
899+
}
900+
cv.notify_all();
901+
auto* storage = static_cast<FSFileCacheStorage*>(cache->_storage.get());
902+
storage->_cache_background_load_thread.join();
903+
ASSERT_TRUE(cache->get_async_open_success());
904+
std::lock_guard cache_lock(cache->_mutex);
905+
EXPECT_EQ(cache->_cur_cache_size, 2 * block_size);
906+
for (const auto& block : blocks) {
907+
EXPECT_EQ(cache->_files.at(hash).at(block->offset()).file_block, block);
908+
}
895909
}
896910

897911
INSTANTIATE_TEST_SUITE_P(Startup, FileCacheLazyRestoreTest,

‎be/test/io/cache/cached_remote_file_reader_peer_test.cpp‎

Lines changed: 83 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -142,10 +142,12 @@ fs::path create_peer_test_file(const std::string& name, std::string_view content
142142
}
143143

144144
void populate_cache_block_with_content(BlockFileCache* cache, const fs::path& file_path,
145-
size_t offset, size_t size, std::string_view content) {
145+
size_t offset, size_t size, std::string_view content,
146+
int64_t tablet_id = 0) {
146147
ReadStatistics stats;
147148
CacheContext context;
148149
context.stats = &stats;
150+
context.tablet_id = tablet_id;
149151
auto hash = BlockFileCache::hash(file_path.filename().native());
150152
auto holder = cache->get_or_set(hash, offset, size, context);
151153
auto blocks = fromHolder(holder);
@@ -551,6 +553,9 @@ class CachedRemoteFileReaderPeerTest : public BlockFileCacheTest {
551553
bool _owned_internal_client_cache = false;
552554
};
553555

556+
class PeerFileCacheLazyRestoreTest : public CachedRemoteFileReaderPeerTest,
557+
public testing::WithParamInterface<bool> {};
558+
554559
} // namespace
555560

556561
TEST_F(CachedRemoteFileReaderPeerTest, read_at_uses_peer_cache_when_available) {
@@ -1005,6 +1010,83 @@ TEST_F(CachedRemoteFileReaderPeerTest, peer_file_cache_handler_allows_cached_blo
10051010
EXPECT_EQ(response.datas(0).data(), content.substr(0, kPeerTestBlockSize));
10061011
}
10071012

1013+
TEST_P(PeerFileCacheLazyRestoreTest, RestoreBlocksDuringStartup) {
1014+
const bool has_rowset_meta = GetParam();
1015+
const std::string content = "abcdefghijklmnop";
1016+
const fs::path file_path = "peer_lazy_restore.dat";
1017+
const auto hash = BlockFileCache::hash(file_path.filename().native());
1018+
const fs::path cache_path = caches_dir / "peer_lazy_restore_cache";
1019+
clear_cached_remote_reader_factory();
1020+
Defer cleanup_cache {[&]() {
1021+
clear_cached_remote_reader_factory();
1022+
fs::remove_all(cache_path);
1023+
}};
1024+
auto* cache = create_peer_test_cache(cache_path, kPeerTestBlockSize);
1025+
const int64_t tablet_id = has_rowset_meta ? kPeerTestTabletId : 0;
1026+
populate_cache_block_with_content(cache, file_path, kPeerTestBlockSize, kPeerTestBlockSize,
1027+
content, tablet_id);
1028+
populate_cache_block_with_content(cache, file_path, 2 * kPeerTestBlockSize, kPeerTestBlockSize,
1029+
content, tablet_id);
1030+
// Closing the cache drains metadata writes before the restarted instance opens it.
1031+
clear_cached_remote_reader_factory();
1032+
1033+
std::mutex mutex;
1034+
std::condition_variable cv;
1035+
bool background_paused = false;
1036+
bool resume_background = false;
1037+
auto* sync_point = SyncPoint::get_instance();
1038+
SyncPoint::CallbackGuard guard;
1039+
sync_point->set_call_back(
1040+
"BlockFileCache::TmpFile1",
1041+
[&](auto&&) {
1042+
std::unique_lock lock(mutex);
1043+
background_paused = true;
1044+
cv.notify_all();
1045+
cv.wait(lock, [&] { return resume_background; });
1046+
},
1047+
&guard);
1048+
Defer resume_and_close {[&]() {
1049+
{
1050+
std::lock_guard lock(mutex);
1051+
resume_background = true;
1052+
}
1053+
cv.notify_all();
1054+
clear_cached_remote_reader_factory();
1055+
sync_point->disable_processing();
1056+
}};
1057+
sync_point->enable_processing();
1058+
ASSERT_TRUE(FileCacheFactory::instance()
1059+
->create_file_cache(cache_path.string(),
1060+
create_peer_test_settings(kPeerTestBlockSize))
1061+
.ok());
1062+
cache = FileCacheFactory::instance()->get_by_path(hash);
1063+
{
1064+
std::unique_lock lock(mutex);
1065+
ASSERT_TRUE(cv.wait_for(lock, std::chrono::seconds(10), [&] { return background_paused; }));
1066+
}
1067+
ASSERT_FALSE(cache->get_async_open_success());
1068+
ASSERT_FALSE(cache->_files.contains(hash));
1069+
1070+
auto request = create_peer_cache_block_request(file_path, kPeerTestBlockSize,
1071+
kPeerTestBlockSize, content.size());
1072+
if (has_rowset_meta) {
1073+
request.mutable_rowset_meta()->set_rowset_id(0);
1074+
request.mutable_rowset_meta()->set_tablet_id(kPeerTestTabletId);
1075+
}
1076+
PFetchPeerDataResponse response;
1077+
brpc::Controller cntl;
1078+
const auto st = doris::test_handle_peer_file_cache_block_request(&request, &response, &cntl);
1079+
ASSERT_TRUE(st.ok()) << st;
1080+
ASSERT_EQ(response.datas_size(), 1);
1081+
EXPECT_EQ(response.datas(0).data(), content.substr(kPeerTestBlockSize, kPeerTestBlockSize));
1082+
// A request for one block must also restore the other persisted offset of this file.
1083+
ASSERT_EQ(cache->_files.at(hash).size(), 2);
1084+
EXPECT_EQ(cache->_files.at(hash).at(2 * kPeerTestBlockSize).file_block->state(),
1085+
FileBlock::State::DOWNLOADED);
1086+
}
1087+
1088+
INSTANTIATE_TEST_SUITE_P(Startup, PeerFileCacheLazyRestoreTest, testing::Bool());
1089+
10081090
TEST_F(CachedRemoteFileReaderPeerTest, peer_fetch_queue_timeout_rejects_expired_request) {
10091091
const int32_t old_peer_fetch_queue_timeout_ms = config::peer_fetch_queue_timeout_ms;
10101092
Defer restore_config {

0 commit comments

Comments
 (0)