Skip to content

Commit 6c92c12

Browse files
authored
[feature](selectdb-cloud) Fix the coredump when eliminating cache because encounting query limit (apache#1583)
The pr fix three problems. 1. When eliminating ghost, it will cause heap-use-after-free. 2. When adding a segment twice, it will coredump because of decheck. 3. If the disposable path is same as the cache path , it will rm the cache in disk.
1 parent c01d3b6 commit 6c92c12

3 files changed

Lines changed: 246 additions & 21 deletions

File tree

‎be/src/cloud/io/cloud_file_cache.cpp‎

Lines changed: 23 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -144,16 +144,24 @@ CloudFileCache::QueryContextPtr CloudFileCache::get_or_set_query_context(
144144

145145
void CloudFileCache::QueryContext::remove(const Key& key, size_t offset,
146146
std::lock_guard<std::mutex>& cache_lock) {
147-
auto record = records.find({key, offset});
147+
// The key maybe comes from the 'lru_queue'
148+
// and we need to erase the 'records' first.
149+
// If not, it will cause heap-use-after-free
150+
auto pair = std::make_pair(key, offset);
151+
auto record = records.find(pair);
148152
DCHECK(record != records.end());
149-
lru_queue.remove(record->second, cache_lock);
150-
records.erase({key, offset});
153+
auto iter = record->second;
154+
records.erase(pair);
155+
lru_queue.remove(iter, cache_lock);
151156
}
152157

153158
void CloudFileCache::QueryContext::reserve(const Key& key, size_t offset, size_t size,
154159
std::lock_guard<std::mutex>& cache_lock) {
155-
auto queue_iter = lru_queue.add(key, offset, size, cache_lock);
156-
records.insert({{key, offset}, queue_iter});
160+
auto pair = std::make_pair(key, offset);
161+
if (records.find(pair) == records.end()) {
162+
auto queue_iter = lru_queue.add(key, offset, size, cache_lock);
163+
records.insert({pair, queue_iter});
164+
}
157165
}
158166

159167
Status CloudFileCache::initialize() {
@@ -593,7 +601,8 @@ const CloudFileCache::LRUQueue& CloudFileCache::get_queue(CacheType type) const
593601

594602
bool CloudFileCache::try_reserve_for_ttl(size_t size, std::lock_guard<std::mutex>& cache_lock) {
595603
size_t removed_size = 0;
596-
auto is_overflow = [&] { return _cur_cache_size + size - removed_size > _total_size; };
604+
size_t cur_cache_size = _cur_cache_size;
605+
auto is_overflow = [&] { return cur_cache_size + size - removed_size > _total_size; };
597606
auto remove_file_segment_if = [&](FileSegmentCell* cell) {
598607
FileSegmentSPtr file_segment = cell->file_segment;
599608
if (file_segment) {
@@ -695,20 +704,19 @@ bool CloudFileCache::try_reserve(const Key& key, const CacheContext& context, si
695704
.count();
696705
auto& queue = get_queue(context.cache_type);
697706
size_t removed_size = 0;
698-
size_t queue_element_size = queue.get_elements_num(cache_lock);
699707
size_t queue_size = queue.get_total_cache_size(cache_lock);
708+
size_t cur_cache_size = _cur_cache_size;
709+
size_t query_context_cache_size = query_context->get_cache_size(cache_lock);
700710

701711
std::vector<CloudFileCache::LRUQueue::Iterator> ghost;
702712
std::vector<FileSegmentCell*> trash;
703713
std::vector<FileSegmentCell*> to_evict;
704714

705715
size_t max_size = queue.get_max_size();
706-
size_t max_element_size = queue.get_max_element_size();
707716
auto is_overflow = [&] {
708-
return _cur_cache_size + size - removed_size > _total_size ||
717+
return cur_cache_size + size - removed_size > _total_size ||
709718
(queue_size + size - removed_size > max_size) ||
710-
queue_element_size >= max_element_size ||
711-
(query_context->get_cache_size(cache_lock) + size - removed_size >
719+
(query_context_cache_size + size - removed_size >
712720
query_context->get_max_cache_size());
713721
};
714722

@@ -744,7 +752,6 @@ bool CloudFileCache::try_reserve(const Key& key, const CacheContext& context, si
744752
}
745753
}
746754
removed_size += cell_size;
747-
--queue_element_size;
748755
}
749756
}
750757
}
@@ -881,7 +888,8 @@ bool CloudFileCache::try_reserve_from_other_queue(CacheType cur_cache_type, size
881888
std::lock_guard<std::mutex>& cache_lock) {
882889
auto other_cache_types = get_other_cache_type(cur_cache_type);
883890
size_t removed_size = 0;
884-
auto is_overflow = [&] { return _cur_cache_size + size - removed_size > _total_size; };
891+
size_t cur_cache_size = _cur_cache_size;
892+
auto is_overflow = [&] { return cur_cache_size + size - removed_size > _total_size; };
885893
std::vector<FileSegmentCell*> to_evict;
886894
std::vector<FileSegmentCell*> trash;
887895
for (CacheType cache_type : other_cache_types) {
@@ -950,11 +958,12 @@ bool CloudFileCache::try_reserve_for_lru(const Key& key, QueryContextPtr query_c
950958
size_t removed_size = 0;
951959
size_t queue_element_size = queue.get_elements_num(cache_lock);
952960
size_t queue_size = queue.get_total_cache_size(cache_lock);
961+
size_t cur_cache_size = _cur_cache_size;
953962

954963
size_t max_size = queue.get_max_size();
955964
size_t max_element_size = queue.get_max_element_size();
956965
auto is_overflow = [&] {
957-
return _cur_cache_size + size - removed_size > _total_size ||
966+
return cur_cache_size + size - removed_size > _total_size ||
958967
(queue_size + size - removed_size > max_size) ||
959968
queue_element_size >= max_element_size;
960969
};

‎be/src/service/doris_main.cpp‎

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@
2727
#include <thread>
2828
#include <unordered_map>
2929

30+
#include "cloud/io/cloud_file_cache_profile.h"
3031
#include "util/jni-util.h"
3132

3233
#if defined(LEAK_SANITIZER)
@@ -405,8 +406,8 @@ int main(int argc, char** argv) {
405406
}
406407
for (auto& cache_path : cache_paths) {
407408
if (cache_path_set.find(cache_path.path) != cache_path_set.end()) {
408-
LOG(FATAL) << fmt::format("cache path {} is duplicate", cache_path.path);
409-
exit(-1);
409+
LOG(WARNING) << fmt::format("cache path {} is duplicate", cache_path.path);
410+
continue;
410411
}
411412
Status st = doris::io::FileCacheFactory::instance().create_file_cache(
412413
cache_path.path, cache_path.init_settings());
@@ -415,15 +416,21 @@ int main(int argc, char** argv) {
415416
exit(-1);
416417
}
417418
}
419+
// use to init the 0 table profile for cloud mgr
420+
doris::io::FileCacheProfile::instance();
418421
std::vector<std::string> rm_paths;
419422
olap_res = doris::parse_conf_rm_paths(doris::config::disposable_file_cache_path, rm_paths);
420423
if (!olap_res) {
421424
LOG(FATAL) << "parse config disposable_file_cache_path path failed, path="
422425
<< doris::config::disposable_file_cache_path;
423426
exit(-1);
424427
}
425-
std::for_each(rm_paths.begin(), rm_paths.end(), [](const std::string& path) {
426-
doris::io::global_local_filesystem()->delete_directory(path);
428+
std::for_each(rm_paths.begin(), rm_paths.end(), [&cache_path_set](const std::string& path) {
429+
if (cache_path_set.find(path) == cache_path_set.end()) {
430+
doris::io::global_local_filesystem()->delete_directory(path);
431+
} else {
432+
LOG(WARNING) << fmt::format("disposable cache path {} is duplicate.", path);
433+
}
427434
});
428435
}
429436

‎be/test/cloud/cloud_file_cache_test.cpp‎

Lines changed: 212 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -492,7 +492,6 @@ void test_file_cache(io::CacheType cache_type) {
492492
assert_range(38, segments[2], io::FileSegment::Range(24, 26),
493493
io::FileSegment::State::DOWNLOADED);
494494

495-
496495
bool lets_start_download = false;
497496
std::mutex mutex;
498497
std::condition_variable cv;
@@ -538,7 +537,7 @@ void test_file_cache(io::CacheType cache_type) {
538537
}
539538
}
540539
/// Current cache: [__________][___][___][_][__]
541-
/// ^ ^ ^ ^ ^ ^
540+
/// ^ ^ ^ ^ ^ ^
542541
/// 0 9 24 26 27 29
543542
{
544543
/// Test LRUCache::restore().
@@ -587,6 +586,9 @@ void test_file_cache(io::CacheType cache_type) {
587586
}
588587

589588
TEST(LRUFileCache, normal) {
589+
if (fs::exists(cache_base_path)) {
590+
fs::remove_all(cache_base_path);
591+
}
590592
fs::create_directories(cache_base_path);
591593
test_file_cache(io::CacheType::DISPOSABLE);
592594
if (fs::exists(cache_base_path)) {
@@ -607,10 +609,13 @@ TEST(LRUFileCache, normal) {
607609
}
608610

609611
TEST(LRUFileCache, resize) {
612+
if (fs::exists(cache_base_path)) {
613+
fs::remove_all(cache_base_path);
614+
}
610615
fs::create_directories(cache_base_path);
611616
test_file_cache(io::CacheType::INDEX);
612617
/// Current cache: [__________][___][___][_][__]
613-
/// ^ ^ ^ ^ ^ ^
618+
/// ^ ^ ^ ^ ^ ^
614619
/// 0 9 24 26 27 29
615620
io::FileCacheSettings settings;
616621
settings.index_queue_elements = 5;
@@ -627,4 +632,208 @@ TEST(LRUFileCache, resize) {
627632
}
628633
}
629634

635+
TEST(LRUFileCache, query_limit_heap_use_after_free) {
636+
if (fs::exists(cache_base_path)) {
637+
fs::remove_all(cache_base_path);
638+
}
639+
doris::config::enable_file_cache_query_limit = true;
640+
fs::create_directories(cache_base_path);
641+
io::FileCacheSettings settings;
642+
settings.index_queue_elements = 0;
643+
settings.index_queue_size = 0;
644+
settings.disposable_queue_size = 0;
645+
settings.disposable_queue_elements = 0;
646+
settings.query_queue_size = 15;
647+
settings.query_queue_elements = 5;
648+
settings.max_file_segment_size = 10;
649+
settings.max_query_cache_size = 15;
650+
settings.total_size = 15;
651+
io::CloudFileCache cache(cache_base_path, settings);
652+
ASSERT_TRUE(cache.initialize());
653+
io::CacheContext context;
654+
context.cache_type = io::CacheType::NORMAL;
655+
auto key = io::CloudFileCache::hash("key1");
656+
{
657+
auto holder = cache.get_or_set(key, 0, 9, context); /// Add range [0, 8]
658+
auto segments = fromHolder(holder);
659+
ASSERT_GE(segments.size(), 1);
660+
assert_range(1, segments[0], io::FileSegment::Range(0, 8), io::FileSegment::State::EMPTY);
661+
ASSERT_TRUE(segments[0]->get_or_set_downloader() == io::FileSegment::get_caller_id());
662+
assert_range(2, segments[0], io::FileSegment::Range(0, 8),
663+
io::FileSegment::State::DOWNLOADING);
664+
download(segments[0]);
665+
}
666+
TUniqueId query_id;
667+
query_id.hi = 1;
668+
query_id.lo = 1;
669+
context.query_id = query_id;
670+
auto query_context_holder = cache.get_query_context_holder(query_id);
671+
{
672+
auto holder = cache.get_or_set(key, 9, 1, context); /// Add range [9, 9]
673+
auto segments = fromHolder(holder);
674+
ASSERT_GE(segments.size(), 1);
675+
assert_range(1, segments[0], io::FileSegment::Range(9, 9), io::FileSegment::State::EMPTY);
676+
ASSERT_TRUE(segments[0]->get_or_set_downloader() == io::FileSegment::get_caller_id());
677+
assert_range(2, segments[0], io::FileSegment::Range(9, 9),
678+
io::FileSegment::State::DOWNLOADING);
679+
download(segments[0]);
680+
}
681+
{
682+
auto holder = cache.get_or_set(key, 10, 5, context); /// Add range [10, 14]
683+
auto segments = fromHolder(holder);
684+
ASSERT_GE(segments.size(), 1);
685+
assert_range(3, segments[0], io::FileSegment::Range(10, 14), io::FileSegment::State::EMPTY);
686+
ASSERT_TRUE(segments[0]->get_or_set_downloader() == io::FileSegment::get_caller_id());
687+
assert_range(4, segments[0], io::FileSegment::Range(10, 14),
688+
io::FileSegment::State::DOWNLOADING);
689+
download(segments[0]);
690+
}
691+
{
692+
auto holder = cache.get_or_set(key, 0, 9, context); /// Add range [0, 8]
693+
auto segments = fromHolder(holder);
694+
ASSERT_GE(segments.size(), 1);
695+
assert_range(5, segments[0], io::FileSegment::Range(0, 8),
696+
io::FileSegment::State::DOWNLOADED);
697+
}
698+
{
699+
auto holder = cache.get_or_set(key, 15, 1, context); /// Add range [15, 15]
700+
auto segments = fromHolder(holder);
701+
ASSERT_GE(segments.size(), 1);
702+
assert_range(6, segments[0], io::FileSegment::Range(15, 15), io::FileSegment::State::EMPTY);
703+
ASSERT_TRUE(segments[0]->get_or_set_downloader() == io::FileSegment::get_caller_id());
704+
assert_range(7, segments[0], io::FileSegment::Range(15, 15),
705+
io::FileSegment::State::DOWNLOADING);
706+
download(segments[0]);
707+
}
708+
{
709+
auto holder = cache.get_or_set(key, 16, 9, context); /// Add range [16, 24]
710+
auto segments = fromHolder(holder);
711+
ASSERT_GE(segments.size(), 1);
712+
assert_range(8, segments[0], io::FileSegment::Range(16, 24),
713+
io::FileSegment::State::SKIP_CACHE);
714+
}
715+
if (fs::exists(cache_base_path)) {
716+
fs::remove_all(cache_base_path);
717+
}
718+
}
719+
720+
TEST(LRUFileCache, query_limit_dcheck) {
721+
if (fs::exists(cache_base_path)) {
722+
fs::remove_all(cache_base_path);
723+
}
724+
doris::config::enable_file_cache_query_limit = true;
725+
fs::create_directories(cache_base_path);
726+
io::FileCacheSettings settings;
727+
settings.index_queue_elements = 0;
728+
settings.index_queue_size = 0;
729+
settings.disposable_queue_size = 0;
730+
settings.disposable_queue_elements = 0;
731+
settings.query_queue_size = 15;
732+
settings.query_queue_elements = 5;
733+
settings.max_file_segment_size = 10;
734+
settings.max_query_cache_size = 15;
735+
settings.total_size = 15;
736+
io::CloudFileCache cache(cache_base_path, settings);
737+
ASSERT_TRUE(cache.initialize());
738+
io::CacheContext context;
739+
context.cache_type = io::CacheType::NORMAL;
740+
auto key = io::CloudFileCache::hash("key1");
741+
{
742+
auto holder = cache.get_or_set(key, 0, 9, context); /// Add range [0, 8]
743+
auto segments = fromHolder(holder);
744+
ASSERT_GE(segments.size(), 1);
745+
assert_range(1, segments[0], io::FileSegment::Range(0, 8), io::FileSegment::State::EMPTY);
746+
ASSERT_TRUE(segments[0]->get_or_set_downloader() == io::FileSegment::get_caller_id());
747+
assert_range(2, segments[0], io::FileSegment::Range(0, 8),
748+
io::FileSegment::State::DOWNLOADING);
749+
download(segments[0]);
750+
}
751+
TUniqueId query_id;
752+
query_id.hi = 1;
753+
query_id.lo = 1;
754+
context.query_id = query_id;
755+
auto query_context_holder = cache.get_query_context_holder(query_id);
756+
{
757+
auto holder = cache.get_or_set(key, 9, 1, context); /// Add range [9, 9]
758+
auto segments = fromHolder(holder);
759+
ASSERT_GE(segments.size(), 1);
760+
assert_range(1, segments[0], io::FileSegment::Range(9, 9), io::FileSegment::State::EMPTY);
761+
ASSERT_TRUE(segments[0]->get_or_set_downloader() == io::FileSegment::get_caller_id());
762+
assert_range(2, segments[0], io::FileSegment::Range(9, 9),
763+
io::FileSegment::State::DOWNLOADING);
764+
download(segments[0]);
765+
}
766+
{
767+
auto holder = cache.get_or_set(key, 10, 5, context); /// Add range [10, 14]
768+
auto segments = fromHolder(holder);
769+
ASSERT_GE(segments.size(), 1);
770+
assert_range(3, segments[0], io::FileSegment::Range(10, 14), io::FileSegment::State::EMPTY);
771+
ASSERT_TRUE(segments[0]->get_or_set_downloader() == io::FileSegment::get_caller_id());
772+
assert_range(4, segments[0], io::FileSegment::Range(10, 14),
773+
io::FileSegment::State::DOWNLOADING);
774+
download(segments[0]);
775+
}
776+
{
777+
auto holder = cache.get_or_set(key, 0, 9, context); /// Add range [0, 8]
778+
auto segments = fromHolder(holder);
779+
ASSERT_GE(segments.size(), 1);
780+
assert_range(5, segments[0], io::FileSegment::Range(0, 8),
781+
io::FileSegment::State::DOWNLOADED);
782+
}
783+
{
784+
auto holder = cache.get_or_set(key, 15, 1, context); /// Add range [15, 15]
785+
auto segments = fromHolder(holder);
786+
ASSERT_GE(segments.size(), 1);
787+
assert_range(6, segments[0], io::FileSegment::Range(15, 15), io::FileSegment::State::EMPTY);
788+
ASSERT_TRUE(segments[0]->get_or_set_downloader() == io::FileSegment::get_caller_id());
789+
assert_range(7, segments[0], io::FileSegment::Range(15, 15),
790+
io::FileSegment::State::DOWNLOADING);
791+
download(segments[0]);
792+
}
793+
// double add
794+
{
795+
auto holder = cache.get_or_set(key, 9, 1, context); /// Add range [9, 9]
796+
auto segments = fromHolder(holder);
797+
ASSERT_GE(segments.size(), 1);
798+
assert_range(1, segments[0], io::FileSegment::Range(9, 9), io::FileSegment::State::EMPTY);
799+
ASSERT_TRUE(segments[0]->get_or_set_downloader() == io::FileSegment::get_caller_id());
800+
assert_range(2, segments[0], io::FileSegment::Range(9, 9),
801+
io::FileSegment::State::DOWNLOADING);
802+
download(segments[0]);
803+
}
804+
{
805+
auto holder = cache.get_or_set(key, 30, 5, context); /// Add range [30, 34]
806+
auto segments = fromHolder(holder);
807+
ASSERT_GE(segments.size(), 1);
808+
assert_range(1, segments[0], io::FileSegment::Range(30, 34), io::FileSegment::State::EMPTY);
809+
ASSERT_TRUE(segments[0]->get_or_set_downloader() == io::FileSegment::get_caller_id());
810+
assert_range(2, segments[0], io::FileSegment::Range(30, 34),
811+
io::FileSegment::State::DOWNLOADING);
812+
download(segments[0]);
813+
}
814+
{
815+
auto holder = cache.get_or_set(key, 40, 5, context); /// Add range [40, 44]
816+
auto segments = fromHolder(holder);
817+
ASSERT_GE(segments.size(), 1);
818+
assert_range(1, segments[0], io::FileSegment::Range(40, 44), io::FileSegment::State::EMPTY);
819+
ASSERT_TRUE(segments[0]->get_or_set_downloader() == io::FileSegment::get_caller_id());
820+
assert_range(2, segments[0], io::FileSegment::Range(40, 44),
821+
io::FileSegment::State::DOWNLOADING);
822+
download(segments[0]);
823+
}
824+
{
825+
auto holder = cache.get_or_set(key, 50, 5, context); /// Add range [50, 54]
826+
auto segments = fromHolder(holder);
827+
ASSERT_GE(segments.size(), 1);
828+
assert_range(1, segments[0], io::FileSegment::Range(50, 54), io::FileSegment::State::EMPTY);
829+
ASSERT_TRUE(segments[0]->get_or_set_downloader() == io::FileSegment::get_caller_id());
830+
assert_range(2, segments[0], io::FileSegment::Range(50, 54),
831+
io::FileSegment::State::DOWNLOADING);
832+
download(segments[0]);
833+
}
834+
if (fs::exists(cache_base_path)) {
835+
fs::remove_all(cache_base_path);
836+
}
837+
}
838+
630839
} // namespace doris::cloud

0 commit comments

Comments
 (0)