@@ -492,7 +492,6 @@ void test_file_cache(io::CacheType cache_type) {
492492 assert_range (38 , segments[2 ], io::FileSegment::Range (24 , 26 ),
493493 io::FileSegment::State::DOWNLOADED );
494494
495-
496495 bool lets_start_download = false ;
497496 std::mutex mutex;
498497 std::condition_variable cv;
@@ -538,7 +537,7 @@ void test_file_cache(io::CacheType cache_type) {
538537 }
539538 }
540539 // / Current cache: [__________][___][___][_][__]
541- // / ^ ^ ^ ^ ^ ^
540+ // / ^ ^ ^ ^ ^ ^
542541 // / 0 9 24 26 27 29
543542 {
544543 // / Test LRUCache::restore().
@@ -587,6 +586,9 @@ void test_file_cache(io::CacheType cache_type) {
587586}
588587
589588TEST (LRUFileCache, normal) {
589+ if (fs::exists (cache_base_path)) {
590+ fs::remove_all (cache_base_path);
591+ }
590592 fs::create_directories (cache_base_path);
591593 test_file_cache (io::CacheType::DISPOSABLE );
592594 if (fs::exists (cache_base_path)) {
@@ -607,10 +609,13 @@ TEST(LRUFileCache, normal) {
607609}
608610
609611TEST (LRUFileCache, resize) {
612+ if (fs::exists (cache_base_path)) {
613+ fs::remove_all (cache_base_path);
614+ }
610615 fs::create_directories (cache_base_path);
611616 test_file_cache (io::CacheType::INDEX );
612617 // / Current cache: [__________][___][___][_][__]
613- // / ^ ^ ^ ^ ^ ^
618+ // / ^ ^ ^ ^ ^ ^
614619 // / 0 9 24 26 27 29
615620 io::FileCacheSettings settings;
616621 settings.index_queue_elements = 5 ;
@@ -627,4 +632,208 @@ TEST(LRUFileCache, resize) {
627632 }
628633}
629634
635+ TEST (LRUFileCache, query_limit_heap_use_after_free) {
636+ if (fs::exists (cache_base_path)) {
637+ fs::remove_all (cache_base_path);
638+ }
639+ doris::config::enable_file_cache_query_limit = true ;
640+ fs::create_directories (cache_base_path);
641+ io::FileCacheSettings settings;
642+ settings.index_queue_elements = 0 ;
643+ settings.index_queue_size = 0 ;
644+ settings.disposable_queue_size = 0 ;
645+ settings.disposable_queue_elements = 0 ;
646+ settings.query_queue_size = 15 ;
647+ settings.query_queue_elements = 5 ;
648+ settings.max_file_segment_size = 10 ;
649+ settings.max_query_cache_size = 15 ;
650+ settings.total_size = 15 ;
651+ io::CloudFileCache cache (cache_base_path, settings);
652+ ASSERT_TRUE (cache.initialize ());
653+ io::CacheContext context;
654+ context.cache_type = io::CacheType::NORMAL ;
655+ auto key = io::CloudFileCache::hash (" key1" );
656+ {
657+ auto holder = cache.get_or_set (key, 0 , 9 , context); // / Add range [0, 8]
658+ auto segments = fromHolder (holder);
659+ ASSERT_GE (segments.size (), 1 );
660+ assert_range (1 , segments[0 ], io::FileSegment::Range (0 , 8 ), io::FileSegment::State::EMPTY );
661+ ASSERT_TRUE (segments[0 ]->get_or_set_downloader () == io::FileSegment::get_caller_id ());
662+ assert_range (2 , segments[0 ], io::FileSegment::Range (0 , 8 ),
663+ io::FileSegment::State::DOWNLOADING );
664+ download (segments[0 ]);
665+ }
666+ TUniqueId query_id;
667+ query_id.hi = 1 ;
668+ query_id.lo = 1 ;
669+ context.query_id = query_id;
670+ auto query_context_holder = cache.get_query_context_holder (query_id);
671+ {
672+ auto holder = cache.get_or_set (key, 9 , 1 , context); // / Add range [9, 9]
673+ auto segments = fromHolder (holder);
674+ ASSERT_GE (segments.size (), 1 );
675+ assert_range (1 , segments[0 ], io::FileSegment::Range (9 , 9 ), io::FileSegment::State::EMPTY );
676+ ASSERT_TRUE (segments[0 ]->get_or_set_downloader () == io::FileSegment::get_caller_id ());
677+ assert_range (2 , segments[0 ], io::FileSegment::Range (9 , 9 ),
678+ io::FileSegment::State::DOWNLOADING );
679+ download (segments[0 ]);
680+ }
681+ {
682+ auto holder = cache.get_or_set (key, 10 , 5 , context); // / Add range [10, 14]
683+ auto segments = fromHolder (holder);
684+ ASSERT_GE (segments.size (), 1 );
685+ assert_range (3 , segments[0 ], io::FileSegment::Range (10 , 14 ), io::FileSegment::State::EMPTY );
686+ ASSERT_TRUE (segments[0 ]->get_or_set_downloader () == io::FileSegment::get_caller_id ());
687+ assert_range (4 , segments[0 ], io::FileSegment::Range (10 , 14 ),
688+ io::FileSegment::State::DOWNLOADING );
689+ download (segments[0 ]);
690+ }
691+ {
692+ auto holder = cache.get_or_set (key, 0 , 9 , context); // / Add range [0, 8]
693+ auto segments = fromHolder (holder);
694+ ASSERT_GE (segments.size (), 1 );
695+ assert_range (5 , segments[0 ], io::FileSegment::Range (0 , 8 ),
696+ io::FileSegment::State::DOWNLOADED );
697+ }
698+ {
699+ auto holder = cache.get_or_set (key, 15 , 1 , context); // / Add range [15, 15]
700+ auto segments = fromHolder (holder);
701+ ASSERT_GE (segments.size (), 1 );
702+ assert_range (6 , segments[0 ], io::FileSegment::Range (15 , 15 ), io::FileSegment::State::EMPTY );
703+ ASSERT_TRUE (segments[0 ]->get_or_set_downloader () == io::FileSegment::get_caller_id ());
704+ assert_range (7 , segments[0 ], io::FileSegment::Range (15 , 15 ),
705+ io::FileSegment::State::DOWNLOADING );
706+ download (segments[0 ]);
707+ }
708+ {
709+ auto holder = cache.get_or_set (key, 16 , 9 , context); // / Add range [16, 24]
710+ auto segments = fromHolder (holder);
711+ ASSERT_GE (segments.size (), 1 );
712+ assert_range (8 , segments[0 ], io::FileSegment::Range (16 , 24 ),
713+ io::FileSegment::State::SKIP_CACHE );
714+ }
715+ if (fs::exists (cache_base_path)) {
716+ fs::remove_all (cache_base_path);
717+ }
718+ }
719+
720+ TEST (LRUFileCache, query_limit_dcheck) {
721+ if (fs::exists (cache_base_path)) {
722+ fs::remove_all (cache_base_path);
723+ }
724+ doris::config::enable_file_cache_query_limit = true ;
725+ fs::create_directories (cache_base_path);
726+ io::FileCacheSettings settings;
727+ settings.index_queue_elements = 0 ;
728+ settings.index_queue_size = 0 ;
729+ settings.disposable_queue_size = 0 ;
730+ settings.disposable_queue_elements = 0 ;
731+ settings.query_queue_size = 15 ;
732+ settings.query_queue_elements = 5 ;
733+ settings.max_file_segment_size = 10 ;
734+ settings.max_query_cache_size = 15 ;
735+ settings.total_size = 15 ;
736+ io::CloudFileCache cache (cache_base_path, settings);
737+ ASSERT_TRUE (cache.initialize ());
738+ io::CacheContext context;
739+ context.cache_type = io::CacheType::NORMAL ;
740+ auto key = io::CloudFileCache::hash (" key1" );
741+ {
742+ auto holder = cache.get_or_set (key, 0 , 9 , context); // / Add range [0, 8]
743+ auto segments = fromHolder (holder);
744+ ASSERT_GE (segments.size (), 1 );
745+ assert_range (1 , segments[0 ], io::FileSegment::Range (0 , 8 ), io::FileSegment::State::EMPTY );
746+ ASSERT_TRUE (segments[0 ]->get_or_set_downloader () == io::FileSegment::get_caller_id ());
747+ assert_range (2 , segments[0 ], io::FileSegment::Range (0 , 8 ),
748+ io::FileSegment::State::DOWNLOADING );
749+ download (segments[0 ]);
750+ }
751+ TUniqueId query_id;
752+ query_id.hi = 1 ;
753+ query_id.lo = 1 ;
754+ context.query_id = query_id;
755+ auto query_context_holder = cache.get_query_context_holder (query_id);
756+ {
757+ auto holder = cache.get_or_set (key, 9 , 1 , context); // / Add range [9, 9]
758+ auto segments = fromHolder (holder);
759+ ASSERT_GE (segments.size (), 1 );
760+ assert_range (1 , segments[0 ], io::FileSegment::Range (9 , 9 ), io::FileSegment::State::EMPTY );
761+ ASSERT_TRUE (segments[0 ]->get_or_set_downloader () == io::FileSegment::get_caller_id ());
762+ assert_range (2 , segments[0 ], io::FileSegment::Range (9 , 9 ),
763+ io::FileSegment::State::DOWNLOADING );
764+ download (segments[0 ]);
765+ }
766+ {
767+ auto holder = cache.get_or_set (key, 10 , 5 , context); // / Add range [10, 14]
768+ auto segments = fromHolder (holder);
769+ ASSERT_GE (segments.size (), 1 );
770+ assert_range (3 , segments[0 ], io::FileSegment::Range (10 , 14 ), io::FileSegment::State::EMPTY );
771+ ASSERT_TRUE (segments[0 ]->get_or_set_downloader () == io::FileSegment::get_caller_id ());
772+ assert_range (4 , segments[0 ], io::FileSegment::Range (10 , 14 ),
773+ io::FileSegment::State::DOWNLOADING );
774+ download (segments[0 ]);
775+ }
776+ {
777+ auto holder = cache.get_or_set (key, 0 , 9 , context); // / Add range [0, 8]
778+ auto segments = fromHolder (holder);
779+ ASSERT_GE (segments.size (), 1 );
780+ assert_range (5 , segments[0 ], io::FileSegment::Range (0 , 8 ),
781+ io::FileSegment::State::DOWNLOADED );
782+ }
783+ {
784+ auto holder = cache.get_or_set (key, 15 , 1 , context); // / Add range [15, 15]
785+ auto segments = fromHolder (holder);
786+ ASSERT_GE (segments.size (), 1 );
787+ assert_range (6 , segments[0 ], io::FileSegment::Range (15 , 15 ), io::FileSegment::State::EMPTY );
788+ ASSERT_TRUE (segments[0 ]->get_or_set_downloader () == io::FileSegment::get_caller_id ());
789+ assert_range (7 , segments[0 ], io::FileSegment::Range (15 , 15 ),
790+ io::FileSegment::State::DOWNLOADING );
791+ download (segments[0 ]);
792+ }
793+ // double add
794+ {
795+ auto holder = cache.get_or_set (key, 9 , 1 , context); // / Add range [9, 9]
796+ auto segments = fromHolder (holder);
797+ ASSERT_GE (segments.size (), 1 );
798+ assert_range (1 , segments[0 ], io::FileSegment::Range (9 , 9 ), io::FileSegment::State::EMPTY );
799+ ASSERT_TRUE (segments[0 ]->get_or_set_downloader () == io::FileSegment::get_caller_id ());
800+ assert_range (2 , segments[0 ], io::FileSegment::Range (9 , 9 ),
801+ io::FileSegment::State::DOWNLOADING );
802+ download (segments[0 ]);
803+ }
804+ {
805+ auto holder = cache.get_or_set (key, 30 , 5 , context); // / Add range [30, 34]
806+ auto segments = fromHolder (holder);
807+ ASSERT_GE (segments.size (), 1 );
808+ assert_range (1 , segments[0 ], io::FileSegment::Range (30 , 34 ), io::FileSegment::State::EMPTY );
809+ ASSERT_TRUE (segments[0 ]->get_or_set_downloader () == io::FileSegment::get_caller_id ());
810+ assert_range (2 , segments[0 ], io::FileSegment::Range (30 , 34 ),
811+ io::FileSegment::State::DOWNLOADING );
812+ download (segments[0 ]);
813+ }
814+ {
815+ auto holder = cache.get_or_set (key, 40 , 5 , context); // / Add range [40, 44]
816+ auto segments = fromHolder (holder);
817+ ASSERT_GE (segments.size (), 1 );
818+ assert_range (1 , segments[0 ], io::FileSegment::Range (40 , 44 ), io::FileSegment::State::EMPTY );
819+ ASSERT_TRUE (segments[0 ]->get_or_set_downloader () == io::FileSegment::get_caller_id ());
820+ assert_range (2 , segments[0 ], io::FileSegment::Range (40 , 44 ),
821+ io::FileSegment::State::DOWNLOADING );
822+ download (segments[0 ]);
823+ }
824+ {
825+ auto holder = cache.get_or_set (key, 50 , 5 , context); // / Add range [50, 54]
826+ auto segments = fromHolder (holder);
827+ ASSERT_GE (segments.size (), 1 );
828+ assert_range (1 , segments[0 ], io::FileSegment::Range (50 , 54 ), io::FileSegment::State::EMPTY );
829+ ASSERT_TRUE (segments[0 ]->get_or_set_downloader () == io::FileSegment::get_caller_id ());
830+ assert_range (2 , segments[0 ], io::FileSegment::Range (50 , 54 ),
831+ io::FileSegment::State::DOWNLOADING );
832+ download (segments[0 ]);
833+ }
834+ if (fs::exists (cache_base_path)) {
835+ fs::remove_all (cache_base_path);
836+ }
837+ }
838+
630839} // namespace doris::cloud
0 commit comments