Skip to content

Commit 5108dd2

Browse files
committed
[fix](regression) Scope condition cache assertions to query profiles
### What problem does this PR solve? Issue Number: None Related PR: #68672 Problem Summary: The eligibility regression compares BE-wide condition-cache lookup counters. Concurrent suites and internal statistics queries can increment these counters, and fuzzy session initialization can override its global-disable attempt. The assertion can therefore fail even when the tested scan bypasses the cache. Count actual segment lookup attempts, including misses, in OLAP reader statistics and expose ConditionCacheLookup in the scan profile. Identify each regression execution with a unique SQL comment, wait for its completed profile, and require every lookup counter to be zero. Preserve result comparisons and positive deterministic lookup/hit coverage. Remove global metric sampling, global session changes and the nonConcurrent restriction. ### Release note Expose ConditionCacheLookup in OLAP scan profiles. ### Check List (For Author) - Test: Regression test / Unit Test / Manual test - ASAN BE build passed using build.sh --be -j 48 with ENABLE_PCH=OFF. - All 37 targeted SegmentIteratorCandidatePushdownTest and VirtualSlotRefTest tests passed using run-be-ut.sh. - condition_cache_eligibility passed with fuzzy session initialization and concurrent cache-eligible queries. The BE-wide metric increased from 36 to 2952 while all 45 ineligible query profiles reported zero lookups; deterministic lookup/hit checks passed. - clang-format/check-format 16.0.6, header hygiene, Groovy compilation, git diff checks and clang-tidy changed-line checks passed. - Behavior changed: Yes. OLAP profiles expose lookup attempts and the regression uses query-scoped counters. Cache eligibility and query results are unchanged. - Does this need documentation: No
1 parent 7f63a6f commit 5108dd2

7 files changed

Lines changed: 198 additions & 162 deletions

File tree

‎be/src/exec/operator/olap_scan_operator.cpp‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -126,6 +126,8 @@ PushDownType OlapScanLocalState::_should_push_down_binary_predicate(
126126

127127
Status OlapScanLocalState::_init_profile() {
128128
RETURN_IF_ERROR(ScanLocalState<OlapScanLocalState>::_init_profile());
129+
_condition_cache_lookup_counter =
130+
ADD_COUNTER(_scanner_profile, "ConditionCacheLookup", TUnit::UNIT);
129131
// Rows read from storage.
130132
// Include the rows read from doris page cache.
131133
_scan_rows = ADD_COUNTER(custom_profile(), "ScanRows", TUnit::UNIT);

‎be/src/exec/operator/olap_scan_operator.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -162,6 +162,7 @@ class OlapScanLocalState final : public ScanLocalState<OlapScanLocalState> {
162162
snii::SniiPrxRuntimeProfileCounters _snii_prx_profile_counters;
163163
snii::SniiPhraseRuntimeProfileCounters _snii_phrase_profile_counters;
164164

165+
RuntimeProfile::Counter* _condition_cache_lookup_counter = nullptr;
165166
RuntimeProfile::Counter* _tablet_counter = nullptr;
166167
RuntimeProfile::Counter* _buckets_pruned_by_rf_counter = nullptr;
167168
RuntimeProfile::Counter* _key_range_counter = nullptr;

‎be/src/exec/scan/olap_scanner.cpp‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -923,6 +923,8 @@ void OlapScanner::_collect_profile_before_close() {
923923
stats.output_index_result_column_timer);
924924
COUNTER_UPDATE(local_state->_filtered_segment_counter, stats.filtered_segment_number);
925925
COUNTER_UPDATE(local_state->_total_segment_counter, stats.total_segment_number);
926+
COUNTER_UPDATE(local_state->_condition_cache_lookup_counter,
927+
stats.condition_cache_lookup_count);
926928
COUNTER_UPDATE(local_state->_condition_cache_hit_counter, stats.condition_cache_hit_seg_nums);
927929
COUNTER_UPDATE(local_state->_condition_cache_filtered_rows_counter,
928930
stats.condition_cache_filtered_rows);

‎be/src/storage/olap_common.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -379,6 +379,8 @@ struct OlapReaderStatistics {
379379
int64_t output_index_result_column_timer = 0;
380380
// number of segment filtered by column stat when creating seg iterator
381381
int64_t filtered_segment_number = 0;
382+
// number of condition cache lookups, including misses
383+
int64_t condition_cache_lookup_count = 0;
382384
// number of segment with condition cache hit
383385
int64_t condition_cache_hit_seg_nums = 0;
384386
// number of rows filtered by condition cache hit

‎be/src/storage/segment/segment_iterator.cpp‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -163,6 +163,7 @@ void SegmentIterator::_init_row_bitmap_by_condition_cache() {
163163

164164
// Increment search count when digest != 0
165165
DorisMetrics::instance()->condition_cache_search_count->increment(1);
166+
_opts.stats->condition_cache_lookup_count++;
166167

167168
ConditionCacheHandle handle;
168169
_find_condition_cache = condition_cache->lookup(cache_key, &handle);

‎be/test/storage/segment/segment_iterator_candidate_pushdown_test.cpp‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -468,6 +468,50 @@ TEST_F(SegmentIteratorCandidatePushdownTest, versioned_deletes_do_not_publish_co
468468
EXPECT_FALSE(older_reader->_find_condition_cache);
469469
EXPECT_EQ(older_reader->_row_bitmap.cardinality(), 100);
470470
EXPECT_EQ(_iter->_opts.condition_cache_digest, 0);
471+
EXPECT_EQ(_stats.condition_cache_lookup_count, 0);
472+
EXPECT_EQ(older_stats.condition_cache_lookup_count, 1);
473+
}
474+
475+
TEST_F(SegmentIteratorCandidatePushdownTest,
476+
condition_cache_lookups_count_misses_and_hits_per_reader) {
477+
ScopedConditionCache cache;
478+
constexpr uint64_t digest = 12345;
479+
_iter->_opts.condition_cache_digest = digest;
480+
_iter->_row_bitmap.addRange(0, 100);
481+
_iter->_init_row_bitmap_by_condition_cache();
482+
EXPECT_FALSE(_iter->_find_condition_cache);
483+
EXPECT_EQ(_stats.condition_cache_lookup_count, 1);
484+
EXPECT_EQ(_stats.condition_cache_hit_seg_nums, 0);
485+
486+
ConditionCache::CacheKey key(_iter->_opts.rowset_id, _iter->segment_id(), digest);
487+
cache.get()->insert(key, std::move(_iter->_condition_cache));
488+
489+
OlapReaderStatistics second_stats;
490+
auto second_reader = std::make_unique<SegmentIterator>(_segment, _read_schema);
491+
second_reader->_opts.stats = &second_stats;
492+
second_reader->_opts.condition_cache_digest = digest;
493+
second_reader->_common_expr_ctxs_push_down = _iter->_common_expr_ctxs_push_down;
494+
second_reader->_row_bitmap.addRange(0, 100);
495+
second_reader->_init_row_bitmap_by_condition_cache();
496+
EXPECT_TRUE(second_reader->_find_condition_cache);
497+
EXPECT_EQ(second_stats.condition_cache_lookup_count, 1);
498+
EXPECT_EQ(second_stats.condition_cache_hit_seg_nums, 1);
499+
EXPECT_TRUE(second_reader->_row_bitmap.isEmpty());
500+
EXPECT_EQ(_stats.condition_cache_lookup_count, 1);
501+
EXPECT_EQ(_stats.condition_cache_hit_seg_nums, 0);
502+
}
503+
504+
TEST_F(SegmentIteratorCandidatePushdownTest, condition_cache_bypass_does_not_count_lookups) {
505+
ScopedConditionCache cache;
506+
_iter->_opts.condition_cache_digest = 0;
507+
_iter->_init_row_bitmap_by_condition_cache();
508+
EXPECT_EQ(_stats.condition_cache_lookup_count, 0);
509+
510+
_iter->_opts.condition_cache_digest = 12345;
511+
_iter->_common_expr_ctxs_push_down.clear();
512+
_iter->_init_row_bitmap_by_condition_cache();
513+
EXPECT_EQ(_stats.condition_cache_lookup_count, 0);
514+
EXPECT_EQ(_iter->_opts.condition_cache_digest, 0);
471515
}
472516

473517
TEST_F(SegmentIteratorCandidatePushdownTest, condition_ranges_engage_candidate_before_expr) {

0 commit comments

Comments
 (0)