Skip to content

Commit f019e5d

Browse files
committed
[fix](be) Bound condition cache coordinate coverage
### What problem does this PR solve? Issue Number: None Related PR: #65496 Problem Summary: A condition-cache MISS allocated an extra guard bit that was published as if it covered real rows. A later plan could therefore interpret the false guard as proof that the next real granule had no matches. In addition, legacy and V2 scanners shared external cache keys even though only V2 understands the persisted bitmap base. Track the authoritative granule count and trim allocation-only guards before publication, and version V2 external cache keys to isolate the base-aware coordinate protocol from legacy entries. ### Release note Fix condition-cache filtering across changed metadata-pruning plans and mixed scanner modes. ### Check List (For Author) - Test: Unit Test - `ParquetScanConditionCacheTest.*` - `TableReaderTest.ConditionCacheMissPublishesBitmapAfterReaderEof` - `TableReaderTest.ConditionCacheMissIsDroppedWhenReaderClosesBeforeEof` - `NewOrcReaderTest.ConditionCacheMissMarksSurvivingGranules` - `NewOrcReaderTest.ConditionCacheHitUsesSplitBaseGranule` - Behavior changed: Yes. Guard bits are not published as authoritative cache coverage, and V2 external condition-cache entries cannot be consumed by legacy scanners. - Does this need documentation: No
1 parent a769e06 commit f019e5d

6 files changed

Lines changed: 50 additions & 8 deletions

File tree

‎be/src/format_v2/orc/orc_reader.cpp‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -893,6 +893,16 @@ void OrcReader::set_condition_cache_context(std::shared_ptr<ConditionCacheContex
893893
!_state->condition_cache_ctx->is_hit) {
894894
_state->condition_cache_ctx->base_granule = static_cast<int64_t>(
895895
_state->row_reader_range_first_row / ConditionCacheContext::GRANULE_SIZE);
896+
const auto end_granule =
897+
(_state->row_reader_range_first_row + _state->row_reader_range_rows +
898+
ConditionCacheContext::GRANULE_SIZE - 1) /
899+
ConditionCacheContext::GRANULE_SIZE;
900+
DORIS_CHECK(end_granule > static_cast<uint64_t>(_state->condition_cache_ctx->base_granule));
901+
_state->condition_cache_ctx->num_granules =
902+
std::min(_state->condition_cache_ctx->filter_result->size(),
903+
static_cast<size_t>(
904+
end_granule -
905+
static_cast<uint64_t>(_state->condition_cache_ctx->base_granule)));
896906
}
897907
}
898908

‎be/src/format_v2/parquet/parquet_scan.cpp‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -608,6 +608,14 @@ void ParquetScanScheduler::set_condition_cache_context(std::shared_ptr<Condition
608608
if (!_condition_cache_ctx->is_hit) {
609609
_condition_cache_ctx->base_granule =
610610
_row_group_plans.front().first_file_row / ConditionCacheContext::GRANULE_SIZE;
611+
const auto& last_plan = _row_group_plans.back();
612+
const int64_t end_granule = (last_plan.first_file_row + last_plan.row_group_rows +
613+
ConditionCacheContext::GRANULE_SIZE - 1) /
614+
ConditionCacheContext::GRANULE_SIZE;
615+
DORIS_CHECK(end_granule > _condition_cache_ctx->base_granule);
616+
_condition_cache_ctx->num_granules =
617+
std::min(_condition_cache_ctx->filter_result->size(),
618+
static_cast<size_t>(end_granule - _condition_cache_ctx->base_granule));
611619
return;
612620
}
613621

‎be/src/format_v2/table_reader.cpp‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -581,7 +581,8 @@ Status TableReader::_init_reader_condition_cache(const FileScanRequest& file_req
581581
const auto& file = *_current_file_description;
582582
_condition_cache_key = segment_v2::ConditionCache::ExternalCacheKey(
583583
file.path, file.mtime, file.file_size, _condition_cache_digest, file.range_start_offset,
584-
file.range_size);
584+
file.range_size,
585+
segment_v2::ConditionCache::ExternalCacheKey::BASE_GRANULE_AWARE_VERSION);
585586

586587
segment_v2::ConditionCacheHandle handle;
587588
const bool condition_cache_hit = cache->lookup(_condition_cache_key, &handle);
@@ -605,6 +606,7 @@ Status TableReader::_init_reader_condition_cache(const FileScanRequest& file_req
605606
_condition_cache_ctx = std::make_shared<ConditionCacheContext>();
606607
_condition_cache_ctx->is_hit = condition_cache_hit;
607608
_condition_cache_ctx->filter_result = _condition_cache;
609+
_condition_cache_ctx->num_granules = _condition_cache->size();
608610
if (condition_cache_hit) {
609611
_condition_cache_ctx->base_granule = handle.get_base_granule();
610612
}
@@ -627,6 +629,8 @@ void TableReader::_finalize_reader_condition_cache() {
627629
_condition_cache_ctx = nullptr;
628630
return;
629631
}
632+
DORIS_CHECK(_condition_cache_ctx->num_granules <= _condition_cache->size());
633+
_condition_cache->resize(_condition_cache_ctx->num_granules);
630634
segment_v2::ConditionCache::instance()->insert(
631635
_condition_cache_key, std::move(_condition_cache), _condition_cache_ctx->base_granule);
632636
_condition_cache = nullptr;

‎be/src/storage/segment/condition_cache.h‎

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,7 @@ struct ConditionCacheContext {
4848
bool is_hit = false;
4949
std::shared_ptr<std::vector<bool>> filter_result; // per-granule: true = has surviving rows
5050
int64_t base_granule = 0; // global granule index of filter_result[0]
51+
size_t num_granules = 0; // authoritative bitmap length; excludes allocation-only guard bits
5152
static constexpr int GRANULE_SIZE = 2048;
5253
};
5354

@@ -87,31 +88,37 @@ class ConditionCache : public LRUCachePolicy {
8788

8889
// Cache key for external tables (Hive ORC/Parquet)
8990
struct ExternalCacheKey {
91+
static constexpr uint8_t BASE_GRANULE_AWARE_VERSION = 1;
92+
9093
ExternalCacheKey() = default;
9194
ExternalCacheKey(const std::string& path_, int64_t modification_time_, int64_t file_size_,
92-
uint64_t digest_, int64_t start_offset_, int64_t size_)
95+
uint64_t digest_, int64_t start_offset_, int64_t size_,
96+
uint8_t format_version_ = 0)
9397
: path(path_),
9498
modification_time(modification_time_),
9599
file_size(file_size_),
96100
digest(digest_),
97101
start_offset(start_offset_),
98-
size(size_) {}
102+
size(size_),
103+
format_version(format_version_) {}
99104
std::string path;
100105
int64_t modification_time = 0;
101106
int64_t file_size = 0;
102107
uint64_t digest = 0;
103108
int64_t start_offset = 0;
104109
int64_t size = 0;
110+
uint8_t format_version = 0;
105111

106112
[[nodiscard]] std::string encode() const {
107113
std::string key = path;
108-
char buf[40];
114+
char buf[41];
109115
memcpy(buf, &modification_time, 8);
110116
memcpy(buf + 8, &file_size, 8);
111117
memcpy(buf + 16, &digest, 8);
112118
memcpy(buf + 24, &start_offset, 8);
113119
memcpy(buf + 32, &size, 8);
114-
key.append(buf, 40);
120+
buf[40] = static_cast<char>(format_version);
121+
key.append(buf, 41);
115122
return key;
116123
}
117124
};

‎be/test/format_v2/parquet/parquet_scan_test.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -551,7 +551,7 @@ TEST(ParquetScanConditionCacheTest, HitKeepsCachedBaseWhenCurrentPlanStartsLater
551551
auto ctx = std::make_shared<ConditionCacheContext>();
552552
ctx->is_hit = true;
553553
ctx->base_granule = 0;
554-
ctx->filter_result = std::make_shared<std::vector<bool>>(std::vector<bool> {false, true});
554+
ctx->filter_result = std::make_shared<std::vector<bool>>(std::vector<bool> {false});
555555
scheduler.set_condition_cache_context(ctx);
556556

557557
EXPECT_FALSE(scheduler.empty());

‎be/test/format_v2/table_reader_test.cpp‎

Lines changed: 15 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -976,6 +976,7 @@ struct FakeFileReaderState {
976976
int64_t total_rows = 2;
977977
int64_t aggregate_count = -1;
978978
int64_t condition_cache_base_granule = 0;
979+
size_t condition_cache_num_granules = 0;
979980
bool eof_with_first_batch = true;
980981
bool inject_delete_conjunct = false;
981982
bool stop_during_aggregate = false;
@@ -1108,6 +1109,9 @@ class FakeFileReader final : public FileReader {
11081109
_state->condition_cache_ctx = std::move(ctx);
11091110
if (_state->condition_cache_ctx != nullptr && !_state->condition_cache_ctx->is_hit) {
11101111
_state->condition_cache_ctx->base_granule = _state->condition_cache_base_granule;
1112+
if (_state->condition_cache_num_granules > 0) {
1113+
_state->condition_cache_ctx->num_granules = _state->condition_cache_num_granules;
1114+
}
11111115
}
11121116
}
11131117

@@ -1848,6 +1852,7 @@ TEST(TableReaderTest, ConditionCacheMissPublishesBitmapAfterReaderEof) {
18481852
auto fake_state = std::make_shared<FakeFileReaderState>();
18491853
fake_state->total_rows = ConditionCacheContext::GRANULE_SIZE;
18501854
fake_state->condition_cache_base_granule = 7;
1855+
fake_state->condition_cache_num_granules = 1;
18511856
FakeTableReader reader(file_schema, fake_state);
18521857
ASSERT_TRUE(reader.init({
18531858
.projected_columns = projected_columns,
@@ -1872,12 +1877,18 @@ TEST(TableReaderTest, ConditionCacheMissPublishesBitmapAfterReaderEof) {
18721877
ASSERT_NE(fake_state->condition_cache_ctx, nullptr);
18731878
EXPECT_FALSE(fake_state->condition_cache_ctx->is_hit);
18741879

1875-
segment_v2::ConditionCache::ExternalCacheKey key("fake-table-reader-input", 0, -1, 7, 0, -1);
1880+
segment_v2::ConditionCache::ExternalCacheKey legacy_key("fake-table-reader-input", 0, -1, 7, 0,
1881+
-1);
18761882
segment_v2::ConditionCacheHandle handle;
1883+
EXPECT_FALSE(cache.get()->lookup(legacy_key, &handle));
1884+
segment_v2::ConditionCache::ExternalCacheKey key(
1885+
"fake-table-reader-input", 0, -1, 7, 0, -1,
1886+
segment_v2::ConditionCache::ExternalCacheKey::BASE_GRANULE_AWARE_VERSION);
18771887
ASSERT_TRUE(cache.get()->lookup(key, &handle));
18781888
const auto cached_bitmap = handle.get_filter_result();
18791889
ASSERT_NE(cached_bitmap, nullptr);
18801890
ASSERT_FALSE(cached_bitmap->empty());
1891+
EXPECT_EQ(cached_bitmap->size(), 1);
18811892
EXPECT_TRUE((*cached_bitmap)[0]);
18821893
EXPECT_EQ(handle.get_base_granule(), 7);
18831894

@@ -1925,7 +1936,9 @@ TEST(TableReaderTest, ConditionCacheMissIsDroppedWhenReaderClosesBeforeEof) {
19251936
EXPECT_FALSE(fake_state->condition_cache_ctx->is_hit);
19261937

19271938
ASSERT_TRUE(reader.close().ok());
1928-
segment_v2::ConditionCache::ExternalCacheKey key("fake-table-reader-input", 0, -1, 7, 0, -1);
1939+
segment_v2::ConditionCache::ExternalCacheKey key(
1940+
"fake-table-reader-input", 0, -1, 7, 0, -1,
1941+
segment_v2::ConditionCache::ExternalCacheKey::BASE_GRANULE_AWARE_VERSION);
19291942
segment_v2::ConditionCacheHandle handle;
19301943
EXPECT_FALSE(cache.get()->lookup(key, &handle));
19311944
}

0 commit comments

Comments
 (0)