Skip to content

Commit 9cd57bf

Browse files
shankar-iyershankar-iyer
committed
Skip index existence check
1 parent 8716008 commit 9cd57bf

File tree

4 files changed

+31
-20
lines changed

4 files changed

+31
-20
lines changed

‎src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp‎

Lines changed: 17 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -953,22 +953,25 @@ RangesInDataParts MergeTreeDataSelectExecutor::filterPartsByPrimaryKeyAndSkipInd
953953
ProfileEventTimeIncrement<Microseconds> watch(ProfileEvents::FilteringMarksWithSecondaryKeysMicroseconds);
954954

955955
auto min_max_granules = getMinMaxIndexGranules(ranges.data_part,
956-
ranges.ranges,
957956
skip_indexes.skip_index_for_top_n_filtering,
957+
ranges.ranges,
958958
top_n_filter_info->direction,
959959
false, /*access_by_mark*/
960960
reader_settings,
961961
mark_cache.get(),
962962
uncompressed_cache.get(),
963963
vector_similarity_index_cache.get());
964964

965-
std::vector<size_t> result;
966-
min_max_granules->getTopNMarks(top_n_filter_info->limit_n, result);
967-
MarkRanges res;
968-
for (auto range : result)
969-
res.push_back({range, range + 1});
970-
std::sort(res.begin(), res.end());
971-
ranges.ranges = res;
965+
if (min_max_granules) /// minmax index may have not been materialized for this part, not a fatal error
966+
{
967+
std::vector<size_t> result;
968+
min_max_granules->getTopNMarks(top_n_filter_info->limit_n, result);
969+
MarkRanges res;
970+
for (auto range : result)
971+
res.push_back({range, range + 1});
972+
std::sort(res.begin(), res.end());
973+
ranges.ranges = res;
974+
}
972975
top_n_elapsed_us.fetch_add(watch.elapsed(), std::memory_order_relaxed);
973976
}
974977
sum_marks_after_top_n.fetch_add(ranges.ranges.getNumberOfMarks(), std::memory_order_relaxed);
@@ -2245,16 +2248,20 @@ void MergeTreeDataSelectExecutor::selectPartsToReadWithUUIDFilter(
22452248
MergeTreeIndexBulkGranulesMinMaxPtr
22462249
MergeTreeDataSelectExecutor::getMinMaxIndexGranules(
22472250
MergeTreeData::DataPartPtr part,
2248-
const MarkRanges & ranges,
22492251
MergeTreeIndexPtr skip_index_minmax,
2252+
const MarkRanges & ranges,
22502253
int direction,
22512254
bool access_by_mark,
22522255
const MergeTreeReaderSettings & reader_settings,
22532256
MarkCache * mark_cache,
22542257
UncompressedCache * uncompressed_cache,
22552258
VectorSimilarityIndexCache * vector_similarity_index_cache)
22562259
{
2257-
/// TODO - make sure it is a minmax index and index is materialized for part
2260+
if (!skip_index_minmax->getDeserializedFormat(part->checksums, skip_index_minmax->getFileName()))
2261+
{
2262+
return nullptr;
2263+
}
2264+
22582265
auto index_granularity = skip_index_minmax->index.granularity;
22592266
size_t marks_count = part->index_granularity->getMarksCountWithoutFinal();
22602267
size_t index_marks_count = (marks_count + index_granularity - 1) / index_granularity;

‎src/Storages/MergeTree/MergeTreeDataSelectExecutor.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -101,8 +101,8 @@ class MergeTreeDataSelectExecutor
101101

102102
static MergeTreeIndexBulkGranulesMinMaxPtr getMinMaxIndexGranules(
103103
MergeTreeData::DataPartPtr part,
104-
const MarkRanges & ranges,
105104
MergeTreeIndexPtr skip_index_minmax,
105+
const MarkRanges & ranges,
106106
int direction,
107107
bool access_by_mark,
108108
const MergeTreeReaderSettings & reader_settings,

‎src/Storages/MergeTree/MergeTreeIndexReadResultPool.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -104,8 +104,8 @@ SkipIndexReadResultPtr MergeTreeSkipIndexReader::read(const RangesInDataPart & p
104104
{
105105
res->min_max_index_for_top_n = MergeTreeDataSelectExecutor::getMinMaxIndexGranules(
106106
part.data_part,
107-
ranges,
108107
skip_indexes.skip_index_for_top_n_filtering,
108+
ranges,
109109
skip_indexes.threshold_tracker->getDirection(),
110110
true,/*access_by_mark*/
111111
reader_settings,

‎src/Storages/MergeTree/MergeTreeReaderIndex.cpp‎

Lines changed: 12 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -89,19 +89,23 @@ size_t MergeTreeReaderIndex::readRows(
8989

9090
bool MergeTreeReaderIndex::canSkipMark(size_t mark, size_t /*current_task_last_mark*/)
9191
{
92-
if (index_read_result->skip_index_read_result)
92+
auto skip_index_read_result = index_read_result->skip_index_read_result;
93+
if (skip_index_read_result)
9394
{
94-
chassert(mark < index_read_result->skip_index_read_result->granules_selected.size());
95+
chassert(mark < skip_index_read_result->granules_selected.size());
9596

96-
if (!index_read_result->skip_index_read_result->granules_selected.at(mark))
97+
if (!skip_index_read_result->granules_selected.at(mark))
9798
return true;
9899

99-
if (index_read_result->skip_index_read_result->threshold_tracker && index_read_result->skip_index_read_result->threshold_tracker->isSet())
100+
if (skip_index_read_result->threshold_tracker && skip_index_read_result->threshold_tracker->isSet())
100101
{
101-
auto granule_num = index_read_result->skip_index_read_result->min_max_index_for_top_n->granules_map[mark];
102-
if (!index_read_result->skip_index_read_result->threshold_tracker->isValueInsideThreshold(
103-
index_read_result->skip_index_read_result->min_max_index_for_top_n->granules[granule_num].min_or_max_value))
104-
return true;
102+
if (skip_index_read_result->min_max_index_for_top_n) /// index may not have been materialized for this part
103+
{
104+
auto granule_num = skip_index_read_result->min_max_index_for_top_n->granules_map[mark];
105+
if (skip_index_read_result->threshold_tracker->isValueInsideThreshold(
106+
skip_index_read_result->min_max_index_for_top_n->granules[granule_num].min_or_max_value))
107+
return true;
108+
}
105109
}
106110
}
107111
if (index_read_result->projection_index_read_result)

0 commit comments

Comments
 (0)