Skip to content

Commit 691c951

Browse files
committed
[fix](be) Harden the hidden-column placeholder index probe
### What problem does this PR solve? Issue Number: None Problem Summary: The compatibility probe that drops an all-placeholder VERSION/COMMIT_TSO inverted index of a multi-version rowset ran for queries that disable inverted index queries, paying index I/O for an iterator those queries never consume. It also kept the iterator when the probe itself downgraded (for example on INVERTED_INDEX_FILE_CORRUPTED): a damaged index can still answer a disjoint predicate such as VERSION = 3 from intact [0, 0] BKD bounds without reading the damaged leaf, returning no rows although the column storage holds the materialized values. The probe now runs only when inverted index queries are enabled, and a probe that downgrades drops the iterator so the predicates evaluate the stored values. ### Release note None ### Check List (For Author) - Test: Unit test - New cases RangeVersionSkipsPlaceholderProbeWhenIndexQueriesDisabled and RangeVersionDropsIndexWhenPlaceholderProbeFails; both fail with the fix reverted. - ./run-be-ut.sh --run --filter='SegmentIteratorExprZonemapTest.RangeVersion*:SegmentIteratorExprZonemapTest.VersionPredicateSkipsPhysicalInvertedIndex' passed 6 tests. - Behavior changed: No - Does this need documentation: No
1 parent d1e3443 commit 691c951

2 files changed

Lines changed: 110 additions & 4 deletions

File tree

‎be/src/storage/segment/segment_iterator.cpp‎

Lines changed: 21 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2068,8 +2068,10 @@ Status SegmentIterator::_drop_placeholder_hidden_column_index(ColumnId cid) {
20682068
}
20692069
// Index queries take their options from the query's RuntimeState (see
20702070
// InvertedIndexReader::handle_searcher_cache); compaction, schema change and index builds read
2071-
// without one and evaluate no index.
2072-
if (_opts.runtime_state == nullptr) {
2071+
// without one and evaluate no index. A query that disables inverted index queries never
2072+
// consumes the iterator either, so the probe would only add index I/O to a scan that opted out.
2073+
if (_opts.runtime_state == nullptr ||
2074+
!_opts.runtime_state->query_options().enable_inverted_index_query) {
20732075
return Status::OK();
20742076
}
20752077
// A singleton rowset serves VERSION/COMMIT_TSO through a ConstantColumnReader, which opens no
@@ -2093,8 +2095,23 @@ Status SegmentIterator::_drop_placeholder_hidden_column_index(ColumnId cid) {
20932095
param.skip_try = true;
20942096
Status st = _index_iterators[cid]->read_from_index(IndexParam {&param});
20952097
if (!st.ok()) {
2096-
// The predicates on this column downgrade on the same status; otherwise it is an error.
2097-
return _downgrade_without_index(st) ? Status::OK() : st;
2098+
// The predicates on this column downgrade on the same statuses; otherwise it is an error.
2099+
if (!_downgrade_without_index(st)) {
2100+
return st;
2101+
}
2102+
// A probe that cannot answer cannot prove the index holds the materialized values, and a
2103+
// damaged index can still answer a disjoint predicate from intact compact bounds alone,
2104+
// without ever touching the damaged part (an old all-zero index keeps `[0, 0]` bounds and
2105+
// may have a damaged lazily decoded leaf). Leaving the iterator in place would then lose
2106+
// the matching rows, so drop it instead of handing it to the predicate path.
2107+
LOG(WARNING)
2108+
<< "ignore the inverted index of " << column.name() << " for tablet " << _tablet_id
2109+
<< ", rowset " << _segment->rowset_id().to_string() << ", segment " << segment_id()
2110+
<< ": the placeholder probe failed (" << st
2111+
<< "), so the index cannot be trusted to hold the materialized values; rebuild "
2112+
"the index";
2113+
_index_iterators[cid].reset();
2114+
return Status::OK();
20982115
}
20992116
if (param.roaring->cardinality() != num_rows()) {
21002117
return Status::OK();

‎be/test/storage/segment/segment_iterator_expr_zonemap_test.cpp‎

Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,7 @@
6464
#include "storage/segment/test_segment_writer.h"
6565
#include "storage/tablet/tablet_schema_helper.h"
6666
#include "storage/utils.h"
67+
#include "util/debug_points.h"
6768

6869
namespace doris::segment_v2 {
6970
namespace {
@@ -385,6 +386,8 @@ class SegmentIteratorExprZonemapTest : public testing::Test {
385386
}
386387

387388
void TearDown() override {
389+
DebugPoints::instance()->clear();
390+
config::enable_debug_points = false;
388391
ExecEnv::GetInstance()->set_inverted_index_searcher_cache(_previous_searcher_cache);
389392
ExecEnv::GetInstance()->set_inverted_index_query_cache(_previous_query_cache);
390393
_inverted_index_searcher_cache.reset();
@@ -1622,6 +1625,92 @@ TEST_F(SegmentIteratorExprZonemapTest, RangeVersionIgnoresPlaceholderInvertedInd
16221625
EXPECT_EQ(0, _stats.rows_inverted_index_filtered);
16231626
}
16241627

1628+
TEST_F(SegmentIteratorExprZonemapTest, RangeVersionSkipsPlaceholderProbeWhenIndexQueriesDisabled) {
1629+
// Case: the query disables inverted index queries, so no predicate consumes the index. The
1630+
// initialization-time placeholder probe must not query it either. The debug point turns any
1631+
// index query into a downgrade, so a probe that still runs is visible in the counter.
1632+
std::shared_ptr<Segment> segment;
1633+
ASSERT_NO_FATAL_FAILURE(build_version_segment_with_placeholder_index(&segment));
1634+
auto read_schema = make_read_schema(_tablet_schema);
1635+
1636+
TQueryOptions query_options;
1637+
query_options.__set_enable_inverted_index_query(false);
1638+
query_options.__set_enable_inverted_index_query_cache(false);
1639+
query_options.__set_enable_inverted_index_searcher_cache(false);
1640+
_runtime_state.set_query_options(query_options);
1641+
1642+
auto predicate = make_version_eq_column_predicate(kVersionCid, kRangeEndVersion);
1643+
auto block_predicate = AndBlockColumnPredicate::create_shared();
1644+
block_predicate->add_column_predicate(SingleColumnBlockPredicate::create_unique(predicate));
1645+
1646+
const std::set<int32_t> output_column_uids {0, kVersionCid};
1647+
StorageReadOptions read_options(_stats);
1648+
read_options.runtime_state = &_runtime_state;
1649+
read_options.version = Version(kRangeStartVersion, kRangeEndVersion);
1650+
read_options.block_row_max = 1024;
1651+
read_options.column_predicates = {predicate};
1652+
read_options.col_id_to_predicates.emplace(kVersionCid, std::move(block_predicate));
1653+
read_options.output_columns = &output_column_uids;
1654+
1655+
config::enable_debug_points = true;
1656+
DebugPoints::instance()->add("return_inverted_index_bypass");
1657+
1658+
std::unique_ptr<RowwiseIterator> iter;
1659+
auto st = segment->new_iterator(read_schema, read_options, &iter);
1660+
ASSERT_TRUE(st.ok()) << st;
1661+
ASSERT_NE(nullptr, iter);
1662+
ASSERT_NO_FATAL_FAILURE(assert_hidden_column_values(iter.get(), read_options, read_schema,
1663+
kNumRows / 2, kRangeEndVersion));
1664+
EXPECT_EQ(0, _stats.inverted_index_downgrade_count);
1665+
}
1666+
1667+
TEST_F(SegmentIteratorExprZonemapTest, RangeVersionDropsIndexWhenPlaceholderProbeFails) {
1668+
// Case: the probe cannot answer, as a damaged index leaf does. The intact `[0, 0]` bounds of
1669+
// the placeholder index would still answer `VERSION = 3` with no rows, so the failed probe
1670+
// must drop the iterator; the predicate then evaluates the stored values.
1671+
std::shared_ptr<Segment> segment;
1672+
ASSERT_NO_FATAL_FAILURE(build_version_segment_with_placeholder_index(&segment));
1673+
auto read_schema = make_read_schema(_tablet_schema);
1674+
1675+
TQueryOptions query_options;
1676+
query_options.__set_enable_inverted_index_query(true);
1677+
query_options.__set_enable_inverted_index_query_cache(false);
1678+
query_options.__set_enable_inverted_index_searcher_cache(false);
1679+
query_options.__set_inverted_index_skip_threshold(0);
1680+
_runtime_state.set_query_options(query_options);
1681+
1682+
auto predicate = make_version_eq_column_predicate(kVersionCid, kRangeEndVersion);
1683+
auto block_predicate = AndBlockColumnPredicate::create_shared();
1684+
block_predicate->add_column_predicate(SingleColumnBlockPredicate::create_unique(predicate));
1685+
1686+
const std::set<int32_t> output_column_uids {0, kVersionCid};
1687+
StorageReadOptions read_options(_stats);
1688+
read_options.runtime_state = &_runtime_state;
1689+
read_options.version = Version(kRangeStartVersion, kRangeEndVersion);
1690+
read_options.block_row_max = 1024;
1691+
read_options.column_predicates = {predicate};
1692+
read_options.col_id_to_predicates.emplace(kVersionCid, std::move(block_predicate));
1693+
read_options.output_columns = &output_column_uids;
1694+
1695+
// The probe runs during init; make its index query return the downgrade status a damaged
1696+
// index reports, then restore the normal path before the first batch evaluates the predicate.
1697+
config::enable_debug_points = true;
1698+
DebugPoints::instance()->add("return_inverted_index_bypass");
1699+
std::unique_ptr<RowwiseIterator> iter;
1700+
auto st = segment->new_iterator(read_schema, read_options, &iter);
1701+
ASSERT_TRUE(st.ok()) << st;
1702+
ASSERT_NE(nullptr, iter);
1703+
st = iter->init(read_options);
1704+
ASSERT_TRUE(st.ok()) << st;
1705+
DebugPoints::instance()->remove("return_inverted_index_bypass");
1706+
config::enable_debug_points = false;
1707+
1708+
ASSERT_FALSE(iter->empty());
1709+
ASSERT_NO_FATAL_FAILURE(assert_hidden_column_values(iter.get(), read_options, read_schema,
1710+
kNumRows / 2, kRangeEndVersion));
1711+
EXPECT_EQ(1, _stats.inverted_index_downgrade_count);
1712+
}
1713+
16251714
TEST_F(SegmentIteratorExprZonemapTest, ExprZonemapUsesReadTimeVersion) {
16261715
constexpr int64_t kVersion = 7;
16271716
_tablet_schema = make_version_tablet_schema();

0 commit comments

Comments
 (0)