Skip to content

Commit 27cdbb5

Browse files
airborne12claude
andcommitted
[fix](be) Skip BKD segments when collecting BM25 statistics
### What problem does this PR solve? Issue Number: None Related PR: #69790 Problem Summary: a VARIANT path is typed per segment, so a numeric path is indexed as BKD in the segments that hold numbers while the tablet schema only knows it as text. score() over a MATCH on that path collects statistics for it and opens the BKD index as a text index: CREATE TABLE t (id INT, v VARIANT, INDEX idx_v(v) USING INVERTED) ...; INSERT INTO t VALUES (1, parse_to_variant('{"n": 1, "s": "abc"}')), ...; SELECT score() FROM t WHERE CAST(v['n'] AS STRING) MATCH '1' ORDER BY score() DESC LIMIT 10; -- V2/V3: CLuceneError occurred opening the index for BM25 statistics: -- No segments* file found -- SNII: segment: logical index is not a text inverted index With an analyzed VARIANT index (parser=english) this failed before as well; keyword MATCH scoring made the plain VARIANT index, the common case, reach it too. Without score() the query works. Fix: a segment that indexed the field with BKD holds no terms, so it adds no statistics, as a segment without the path already does. V2/V3 detect it by the BKD data file in the compound directory before opening a CLucene reader; SNII by the logical index kind in its directory. The rows of such a segment still match without the index and score 0; the text path of the same column keeps its BM25 score. ### Release note None ### Check List (For Author) - Test: Regression test / Manual test - Regression test: test_match_keyword_score adds a VARIANT case with a numeric path (BKD in one segment, text in another) and a text path, on V2/V3/SNII; before this change it failed on all three. 26 score-related suites in inverted_index_p0, search and variant_p0 pass. - Manual test: keyword and analyzed VARIANT indexes on V2/V3/SNII. - Unit Test: CollectionStatistics*, IndexReaderHelperTest. No new unit test: the skip needs a BKD index on disk, which the regression case builds on every format. - Behavior changed: Yes. The query above returns its rows (score 0) instead of failing. - Does this need documentation: No Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent 7881be5 commit 27cdbb5

3 files changed

Lines changed: 92 additions & 0 deletions

File tree

‎be/src/storage/index/inverted/reader/collection_statistics.cpp‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,15 @@ Result<std::shared_ptr<lucene::search::IndexSearcher>> open_optional_clucene_sta
7373
}
7474
return ResultError(status);
7575
}
76+
// A segment can index a field with BKD although the tablet schema calls it text: a VARIANT
77+
// path is typed per segment, so CAST(v['n'] AS STRING) MATCH '1' collects statistics for v.n,
78+
// which a segment holding numbers indexed as BKD. That index has no terms and no segments file
79+
// to open, so it adds no statistics, as a segment without the path does; its rows match
80+
// without the index and score nothing.
81+
if (compound.value()->fileExists(
82+
InvertedIndexDescriptor::get_temporary_bkd_index_data_file_name())) {
83+
return std::shared_ptr<lucene::search::IndexSearcher> {};
84+
}
7685
lucene::index::IndexReader* reader = nullptr;
7786
try {
7887
reader = lucene::index::IndexReader::open(compound.value().get());
@@ -89,6 +98,17 @@ Result<std::shared_ptr<lucene::search::IndexSearcher>> open_optional_clucene_sta
8998
return std::make_shared<lucene::search::IndexSearcher>(reader, true);
9099
}
91100

101+
// Whether a SNII segment indexed this field with BKD; see open_optional_clucene_stats_searcher.
102+
Result<bool> is_snii_bkd_index(const segment_v2::IndexFileReader& file_reader,
103+
const TabletIndex& index) {
104+
const auto entries = DORIS_TRY(file_reader.snii_logical_indexes());
105+
const auto entry = std::ranges::find_if(entries, [&](const auto& candidate) {
106+
return candidate.index_id == static_cast<uint64_t>(index.index_id()) &&
107+
candidate.index_suffix == index.get_index_suffix();
108+
});
109+
return entry != entries.end() && entry->kind == ::doris::snii::format::LogicalIndexKind::kBkd;
110+
}
111+
92112
} // namespace
93113

94114
namespace collection_statistics_detail {
@@ -547,6 +567,11 @@ Status CollectionStatistics::process_segment(const RowsetSharedPtr& rowset,
547567
segment_v2::snii_doris::DorisSniiFileReader::ScopedIOContext io_context_scope(io_ctx);
548568
SniiScoringSegmentAccumulator segment_accumulator;
549569
for (const auto& [ws_field_name, collect_info] : collect_infos) {
570+
const bool bkd =
571+
DORIS_TRY(is_snii_bkd_index(*idx_file_reader, *collect_info.index_meta));
572+
if (bkd) {
573+
continue; // The field holds numbers in this segment: no terms, no statistics.
574+
}
550575
auto logical_reader_result =
551576
idx_file_reader->open_snii_index(collect_info.index_meta, io_ctx);
552577
if (!logical_reader_result.has_value()) {

‎regression-test/data/inverted_index_p0/test_match_keyword_score.out‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,15 @@
6666
1 0.290716
6767
4 0.268636
6868

69+
-- !V2_variant_numeric_path --
70+
1 0
71+
3 0
72+
4 0.287682
73+
74+
-- !V2_variant_text_path --
75+
1 0.693147
76+
4 0.693147
77+
6978
-- !V3_single_row_match --
7079
1 0.287682
7180

@@ -133,6 +142,15 @@
133142
1 0.290716
134143
4 0.268636
135144

145+
-- !V3_variant_numeric_path --
146+
1 0
147+
3 0
148+
4 0.287682
149+
150+
-- !V3_variant_text_path --
151+
1 0.693147
152+
4 0.693147
153+
136154
-- !SNII_single_row_match --
137155
1 0.287682
138156

@@ -200,3 +218,12 @@
200218
1 0.290716
201219
4 0.268636
202220

221+
-- !SNII_variant_numeric_path --
222+
1 0
223+
3 0
224+
4 0.287682
225+
226+
-- !SNII_variant_text_path --
227+
1 0.693147
228+
4 0.693147
229+

‎regression-test/suites/inverted_index_p0/test_match_keyword_score.groovy‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -93,5 +93,45 @@ suite("test_match_keyword_score", "p0") {
9393
) hits ORDER BY id
9494
""", true)
9595
}
96+
97+
// A VARIANT path is typed per segment: n holds numbers, indexed as BKD, in the first
98+
// insert and text in the second. The plain VARIANT index makes every path a keyword
99+
// index, so CAST(v['n'] AS STRING) MATCH collects statistics for n; the BKD segment adds
100+
// none instead of failing, and its rows match without the index with score 0.
101+
sql "set default_variant_enable_typed_paths_to_sparse = false"
102+
sql "set default_variant_enable_doc_mode = false"
103+
sql "DROP TABLE IF EXISTS test_match_keyword_score_variant"
104+
sql """
105+
CREATE TABLE test_match_keyword_score_variant (
106+
id INT NOT NULL,
107+
v VARIANT NULL,
108+
INDEX idx_v(v) USING INVERTED
109+
) ENGINE=OLAP
110+
DUPLICATE KEY(id)
111+
DISTRIBUTED BY HASH(id) BUCKETS 1
112+
PROPERTIES (
113+
"replication_allocation" = "tag.location.default:1",
114+
"inverted_index_storage_format" = "${storageFormat}",
115+
"disable_auto_compaction" = "true"
116+
)
117+
"""
118+
sql """INSERT INTO test_match_keyword_score_variant VALUES
119+
(1, parse_to_variant('{"n": 1, "s": "abc"}')), (2, parse_to_variant('{"n": 2, "s": "abd"}')),
120+
(3, parse_to_variant('{"n": 1, "s": "xyz"}'))
121+
"""
122+
sql """INSERT INTO test_match_keyword_score_variant VALUES
123+
(4, parse_to_variant('{"n": "1", "s": "abc"}'))
124+
"""
125+
sql "sync"
126+
for (def query : [
127+
["variant_numeric_path", "CAST(v['n'] AS STRING) MATCH '1'"],
128+
["variant_text_path", "CAST(v['s'] AS STRING) MATCH 'abc'"]]) {
129+
quickTest("${storageFormat}_${query[0]}", """
130+
SELECT id, ROUND(s, 6) FROM (
131+
SELECT id, score() AS s FROM test_match_keyword_score_variant
132+
WHERE ${query[1]} ORDER BY s DESC LIMIT 10
133+
) hits ORDER BY id
134+
""", true)
135+
}
96136
}
97137
}

0 commit comments

Comments
 (0)