Skip to content

Commit c4b11b5

Browse files
authored
[fix](be) Keep BM25 statistics per index for MATCH on one column (#69901)
### What problem does this PR solve? Issue Number: None Related PR: #69790 Problem Summary: score() over MATCH clauses that read different inverted indexes of one column fails: INDEX idx_english(name) USING INVERTED PROPERTIES("parser" = "english", ...), INDEX idx_standard(name) USING INVERTED PROPERTIES("parser" = "standard", ...) SELECT id, score() FROM t WHERE name MATCH 'apple' USING ANALYZER english AND name MATCH 'apple' USING ANALYZER standard ORDER BY score() DESC LIMIT 10; -- Scoring predicates for one field select different inverted indexes BM25 statistics are keyed by field (column unique id plus suffix path), but each index of a column holds its own terms and lengths: over Apple and apple, an index that folds case has df(apple) = 2 and one that does not has df(apple) = 1. One entry cannot serve both, so the collector refuses the query. Fix: a MATCH keeps its statistics under the field and the index id. CollectInfo records the field as indexed for opening the index, since the map key no longer is one. The readers bind their source to the real field and pass the statistics name to run_leaf, so the scorers and similarities need no change: they already look statistics up by the name they are given. Each clause now scores with the statistics of its own index, and the combined score is the sum of what each clause scores alone. SEARCH keeps its own statistics names and its rejection of two indexes on one field; only MATCH changes. A MATCH and a SEARCH on one index keep separate entries with the same values.
1 parent eb16c4d commit c4b11b5

13 files changed

Lines changed: 256 additions & 51 deletions

File tree

‎be/src/exprs/function/search/search_predicate_collector.cpp‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -241,6 +241,7 @@ Status SearchPredicateCollector::collect_from_leaf(const TSearchClause& clause,
241241
auto iter = collect_infos->find(ws_field_name);
242242
if (iter == collect_infos->end()) {
243243
CollectInfo collect_info;
244+
collect_info.lucene_field = ws_field_name;
244245
RETURN_IF_ERROR(append_scoring_leaf(&collect_info, term_infos));
245246
preserve_selected_index_metadata(candidates, index_meta, &collect_info);
246247
(*collect_infos)[ws_field_name] = std::move(collect_info);

‎be/src/storage/index/inverted/clucene/clucene_index_reader.cpp‎

Lines changed: 17 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -338,7 +338,8 @@ Status create_index_searcher(IndexSearcherBuilder* index_searcher_builder,
338338
}
339339

340340
Status run_clucene_leaf(const IndexQueryContextPtr& context, std::wstring field,
341-
const logical::Node& leaf, const roaring::Roaring* candidates, bool scoring,
341+
const std::wstring& statistics_field, const logical::Node& leaf,
342+
const roaring::Roaring* candidates, bool scoring,
342343
const FulltextIndexSearcherPtr& searcher,
343344
const std::shared_ptr<roaring::Roaring>& result) {
344345
auto* reader = searcher->getReader();
@@ -350,8 +351,7 @@ Status run_clucene_leaf(const IndexQueryContextPtr& context, std::wstring field,
350351
auto source =
351352
clucene_index_source(std::shared_ptr<lucene::index::IndexReader>(searcher, reader),
352353
std::move(field), context->io_ctx);
353-
const auto& source_field = source->field();
354-
RETURN_IF_ERROR(run_leaf(context, source_field, leaf, candidates, scoring,
354+
RETURN_IF_ERROR(run_leaf(context, statistics_field, leaf, candidates, scoring,
355355
std::move(source), reader->maxDoc(), result));
356356
} catch (const CLuceneError& e) {
357357
return clucene_error_status(fmt::format("CLuceneError occurred: {}", e.what()));
@@ -413,15 +413,26 @@ Status CluceneTextIndexReader::_term_document_frequency(const std::string& colum
413413
return Status::OK();
414414
}
415415

416+
Status run_clucene_leaf(const IndexQueryContextPtr& context, std::wstring field,
417+
const logical::Node& leaf, const roaring::Roaring* candidates, bool scoring,
418+
const FulltextIndexSearcherPtr& searcher,
419+
const std::shared_ptr<roaring::Roaring>& result) {
420+
const std::wstring statistics_field = field;
421+
return run_clucene_leaf(context, std::move(field), statistics_field, leaf, candidates, scoring,
422+
searcher, result);
423+
}
424+
416425
Status CluceneTextIndexReader::_run_leaf(const IndexQueryContextPtr& context,
417426
const std::string& column_name, OpenedIndex& index,
418427
const logical::Node& leaf,
419428
const roaring::Roaring* candidates, bool scoring,
420429
std::shared_ptr<roaring::Roaring>* out) {
421430
auto result = std::make_shared<roaring::Roaring>();
422-
RETURN_IF_ERROR(run_clucene_leaf(context, StringUtil::string_to_wstring(column_name), leaf,
423-
candidates, scoring,
424-
static_cast<CluceneOpenedIndex&>(index).searcher, result));
431+
auto field = StringUtil::string_to_wstring(column_name);
432+
auto statistics_field = _statistics_field(field);
433+
RETURN_IF_ERROR(run_clucene_leaf(context, std::move(field), statistics_field, leaf, candidates,
434+
scoring, static_cast<CluceneOpenedIndex&>(index).searcher,
435+
result));
425436
result->runOptimize();
426437
*out = std::move(result);
427438
return Status::OK();

‎be/src/storage/index/inverted/clucene/clucene_index_reader.h‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,12 @@ Status read_clucene_null_bitmap(IndexFileReader& file_reader, const TabletIndex&
5959
lucene::store::Directory* dir = nullptr);
6060

6161
// run_leaf on the CLucene field `field` of `searcher`, its errors reported as CLucene ones.
62+
// `statistics_field` names the leaf inside the query, as _statistics_field does for an index.
63+
Status run_clucene_leaf(const IndexQueryContextPtr& context, std::wstring field,
64+
const std::wstring& statistics_field,
65+
const index_query::logical::Node& leaf, const roaring::Roaring* candidates,
66+
bool scoring, const FulltextIndexSearcherPtr& searcher,
67+
const std::shared_ptr<roaring::Roaring>& result);
6268
Status run_clucene_leaf(const IndexQueryContextPtr& context, std::wstring field,
6369
const index_query::logical::Node& leaf, const roaring::Roaring* candidates,
6470
bool scoring, const FulltextIndexSearcherPtr& searcher,

‎be/src/storage/index/inverted/common/query_info.h‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,4 +40,11 @@ class TermInfo {
4040
}
4141
};
4242

43+
// The name BM25 statistics of a MATCH are kept under. Two indexes of one column hold different
44+
// terms and lengths under the same field, so the field alone cannot key them: over Apple and
45+
// apple, an index that folds case has df(apple) = 2 and one that does not has df(apple) = 1.
46+
inline std::wstring match_statistics_field(const std::wstring& field, int64_t index_id) {
47+
return field + L"@" + std::to_wstring(index_id);
48+
}
49+
4350
} // namespace doris::segment_v2

‎be/src/storage/index/inverted/reader/collection_statistics.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -664,7 +664,7 @@ Status CollectionStatistics::process_segment(const RowsetSharedPtr& rowset,
664664
index_reader = index_searcher->getReader();
665665
#endif
666666
auto source = segment_v2::clucene_index_source(segment_v2::non_owning_reader(index_reader),
667-
ws_field_name, io_ctx);
667+
collect_info.lucene_field, io_ctx);
668668
index_query::IndexScoringStats scoring_stats;
669669
RETURN_IF_ERROR(source->scoring_stats(&scoring_stats));
670670
if (hidden_rows != nullptr) {

‎be/src/storage/index/inverted/reader/predicate_collector.cpp‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -391,11 +391,14 @@ Status MatchPredicateCollector::collect(RuntimeState* state, const TabletSchemaS
391391

392392
std::string field_name =
393393
build_field_name(index_meta->col_unique_ids()[0], candidates.index_suffix_path);
394-
std::wstring ws_field_name = inverted_index::StringHelper::to_wstring(field_name);
394+
std::wstring lucene_field = inverted_index::StringHelper::to_wstring(field_name);
395+
std::wstring ws_field_name =
396+
segment_v2::match_statistics_field(lucene_field, index_meta->index_id());
395397

396398
auto iter = collect_infos->find(ws_field_name);
397399
if (iter == collect_infos->end()) {
398400
CollectInfo collect_info;
401+
collect_info.lucene_field = std::move(lucene_field);
399402
RETURN_IF_ERROR(append_scoring_leaf(&collect_info, term_infos));
400403
preserve_selected_index_metadata(candidates, index_meta, &collect_info);
401404
iter = collect_infos->emplace(ws_field_name, std::move(collect_info)).first;

‎be/src/storage/index/inverted/reader/predicate_collector.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,8 @@ struct LogicalScoringPrefix {
6767
};
6868

6969
struct CollectInfo {
70+
// The field as indexed; the key of a CollectInfoMap may also name the index.
71+
std::wstring lucene_field;
7072
std::vector<std::string> unique_terms;
7173
std::unordered_map<std::string, uint32_t> unique_term_slots;
7274
std::vector<LogicalScoringPrefix> scoring_prefixes;

‎be/src/storage/index/inverted/reader/text_index_reader.h‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
#include <functional>
2121
#include <string_view>
2222

23+
#include "storage/index/inverted/common/query_info.h"
2324
#include "storage/index/inverted/inverted_index_reader.h"
2425

2526
namespace doris::segment_v2 {
@@ -123,6 +124,10 @@ class TextIndexReader : public InvertedIndexReader {
123124
virtual Status _term_document_frequency(const std::string& column_name, OpenedIndex& index,
124125
const std::string& term, uint64_t* df,
125126
uint64_t* document_count) = 0;
127+
// The name of this index's statistics in the query's CollectionStatistics.
128+
std::wstring _statistics_field(const std::wstring& field) const {
129+
return match_statistics_field(field, _index_meta.index_id());
130+
}
126131
// Runs `leaf` over the open index into `out`. With `candidates`, a phrase matches only those
127132
// rows; with `scoring`, the BM25 values reach the context's collection similarity.
128133
virtual Status _run_leaf(const IndexQueryContextPtr& context, const std::string& column_name,

‎be/src/storage/index/inverted/snii/snii_index_reader.cpp‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -311,8 +311,8 @@ Status SniiIndexReader::_run_leaf(const IndexQueryContextPtr& context,
311311
auto source = _bind_source(context, field, index);
312312
const uint32_t doc_count = source->doc_count();
313313
auto result = std::make_shared<roaring::Roaring>();
314-
RETURN_IF_ERROR_OR_CATCH_EXCEPTION(run_leaf(context, field, leaf, candidates, scoring,
315-
std::move(source), doc_count, result));
314+
RETURN_IF_ERROR_OR_CATCH_EXCEPTION(run_leaf(context, _statistics_field(field), leaf, candidates,
315+
scoring, std::move(source), doc_count, result));
316316
result->runOptimize();
317317
*out = std::move(result);
318318
return Status::OK();

0 commit comments

Comments
 (0)