Skip to content

Commit 34a123b

Browse files
committed
[fix](be) Score regexp and wildcard search terms with BM25
1 parent 379f648 commit 34a123b

4 files changed

Lines changed: 136 additions & 15 deletions

File tree

‎be/src/storage/index/inverted/query_v2/regexp_query/regexp_weight.cpp‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,12 +34,18 @@
3434
#include <gen_cpp/PaloBrokerService_types.h>
3535

3636
#include <algorithm>
37+
#include <cmath>
3738
#include <string>
39+
#include <vector>
3840

3941
#include "storage/index/inverted/query_v2/bit_set_query/bit_set_scorer.h"
4042
#include "storage/index/inverted/query_v2/const_score_query/const_score_scorer.h"
4143
#include "storage/index/inverted/query_v2/nullable_scorer.h"
44+
#include "storage/index/inverted/query_v2/score_combiner.h"
4245
#include "storage/index/inverted/query_v2/segment_postings.h"
46+
#include "storage/index/inverted/query_v2/term_query/term_scorer.h"
47+
#include "storage/index/inverted/query_v2/union/buffered_union.h"
48+
#include "storage/index/inverted/similarity/bm25_similarity.h"
4349
#include "storage/index/inverted/util/string_helper.h"
4450
#include "util/hyperscan_util.h"
4551

@@ -110,6 +116,10 @@ ScorerPtr RegexpWeight::regexp_scorer(const QueryExecutionContext& context,
110116
}
111117

112118
auto reader = lookup_reader(_field, context, binding_key);
119+
if (_enable_scoring) {
120+
return scored_terms_scorer(reader.get(), context, binding_key, matching_terms);
121+
}
122+
113123
auto doc_bitset = std::make_shared<roaring::Roaring>();
114124
for (const auto& term : matching_terms) {
115125
auto segment_postings =
@@ -126,6 +136,46 @@ ScorerPtr RegexpWeight::regexp_scorer(const QueryExecutionContext& context,
126136
return const_score;
127137
}
128138

139+
ScorerPtr RegexpWeight::scored_terms_scorer(lucene::index::IndexReader* reader,
140+
const QueryExecutionContext& context,
141+
const std::string& binding_key,
142+
const std::vector<std::wstring>& matching_terms) {
143+
// Expanded terms are not collected in collection statistics, so use reader statistics.
144+
const auto num_docs = reader->maxDoc();
145+
DORIS_CHECK_GT(num_docs, 0);
146+
const auto total_term_freq = reader->sumTotalTermFreq(_field.c_str());
147+
// Without norms, postings use a document length of one.
148+
const float avgdl = static_cast<float>(total_term_freq.value_or(num_docs)) / num_docs;
149+
DORIS_CHECK_GT(avgdl, 0.0F);
150+
151+
std::vector<ScorerPtr> term_scorers;
152+
term_scorers.reserve(matching_terms.size());
153+
auto logical_field = logical_field_or_fallback(context, binding_key, _field);
154+
for (const auto& term : matching_terms) {
155+
Term lucene_term(_field.c_str(), term.c_str());
156+
const int32_t doc_freq = reader->docFreq(&lucene_term);
157+
if (doc_freq == 0) {
158+
continue;
159+
}
160+
const auto idf = static_cast<float>(
161+
std::log1p((static_cast<double>(num_docs) - doc_freq + 0.5) / (doc_freq + 0.5)));
162+
auto similarity = std::make_shared<BM25Similarity>(idf, avgdl);
163+
auto posting =
164+
create_term_posting(reader, _field, term, true, similarity, _context->io_ctx);
165+
if (posting) {
166+
term_scorers.emplace_back(std::make_shared<TermScorer>(
167+
std::move(posting), std::move(similarity), logical_field));
168+
}
169+
}
170+
if (term_scorers.empty()) {
171+
return std::make_shared<EmptyScorer>();
172+
}
173+
if (term_scorers.size() == 1) {
174+
return std::move(term_scorers.front());
175+
}
176+
return make_buffered_union(term_scorers, std::make_shared<SumCombiner>());
177+
}
178+
129179
std::optional<std::string> RegexpWeight::get_regex_prefix(const std::string& pattern) {
130180
DBUG_EXECUTE_IF("RegexpQuery.get_regex_prefix", { return std::nullopt; });
131181

‎be/src/storage/index/inverted/query_v2/regexp_query/regexp_weight.h‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,10 @@ class RegexpWeight : public Weight {
3535

3636
private:
3737
ScorerPtr regexp_scorer(const QueryExecutionContext& context, const std::string& binding_key);
38+
ScorerPtr scored_terms_scorer(lucene::index::IndexReader* reader,
39+
const QueryExecutionContext& context,
40+
const std::string& binding_key,
41+
const std::vector<std::wstring>& matching_terms);
3842

3943
std::optional<std::string> get_regex_prefix(const std::string& pattern);
4044
void collect_matching_terms(const QueryExecutionContext& context,
@@ -46,7 +50,7 @@ class RegexpWeight : public Weight {
4650

4751
std::wstring _field;
4852
std::string _pattern;
49-
[[maybe_unused]] bool _enable_scoring = false;
53+
bool _enable_scoring = false;
5054
bool _nullable = true;
5155
// Set to 0 to disable limit (ES has no default limit for prefix queries)
5256
// The limit prevents collecting too many terms, but can cause incorrect results

‎be/test/storage/index/inverted/query_v2/regexp_query_test.cpp‎

Lines changed: 37 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -59,10 +59,13 @@ class RegexpQueryV2Test : public testing::Test {
5959
private:
6060
void create_test_index(const std::string& field_name, const std::string& dir) {
6161
std::vector<std::string> test_data = {
62-
"apple123", "apple456", "banana789", "test123abc", "pattern456",
63-
"regex999", "match123", "search456", "prefix123", "suffix789",
64-
"apple_test", "banana_data", "test_pattern", "data_regex", "apple_banana",
65-
"test_match", "pattern_test", "prefix_suffix", "abc123xyz", "def456ghi"};
62+
"apple123", "apple456", "banana789",
63+
"test123abc", "pattern456", "regex999",
64+
"match123", "search456", "prefix123",
65+
"suffix789", "apple_test", "banana_data",
66+
"test_pattern", "data_regex", "apple_banana",
67+
"test_match", "pattern_test", "prefix_suffix",
68+
"abc123xyz", "def456ghi", "apple789 apple789 apple456"};
6669

6770
CustomAnalyzerConfig::Builder builder;
6871
builder.with_tokenizer_config("standard", {});
@@ -168,6 +171,35 @@ TEST_F(RegexpQueryV2Test, test_regexp_query_with_scoring) {
168171
ASSERT_NE(regexp_weight, nullptr);
169172
}
170173

174+
TEST_F(RegexpQueryV2Test, test_regexp_query_scores_matching_terms) {
175+
auto context = std::make_shared<IndexQueryContext>();
176+
context->collection_statistics = std::make_shared<CollectionStatistics>();
177+
context->collection_similarity = std::make_shared<CollectionSimilarity>();
178+
179+
auto* dir = FSDirectory::getDirectory(kTestDir.c_str());
180+
auto reader_holder = make_shared_reader(lucene::index::IndexReader::open(dir, true));
181+
ASSERT_NE(reader_holder, nullptr);
182+
183+
std::wstring field = StringHelper::to_wstring("content");
184+
auto query = std::make_shared<query_v2::RegexpQuery>(context, field, "apple.*");
185+
auto weight = query->weight(true);
186+
187+
query_v2::QueryExecutionContext exec_ctx;
188+
exec_ctx.segment_num_rows = reader_holder->maxDoc();
189+
exec_ctx.readers = {reader_holder};
190+
exec_ctx.field_reader_bindings.emplace(field, reader_holder);
191+
192+
auto scorer = weight->scorer(exec_ctx);
193+
ASSERT_EQ(scorer->seek(0), 0);
194+
float single_term_score = scorer->score();
195+
EXPECT_NE(single_term_score, 1.0F);
196+
uint32_t last_doc = reader_holder->maxDoc() - 1;
197+
ASSERT_EQ(scorer->seek(last_doc), last_doc);
198+
EXPECT_GT(scorer->score(), single_term_score);
199+
200+
_CLDECDELETE(dir);
201+
}
202+
171203
// Test regexp query execution
172204
TEST_F(RegexpQueryV2Test, test_regexp_query_execution) {
173205
auto context = std::make_shared<IndexQueryContext>();
@@ -575,7 +607,7 @@ TEST_F(RegexpQueryV2Test, test_make_exact_match_wildcard_pattern) {
575607
doc = scorer->advance();
576608
}
577609

578-
EXPECT_EQ(result.cardinality(), 20);
610+
EXPECT_EQ(result.cardinality(), 21);
579611

580612
_CLDECDELETE(dir);
581613
}

‎be/test/storage/index/inverted/query_v2/wildcard_query_test.cpp‎

Lines changed: 44 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -58,14 +58,20 @@ class WildcardQueryV2Test : public testing::Test {
5858

5959
private:
6060
void create_test_index(const std::string& field_name, const std::string& dir) {
61-
std::vector<std::string> test_data = {
62-
"apple", "application", "apply", "apricot", "banana", "band",
63-
"bandana", "can", "candy", "cat", "catalog", "dog",
64-
"document", "door", "hello", "help", "helpful", "test",
65-
"testing", "testcase", "wildcard", "wild", "wilderness", "card",
66-
"cardboard", "abc", "abcd", "abcde", "prefix123", "123suffix",
67-
"pre123", "123suf", "both123both", "star", "start", "started",
68-
"starter", "question", "quest", "query", "queries"};
61+
std::vector<std::string> test_data = {"apple", "application", "apply",
62+
"apricot", "banana", "band",
63+
"bandana", "can", "candy",
64+
"cat", "catalog", "dog",
65+
"document", "door", "hello",
66+
"help", "helpful", "test",
67+
"testing", "testcase", "wildcard",
68+
"wild", "wilderness", "card",
69+
"cardboard", "abc", "abcd",
70+
"abcde", "prefix123", "123suffix",
71+
"pre123", "123suf", "both123both",
72+
"star", "start", "started",
73+
"starter", "question", "quest",
74+
"query", "queries", "test test testing"};
6975

7076
CustomAnalyzerConfig::Builder builder;
7177
builder.with_tokenizer_config("standard", {});
@@ -156,6 +162,35 @@ TEST_F(WildcardQueryV2Test, test_wildcard_query_with_scoring) {
156162
ASSERT_NE(wildcard_weight, nullptr);
157163
}
158164

165+
TEST_F(WildcardQueryV2Test, test_wildcard_query_scores_matching_terms) {
166+
auto context = std::make_shared<IndexQueryContext>();
167+
context->collection_statistics = std::make_shared<CollectionStatistics>();
168+
context->collection_similarity = std::make_shared<CollectionSimilarity>();
169+
170+
auto* dir = FSDirectory::getDirectory(kTestDir.c_str());
171+
auto reader_holder = make_shared_reader(lucene::index::IndexReader::open(dir, true));
172+
ASSERT_NE(reader_holder, nullptr);
173+
174+
std::wstring field = StringHelper::to_wstring("field");
175+
auto query = std::make_shared<query_v2::WildcardQuery>(context, field, "test*");
176+
auto weight = query->weight(true);
177+
178+
query_v2::QueryExecutionContext exec_ctx;
179+
exec_ctx.segment_num_rows = reader_holder->maxDoc();
180+
exec_ctx.readers = {reader_holder};
181+
exec_ctx.field_reader_bindings.emplace(field, reader_holder);
182+
183+
auto scorer = weight->scorer(exec_ctx);
184+
ASSERT_EQ(scorer->seek(17), 17);
185+
float single_term_score = scorer->score();
186+
EXPECT_NE(single_term_score, 1.0F);
187+
uint32_t last_doc = reader_holder->maxDoc() - 1;
188+
ASSERT_EQ(scorer->seek(last_doc), last_doc);
189+
EXPECT_GT(scorer->score(), single_term_score);
190+
191+
_CLDECDELETE(dir);
192+
}
193+
159194
// Test wildcard query with asterisk prefix pattern
160195
TEST_F(WildcardQueryV2Test, test_wildcard_query_prefix_pattern) {
161196
auto context = std::make_shared<IndexQueryContext>();
@@ -525,4 +560,4 @@ TEST_F(WildcardQueryV2Test, test_wildcard_query_move_semantics) {
525560
ASSERT_NE(weight2, nullptr);
526561
}
527562

528-
} // namespace doris::segment_v2
563+
} // namespace doris::segment_v2

0 commit comments

Comments
 (0)