Skip to content

Commit 4a10ba3

Browse files
committed
[fix](be) Preserve normalizer phrase and scoring behavior
### What problem does this PR solve? Issue Number: None Related PR: #69199 Problem Summary: A custom normalizer can split one value into multiple terms, so omitting phrase positions by default can turn indexed phrase queries into scans. A built-in lowercase normalizer emits one term, but its docs-only postings have no frequencies or norms, which caused scoring statistics collection to reject the index. Keep positions for custom normalizers and use unit frequency and document length when scoring built-in lowercase postings without positions or norms. ### Release note Custom normalizers retain phrase positions by default. Built-in lowercase normalizer indexes keep phrase matching and positive relevance scores with docs-only postings. ### Check List (For Author) - Test: FE unit tests for normalizer defaults and policy binding; BE unit tests for SNII postings, collection statistics, reader capabilities, and missing norms; C++ format and build hygiene checks. - Behavior changed: Yes. Custom normalizers retain positions by default, and docs-only lowercase postings remain eligible for scoring. - Does this need documentation: No
1 parent e145fed commit 4a10ba3

12 files changed

Lines changed: 195 additions & 31 deletions

File tree

‎be/src/storage/index/index_reader_helper.h‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717

1818
#pragma once
1919

20+
#include <boost/algorithm/string/predicate.hpp>
2021
#include <map>
2122
#include <string>
2223

@@ -30,9 +31,9 @@ class IndexReaderHelper {
3031
static bool is_normalizer_only(const std::map<std::string, std::string>& properties) {
3132
const auto normalizer = properties.find(INVERTED_INDEX_NORMALIZER_NAME_KEY);
3233
const auto analyzer = properties.find(INVERTED_INDEX_ANALYZER_NAME_KEY);
33-
// A normalizer keeps the whole value as one term, so phrase queries need no positions.
34+
// A custom normalizer can split a value into multiple terms that need positions.
3435
return !properties.contains(INVERTED_INDEX_PARSER_PHRASE_SUPPORT_KEY) &&
35-
normalizer != properties.end() && !normalizer->second.empty() &&
36+
normalizer != properties.end() && boost::iequals(normalizer->second, "lowercase") &&
3637
(analyzer == properties.end() || analyzer->second.empty()) &&
3738
get_parser_string_from_properties(properties) == INVERTED_INDEX_PARSER_NONE;
3839
}

‎be/src/storage/index/inverted/query/spi/postings_cursor.h‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,9 @@ struct PostingsBlock {
4040
return dense ? static_cast<uint32_t>(range_begin + ordinal) : docs[ordinal];
4141
}
4242
uint32_t freq_at(uint64_t ordinal) const { return freqs.empty() ? 1 : freqs[ordinal]; }
43-
uint32_t norm_at(uint64_t ordinal) const { return norms.empty() ? 1 : norms[ordinal]; }
43+
uint32_t norm_at(uint64_t ordinal) const {
44+
return norms.empty() || norms[ordinal] == 0 ? 1 : norms[ordinal];
45+
}
4446

4547
PostingsBlock suffix(uint64_t ordinal) const {
4648
auto result = *this;

‎be/src/storage/index/inverted/reader/collection_statistics.cpp‎

Lines changed: 21 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -43,8 +43,10 @@ namespace collection_statistics_detail {
4343

4444
Result<SniiScoringSegmentStats> resolve_snii_scoring_segment(uint64_t index_doc_count,
4545
uint64_t sum_total_term_freq,
46-
bool has_positions, bool has_norms) {
47-
if (!has_positions || !has_norms) {
46+
bool has_positions, bool has_norms,
47+
bool single_term_normalizer) {
48+
// Lowercase-normalized docs-only postings use unit frequency and length for scoring.
49+
if ((!has_positions || !has_norms) && !single_term_normalizer) {
4850
return ResultError(Status::Error<ErrorCode::INVERTED_INDEX_NOT_SUPPORTED, false>(
4951
"SNII scoring requires positions and norms; this segment was written without "
5052
"positions or without norms. Norms are left out when the index sets \"norms\" = "
@@ -262,7 +264,10 @@ Status CollectionStatistics::process_segment(const RowsetSharedPtr& rowset,
262264
const uint64_t segment_doc_count = scoring_stats.doc_count;
263265
RETURN_IF_ERROR(admit_snii_scoring_segment(
264266
ws_field_name, segment_doc_count, scoring_stats.token_count,
265-
scoring_stats.has_positions, scoring_stats.has_norms, &segment_accumulator));
267+
scoring_stats.has_positions, scoring_stats.has_norms,
268+
segment_v2::IndexReaderHelper::is_normalizer_only(
269+
collect_info.index_meta->properties()),
270+
&segment_accumulator));
266271

267272
for (const auto& logical_term_bytes : collect_info.unique_terms) {
268273
const auto logical_term =
@@ -332,9 +337,12 @@ Status CollectionStatistics::process_segment(const RowsetSharedPtr& rowset,
332337
// BM25 on an analyzed index needs the record length of every row, and CLucene keeps
333338
// them, together with the field's token count, in the norms. A segment written without
334339
// norms would feed a zero avgdl, or rank its rows as zero-length documents next to the
335-
// segments that have norms, so refuse to score the collection, as SNII does. An index
336-
// that is not analyzed never writes norms and is left as it is.
337-
if (!scoring_stats.has_norms &&
340+
// segments that have norms, so refuse to score the collection, as SNII does. A
341+
// single-term normalizer can instead use unit frequency and length.
342+
const bool single_term_normalizer = segment_v2::IndexReaderHelper::is_normalizer_only(
343+
collect_info.index_meta->properties());
344+
// Docs-only lowercase indexes have no norms, but each matching row has one term.
345+
if (!scoring_stats.has_norms && !single_term_normalizer &&
338346
segment_v2::inverted_index::InvertedIndexAnalyzer::should_analyzer(
339347
collect_info.index_meta->properties())) {
340348
return Status::Error<ErrorCode::INVERTED_INDEX_NOT_SUPPORTED>(
@@ -343,7 +351,10 @@ Status CollectionStatistics::process_segment(const RowsetSharedPtr& rowset,
343351
"for a variant path, when inverted_index_skip_norms_for_variant is on",
344352
seg_path, segment_v2::inverted_index::StringHelper::to_string(ws_field_name));
345353
}
346-
_total_num_tokens[ws_field_name] += scoring_stats.token_count;
354+
// CLucene omits term totals without norms; one token per row gives a unit-length estimate.
355+
_total_num_tokens[ws_field_name] += !scoring_stats.has_norms && single_term_normalizer
356+
? scoring_stats.doc_count
357+
: scoring_stats.token_count;
347358

348359
for (const auto& logical_term_bytes : collect_info.unique_terms) {
349360
const auto logical_term =
@@ -364,10 +375,11 @@ Status CollectionStatistics::process_segment(const RowsetSharedPtr& rowset,
364375

365376
Status CollectionStatistics::admit_snii_scoring_segment(
366377
const std::wstring& field_name, uint64_t index_doc_count, uint64_t sum_total_term_freq,
367-
bool has_positions, bool has_norms, SniiScoringSegmentAccumulator* segment_accumulator) {
378+
bool has_positions, bool has_norms, bool single_term_normalizer,
379+
SniiScoringSegmentAccumulator* segment_accumulator) {
368380
DORIS_CHECK(segment_accumulator != nullptr);
369381
auto segment_stats = collection_statistics_detail::resolve_snii_scoring_segment(
370-
index_doc_count, sum_total_term_freq, has_positions, has_norms);
382+
index_doc_count, sum_total_term_freq, has_positions, has_norms, single_term_normalizer);
371383
if (!segment_stats.has_value()) {
372384
clear();
373385
return segment_stats.error();

‎be/src/storage/index/inverted/reader/collection_statistics.h‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -86,7 +86,7 @@ class CollectionStatistics {
8686
io::IOContext* io_ctx);
8787
Status admit_snii_scoring_segment(const std::wstring& field_name, uint64_t index_doc_count,
8888
uint64_t sum_total_term_freq, bool has_positions,
89-
bool has_norms,
89+
bool has_norms, bool single_term_normalizer,
9090
SniiScoringSegmentAccumulator* segment_accumulator);
9191
void commit_snii_scoring_segment(SniiScoringSegmentAccumulator&& segment_accumulator);
9292
void clear();
@@ -121,12 +121,12 @@ struct SniiScoringSegmentStats {
121121
uint64_t token_count = 0;
122122
};
123123

124-
// SNII scoring requires positions (which provide term frequencies) and norms. The current writer
125-
// emits norms for every analyzed index with positions. Older segments without norms return
126-
// NOT_SUPPORTED until an index rebuild or compaction supplies them.
124+
// Single-term normalizers can score docs-only postings with frequency and length one.
125+
// Other analyzed indexes need positions and norms for accurate BM25 scores.
127126
Result<SniiScoringSegmentStats> resolve_snii_scoring_segment(uint64_t index_doc_count,
128127
uint64_t sum_total_term_freq,
129-
bool has_positions, bool has_norms);
128+
bool has_positions, bool has_norms,
129+
bool single_term_normalizer = false);
130130

131131
void add_term_doc_frequency(
132132
std::unordered_map<std::wstring, std::unordered_map<std::wstring, uint64_t>>*

‎be/test/storage/index/inverted/query/exec/block_doc_set_test.cpp‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,19 @@ class TestBlockCursor final : public PostingsCursor {
6767
size_t _next = 0;
6868
};
6969

70+
TEST(BlockDocSetTest, MissingNormsUseUnitDocumentLength) {
71+
const std::array<uint32_t, 1> docs {5};
72+
const std::array<uint32_t, 1> zero_norms {0};
73+
PostingsBlock block {.docs = docs,
74+
.freqs = {},
75+
.norms = zero_norms,
76+
.range_begin = 0,
77+
.range_end = 0,
78+
.dense = false};
79+
EXPECT_EQ(block.freq_at(0), 1U);
80+
EXPECT_EQ(block.norm_at(0), 1U);
81+
}
82+
7083
TEST(BlockDocSetTest, ReadFailureDoesNotBecomeEndOfPostings) {
7184
TestBlockCursor source({{.docs = {},
7285
.freqs = {},

‎be/test/storage/index/inverted/reader/collection_statistics_test.cpp‎

Lines changed: 55 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -414,7 +414,7 @@ class CollectionStatisticsTest : public ::testing::Test {
414414
return tablet_schema;
415415
}
416416

417-
TabletSchemaSPtr create_snii_schema(int64_t index_id = 1) {
417+
TabletSchemaSPtr create_snii_schema(int64_t index_id = 1, bool single_term_normalizer = false) {
418418
TabletSchemaPB schema_pb;
419419
schema_pb.set_keys_type(DUP_KEYS);
420420
schema_pb.set_inverted_index_storage_format(InvertedIndexStorageFormatPB::SNII);
@@ -431,8 +431,12 @@ class CollectionStatisticsTest : public ::testing::Test {
431431
index._index_id = index_id;
432432
index._index_type = IndexType::INVERTED;
433433
index._col_unique_ids.push_back(1);
434-
index._properties["parser"] = "standard";
435-
index._properties["support_phrase"] = "true";
434+
if (single_term_normalizer) {
435+
index._properties["normalizer"] = "lowercase";
436+
} else {
437+
index._properties["parser"] = "standard";
438+
index._properties["support_phrase"] = "true";
439+
}
436440
tablet_schema->append_index(std::move(index));
437441
return tablet_schema;
438442
}
@@ -470,7 +474,7 @@ class CollectionStatisticsTest : public ::testing::Test {
470474
}
471475

472476
// A normal analyzed SNII segment with positions and norms, as emitted for scoring indexes.
473-
Status write_snii_scoring_segment(const std::string& segment_path) {
477+
Status write_snii_scoring_segment(const std::string& segment_path, bool docs_only = false) {
474478
const std::string index_path_prefix {
475479
segment_v2::InvertedIndexDescriptor::get_index_file_path_prefix(segment_path)};
476480
io::FileWriterPtr file_writer;
@@ -486,8 +490,11 @@ class CollectionStatisticsTest : public ::testing::Test {
486490
snii::writer::TermPostings alpha;
487491
alpha.term = "alpha";
488492
alpha.docids = {0, 1};
489-
alpha.freqs = {1, 1};
490-
alpha.positions_flat = {0, 0};
493+
alpha.retain_positions = !docs_only;
494+
if (!docs_only) {
495+
alpha.freqs = {1, 1};
496+
alpha.positions_flat = {0, 0};
497+
}
491498

492499
snii::writer::TermPostings beta;
493500
beta.term = "beta";
@@ -497,10 +504,15 @@ class CollectionStatisticsTest : public ::testing::Test {
497504

498505
snii::writer::SniiIndexInput input;
499506
input.index_id = 1;
500-
input.config = snii::format::IndexConfig::kDocsPositions;
507+
input.config = docs_only ? snii::format::IndexConfig::kDocsOnly
508+
: snii::format::IndexConfig::kDocsPositions;
501509
input.doc_count = 2;
502-
input.encoded_norms = {snii::query::encode_norm(2), snii::query::encode_norm(1)};
503-
input.terms = {std::move(alpha), std::move(beta)};
510+
if (!docs_only) {
511+
input.encoded_norms = {snii::query::encode_norm(2), snii::query::encode_norm(1)};
512+
input.terms = {std::move(alpha), std::move(beta)};
513+
} else {
514+
input.terms = {std::move(alpha)};
515+
}
504516

505517
RETURN_IF_ERROR(writer.add_logical_index(input));
506518
RETURN_IF_ERROR(writer.finish());
@@ -655,6 +667,7 @@ class CollectionStatisticsTest : public ::testing::Test {
655667
uint64_t sum_total_term_freq = 0;
656668
bool has_positions = true;
657669
bool has_norms = true;
670+
bool single_term_normalizer = false;
658671
};
659672

660673
Status stage_snii_fields_for_test(
@@ -663,7 +676,8 @@ class CollectionStatisticsTest : public ::testing::Test {
663676
for (const auto& field : fields) {
664677
RETURN_IF_ERROR(statistics->admit_snii_scoring_segment(
665678
field.field_name, field.index_doc_count, field.sum_total_term_freq,
666-
field.has_positions, field.has_norms, segment_accumulator));
679+
field.has_positions, field.has_norms, field.single_term_normalizer,
680+
segment_accumulator));
667681
}
668682
return Status::OK();
669683
}
@@ -1006,6 +1020,27 @@ TEST_F(CollectionStatisticsTest, SniiScoringUsesPhysicalStatistics) {
10061020
expect_collected_term(L"1", L"alpha", 2);
10071021
}
10081022

1023+
TEST_F(CollectionStatisticsTest, SniiDocsOnlyNormalizerStillCollectsScoringStatistics) {
1024+
auto tablet_schema = create_snii_schema(1, /*single_term_normalizer=*/true);
1025+
const std::string segment_path = test_dir_ + "/snii_normalizer_0.dat";
1026+
auto write_status = write_snii_scoring_segment(segment_path, /*docs_only=*/true);
1027+
ASSERT_TRUE(write_status.ok()) << write_status;
1028+
1029+
auto rowset_meta = std::make_shared<collection_statistics::MockRowsetMeta>();
1030+
auto rowset = std::make_shared<collection_statistics::MockRowset>(tablet_schema, rowset_meta);
1031+
rowset->set_num_segments(1);
1032+
rowset->set_segment_path(0, segment_path);
1033+
auto reader = std::make_shared<collection_statistics::MockRowsetReader>(rowset);
1034+
std::vector<RowSetSplits> splits {RowSetSplits(reader)};
1035+
1036+
auto status = stats_->collect(runtime_state_.get(), splits, tablet_schema,
1037+
create_match_expr_contexts("alpha"), nullptr);
1038+
1039+
ASSERT_TRUE(status.ok()) << status;
1040+
expect_collected_stats(L"1", 2, 2);
1041+
expect_collected_term(L"1", L"alpha", 2);
1042+
}
1043+
10091044
TEST_F(CollectionStatisticsTest, SniiScoringLookupUsesCallerIoContext) {
10101045
snii::snii_test::ScopedEnv force_nonresident_dict("SNII_DICT_RESIDENT_MAX", "0");
10111046
auto tablet_schema = create_snii_schema();
@@ -1194,6 +1229,16 @@ TEST(CollectionStatisticsSniiScoringTest, ResolveRejectsSegmentWithoutPositions)
11941229
EXPECT_EQ(result.error().code(), ErrorCode::INVERTED_INDEX_NOT_SUPPORTED);
11951230
}
11961231

1232+
TEST(CollectionStatisticsSniiScoringTest, ResolveAcceptsSingleTermNormalizerWithoutPositions) {
1233+
auto result = resolve_snii_scoring_segment(3, 2, /*has_positions=*/false,
1234+
/*has_norms=*/false,
1235+
/*single_term_normalizer=*/true);
1236+
1237+
ASSERT_TRUE(result.has_value()) << result.error();
1238+
EXPECT_EQ(result->doc_count, 3U);
1239+
EXPECT_EQ(result->token_count, 2U);
1240+
}
1241+
11971242
TEST_F(CollectionStatisticsTest, CollectionStatisticsInstancesKeepAdmissionStateIsolated) {
11981243
CollectionStatistics first;
11991244
CollectionStatistics second;

‎be/test/storage/index/inverted/snii/snii_writer_test.cpp‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -137,6 +137,28 @@ class ScopedGramPolicies {
137137
doris::IndexPolicyMgr* previous_ = nullptr;
138138
};
139139

140+
class ScopedSplitNormalizerPolicy {
141+
public:
142+
ScopedSplitNormalizerPolicy() {
143+
auto* exec_env = doris::ExecEnv::GetInstance();
144+
previous_ = exec_env->index_policy_mgr();
145+
exec_env->_index_policy_mgr = &manager_;
146+
147+
doris::TIndexPolicy normalizer;
148+
normalizer.id = 9020;
149+
normalizer.name = "split_norm";
150+
normalizer.type = doris::TIndexPolicyType::NORMALIZER;
151+
normalizer.properties["token_filter"] = "word_delimiter";
152+
manager_.apply_policy_changes({normalizer}, {});
153+
}
154+
155+
~ScopedSplitNormalizerPolicy() { doris::ExecEnv::GetInstance()->_index_policy_mgr = previous_; }
156+
157+
private:
158+
doris::IndexPolicyMgr manager_;
159+
doris::IndexPolicyMgr* previous_ = nullptr;
160+
};
161+
140162
// A fatal assertion inside a helper FUNCTION only aborts the helper; the calling
141163
// test keeps running and may dereference state that failed to initialize (this
142164
// bit us as a null-analyzer SEGV). A macro expands in the test body, so the
@@ -551,6 +573,35 @@ TEST(SniiWriterNorms, WritesNormsFollowSharedNormsPolicy) {
551573
doris::config::inverted_index_skip_norms_for_variant = original_skip_norms_for_variant;
552574
}
553575

576+
TEST(SniiWriterTest, SplitNormalizerRetainsPhrasePositions) {
577+
ScopedSplitNormalizerPolicy policies;
578+
doris::TabletIndex index_meta;
579+
init_gram_index_meta(&index_meta, 9021,
580+
{{"normalizer", "split_norm"}, {"support_phrase", "true"}});
581+
582+
doris::segment_v2::SniiIndexColumnWriter writer(nullptr, &index_meta,
583+
doris::FieldType::OLAP_FIELD_TYPE_VARCHAR);
584+
ASSERT_OK(writer.init());
585+
ASSERT_EQ(writer.config_for_test(), IndexConfig::kDocsPositions);
586+
587+
const doris::Slice value("foo-bar");
588+
ASSERT_OK(writer.add_values("content", &value, 1));
589+
auto postings = writer.term_buffer_for_test()->finalize_sorted();
590+
bool saw_foo = false;
591+
bool saw_bar = false;
592+
for (const auto& posting : postings) {
593+
if (posting.term == "foo") {
594+
saw_foo = true;
595+
EXPECT_EQ(posting.positions_flat, std::vector<uint32_t>({1}));
596+
} else if (posting.term == "bar") {
597+
saw_bar = true;
598+
EXPECT_EQ(posting.positions_flat, std::vector<uint32_t>({2}));
599+
}
600+
}
601+
EXPECT_TRUE(saw_foo);
602+
EXPECT_TRUE(saw_bar);
603+
}
604+
554605
TEST(SniiWriterTest, GramTokenizerForcesDocsOnlyAndIsRecognised) {
555606
ScopedConfiguredDensity fixed_density;
556607
ScopedGramPolicies policies;

‎be/test/storage/segment/index_reader_helper_test.cpp‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -421,6 +421,22 @@ TEST_F(IndexReaderHelperTest, NormalizerOnlyPhraseAndScoringNeedNoPositions) {
421421
auto disabled_reader = std::make_shared<MockInvertedIndexReader>(
422422
InvertedIndexReaderType::FULLTEXT, explicit_disabled);
423423
EXPECT_FALSE(IndexReaderHelper::is_support_phrase(disabled_reader));
424+
425+
auto builtin_alias = index_meta.properties();
426+
builtin_alias[INVERTED_INDEX_NORMALIZER_NAME_KEY] = "LowerCase";
427+
auto alias_reader = std::make_shared<MockInvertedIndexReader>(InvertedIndexReaderType::FULLTEXT,
428+
builtin_alias);
429+
EXPECT_TRUE(IndexReaderHelper::is_support_phrase(alias_reader));
430+
431+
TabletIndexPB custom_pb = index_pb;
432+
(*custom_pb.mutable_properties())[INVERTED_INDEX_NORMALIZER_NAME_KEY] = "split_norm";
433+
TabletIndex custom_meta;
434+
custom_meta.init_from_pb(custom_pb);
435+
auto custom_reader = std::make_shared<MockInvertedIndexReader>(
436+
InvertedIndexReaderType::FULLTEXT, custom_meta.properties());
437+
EXPECT_FALSE(IndexReaderHelper::is_support_phrase(custom_reader));
438+
EXPECT_FALSE(IndexReaderHelper::is_need_similarity_score(
439+
InvertedIndexQueryType::MATCH_ANY_QUERY, &custom_meta));
424440
}
425441

426442
TEST_F(IndexReaderHelperTest, IsNeedSimilarityScoreWithTExprOpcodeTest) {

‎fe/fe-catalog/src/main/java/org/apache/doris/catalog/Index.java‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -75,9 +75,13 @@ public Index(long indexId, String indexName, List<String> columns,
7575
this.comment = comment;
7676
if (indexType == IndexType.INVERTED) {
7777
if (this.properties != null && !this.properties.isEmpty()) {
78+
String normalizerName = this.properties.get(
79+
InvertedIndexProperties.INVERTED_INDEX_NORMALIZER_NAME_KEY);
7880
if (!InvertedIndexProperties.INVERTED_INDEX_PARSER_NONE.equalsIgnoreCase(getInvertedIndexParser())
7981
|| StringUtils.isNotEmpty(this.properties.get(
80-
InvertedIndexProperties.INVERTED_INDEX_ANALYZER_NAME_KEY))) {
82+
InvertedIndexProperties.INVERTED_INDEX_ANALYZER_NAME_KEY))
83+
|| (StringUtils.isNotEmpty(normalizerName)
84+
&& !"lowercase".equalsIgnoreCase(normalizerName))) {
8185
String supportPhraseKey = InvertedIndexProperties
8286
.INVERTED_INDEX_SUPPORT_PHRASE_KEY;
8387
if (!this.properties.containsKey(supportPhraseKey)) {

0 commit comments

Comments
 (0)