Skip to content

Commit e145fed

Browse files
airborne12airborne12
committed
[fix](inverted index) Preserve legacy phrase metadata behavior
### What problem does this PR solve? Issue Number: CIR-22263 Related PR: #69192 Problem Summary: Inferring phrase positions from an omitted support_phrase property changes how older docs-only indexes are read and adds positions to TOKENIZE output. Newly created tokenized indexes already persist support_phrase=true in FE. Keep an absent property false when reading persisted metadata, while allowing normalizer-only single-term indexes to use phrase predicates and similarity scoring without stored positions. Update SHOW INDEX expectations for parser=none. ### Release note Existing indexes with an omitted support_phrase property retain their stored docs-only interpretation. ### Check List (For Author) - Test: Unit Test: focused FE IndexTest and two BE tests passed. - Behavior changed: Yes. Legacy omitted phrase metadata remains docs-only and normalizer-only queries remain available. - Does this need documentation: No
1 parent ee2af95 commit e145fed

5 files changed

Lines changed: 52 additions & 13 deletions

File tree

‎be/src/storage/index/index_reader_helper.h‎

Lines changed: 20 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -17,13 +17,26 @@
1717

1818
#pragma once
1919

20+
#include <map>
21+
#include <string>
22+
2023
#include "storage/index/index_iterator.h"
2124
#include "storage/index/inverted/inverted_index_reader.h"
2225

2326
namespace doris::segment_v2 {
2427

2528
class IndexReaderHelper {
2629
public:
30+
static bool is_normalizer_only(const std::map<std::string, std::string>& properties) {
31+
const auto normalizer = properties.find(INVERTED_INDEX_NORMALIZER_NAME_KEY);
32+
const auto analyzer = properties.find(INVERTED_INDEX_ANALYZER_NAME_KEY);
33+
// A normalizer keeps the whole value as one term, so phrase queries need no positions.
34+
return !properties.contains(INVERTED_INDEX_PARSER_PHRASE_SUPPORT_KEY) &&
35+
normalizer != properties.end() && !normalizer->second.empty() &&
36+
(analyzer == properties.end() || analyzer->second.empty()) &&
37+
get_parser_string_from_properties(properties) == INVERTED_INDEX_PARSER_NONE;
38+
}
39+
2740
static bool is_fulltext_index(const IndexReaderPtr& reader) {
2841
if (reader == nullptr || reader->index_type() != IndexType::INVERTED) {
2942
return false;
@@ -59,7 +72,8 @@ class IndexReaderHelper {
5972
auto inverted_index_reader = std::static_pointer_cast<InvertedIndexReader>(reader);
6073
const auto& properties = inverted_index_reader->get_index_properties();
6174
return get_parser_phrase_support_string_from_properties(properties) ==
62-
INVERTED_INDEX_PARSER_PHRASE_SUPPORT_YES;
75+
INVERTED_INDEX_PARSER_PHRASE_SUPPORT_YES ||
76+
is_normalizer_only(properties);
6377
}
6478

6579
// only string type or bkd index reader can be used for equal
@@ -96,7 +110,8 @@ class IndexReaderHelper {
96110
query_type == InvertedIndexQueryType::MATCH_PHRASE_PREFIX_QUERY) {
97111
const auto& properties = index_meta->properties();
98112
if (get_parser_phrase_support_string_from_properties(properties) ==
99-
INVERTED_INDEX_PARSER_PHRASE_SUPPORT_YES) {
113+
INVERTED_INDEX_PARSER_PHRASE_SUPPORT_YES ||
114+
is_normalizer_only(properties)) {
100115
return true;
101116
}
102117
}
@@ -110,12 +125,13 @@ class IndexReaderHelper {
110125
query_type == TExprOpcode::MATCH_PHRASE_PREFIX) {
111126
const auto& properties = index_meta->properties();
112127
if (get_parser_phrase_support_string_from_properties(properties) ==
113-
INVERTED_INDEX_PARSER_PHRASE_SUPPORT_YES) {
128+
INVERTED_INDEX_PARSER_PHRASE_SUPPORT_YES ||
129+
is_normalizer_only(properties)) {
114130
return true;
115131
}
116132
}
117133
return false;
118134
}
119135
};
120136

121-
} // namespace doris::segment_v2
137+
} // namespace doris::segment_v2

‎be/src/storage/index/inverted/analysis/inverted_index_parser.cpp‎

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -121,11 +121,7 @@ std::string get_parser_phrase_support_string_from_properties(
121121
it != properties.end()) {
122122
return it->second;
123123
}
124-
const auto analyzer = properties.find(INVERTED_INDEX_ANALYZER_NAME_KEY);
125-
return get_parser_string_from_properties(properties) != INVERTED_INDEX_PARSER_NONE ||
126-
(analyzer != properties.end() && !analyzer->second.empty())
127-
? INVERTED_INDEX_PARSER_PHRASE_SUPPORT_YES
128-
: INVERTED_INDEX_PARSER_PHRASE_SUPPORT_NO;
124+
return INVERTED_INDEX_PARSER_PHRASE_SUPPORT_NO;
129125
}
130126

131127
bool should_write_index_norms(const TabletIndex& index_meta) {

‎be/test/storage/index/inverted/analysis/inverted_index_parser_test.cpp‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -136,12 +136,12 @@ TEST_F(InvertedIndexParserTest, TestGetParserPhraseSupportStringFromProperties)
136136

137137
properties[INVERTED_INDEX_PARSER_KEY] = INVERTED_INDEX_PARSER_ENGLISH;
138138
EXPECT_EQ(get_parser_phrase_support_string_from_properties(properties),
139-
INVERTED_INDEX_PARSER_PHRASE_SUPPORT_YES);
139+
INVERTED_INDEX_PARSER_PHRASE_SUPPORT_NO);
140140

141141
properties.clear();
142142
properties[INVERTED_INDEX_ANALYZER_NAME_KEY] = "english";
143143
EXPECT_EQ(get_parser_phrase_support_string_from_properties(properties),
144-
INVERTED_INDEX_PARSER_PHRASE_SUPPORT_YES);
144+
INVERTED_INDEX_PARSER_PHRASE_SUPPORT_NO);
145145

146146
properties.clear();
147147
properties[INVERTED_INDEX_NORMALIZER_NAME_KEY] = "lowercase";

‎be/test/storage/segment/index_reader_helper_test.cpp‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -396,6 +396,33 @@ TEST_F(IndexReaderHelperTest, IsNeedSimilarityScoreWithInvertedIndexQueryTypeTes
396396
InvertedIndexQueryType::LESS_THAN_QUERY, &index_meta));
397397
}
398398

399+
TEST_F(IndexReaderHelperTest, NormalizerOnlyPhraseAndScoringNeedNoPositions) {
400+
TabletIndexPB index_pb;
401+
index_pb.set_index_type(IndexType::INVERTED);
402+
index_pb.set_index_id(2);
403+
index_pb.set_index_name("normalized");
404+
index_pb.add_col_unique_id(1);
405+
(*index_pb.mutable_properties())[INVERTED_INDEX_NORMALIZER_NAME_KEY] = "lowercase";
406+
TabletIndex index_meta;
407+
index_meta.init_from_pb(index_pb);
408+
409+
auto reader = std::make_shared<MockInvertedIndexReader>(InvertedIndexReaderType::FULLTEXT,
410+
index_meta.properties());
411+
EXPECT_EQ(get_parser_phrase_support_string_from_properties(index_meta.properties()),
412+
INVERTED_INDEX_PARSER_PHRASE_SUPPORT_NO);
413+
EXPECT_TRUE(IndexReaderHelper::is_support_phrase(reader));
414+
EXPECT_TRUE(IndexReaderHelper::is_need_similarity_score(InvertedIndexQueryType::MATCH_ANY_QUERY,
415+
&index_meta));
416+
EXPECT_TRUE(IndexReaderHelper::is_need_similarity_score(TExprOpcode::MATCH_ANY, &index_meta));
417+
418+
auto explicit_disabled = index_meta.properties();
419+
explicit_disabled[INVERTED_INDEX_PARSER_PHRASE_SUPPORT_KEY] =
420+
INVERTED_INDEX_PARSER_PHRASE_SUPPORT_NO;
421+
auto disabled_reader = std::make_shared<MockInvertedIndexReader>(
422+
InvertedIndexReaderType::FULLTEXT, explicit_disabled);
423+
EXPECT_FALSE(IndexReaderHelper::is_support_phrase(disabled_reader));
424+
}
425+
399426
TEST_F(IndexReaderHelperTest, IsNeedSimilarityScoreWithTExprOpcodeTest) {
400427
TabletIndex index_meta;
401428
auto index_meta_pb = std::make_unique<TabletIndexPB>();

‎regression-test/suites/index_p0/test_index_meta.groovy‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -71,7 +71,7 @@ suite("index_meta", "p0") {
7171
assertEquals(show_result[1][4], "name")
7272
assertEquals(show_result[1][10], "INVERTED")
7373
assertEquals(show_result[1][11], "index for name")
74-
assertEquals(show_result[1][12], "(\"lower_case\" = \"true\", \"parser\" = \"none\", \"support_phrase\" = \"true\")")
74+
assertEquals(show_result[1][12], "(\"lower_case\" = \"true\", \"parser\" = \"none\")")
7575

7676
// add index on column description
7777
sql "create index idx_desc on ${tableName}(description) USING INVERTED PROPERTIES(\"parser\"=\"standard\") COMMENT 'index for description';"
@@ -90,7 +90,7 @@ suite("index_meta", "p0") {
9090
assertEquals(show_result[1][4], "name")
9191
assertEquals(show_result[1][10], "INVERTED")
9292
assertEquals(show_result[1][11], "index for name")
93-
assertEquals(show_result[1][12], "(\"lower_case\" = \"true\", \"parser\" = \"none\", \"support_phrase\" = \"true\")")
93+
assertEquals(show_result[1][12], "(\"lower_case\" = \"true\", \"parser\" = \"none\")")
9494
assertEquals(show_result[2][2], "idx_desc")
9595
assertEquals(show_result[2][4], "description")
9696
assertEquals(show_result[2][10], "INVERTED")

0 commit comments

Comments
 (0)