Skip to content

Commit ee2af95

Browse files
committed
[fix](inverted index) Align phrase defaults with tokenization
1 parent f605375 commit ee2af95

4 files changed

Lines changed: 58 additions & 5 deletions

File tree

‎be/src/storage/index/inverted/analysis/inverted_index_parser.cpp‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -121,7 +121,11 @@ std::string get_parser_phrase_support_string_from_properties(
121121
it != properties.end()) {
122122
return it->second;
123123
}
124-
return INVERTED_INDEX_PARSER_PHRASE_SUPPORT_NO;
124+
const auto analyzer = properties.find(INVERTED_INDEX_ANALYZER_NAME_KEY);
125+
return get_parser_string_from_properties(properties) != INVERTED_INDEX_PARSER_NONE ||
126+
(analyzer != properties.end() && !analyzer->second.empty())
127+
? INVERTED_INDEX_PARSER_PHRASE_SUPPORT_YES
128+
: INVERTED_INDEX_PARSER_PHRASE_SUPPORT_NO;
125129
}
126130

127131
bool should_write_index_norms(const TabletIndex& index_meta) {

‎be/test/storage/index/inverted/analysis/inverted_index_parser_test.cpp‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -130,6 +130,29 @@ TEST_F(InvertedIndexParserTest, TestGetParserPhraseSupportStringFromProperties)
130130
EXPECT_EQ(get_parser_phrase_support_string_from_properties(properties),
131131
INVERTED_INDEX_PARSER_PHRASE_SUPPORT_NO);
132132

133+
properties[INVERTED_INDEX_PARSER_KEY] = INVERTED_INDEX_PARSER_NONE;
134+
EXPECT_EQ(get_parser_phrase_support_string_from_properties(properties),
135+
INVERTED_INDEX_PARSER_PHRASE_SUPPORT_NO);
136+
137+
properties[INVERTED_INDEX_PARSER_KEY] = INVERTED_INDEX_PARSER_ENGLISH;
138+
EXPECT_EQ(get_parser_phrase_support_string_from_properties(properties),
139+
INVERTED_INDEX_PARSER_PHRASE_SUPPORT_YES);
140+
141+
properties.clear();
142+
properties[INVERTED_INDEX_ANALYZER_NAME_KEY] = "english";
143+
EXPECT_EQ(get_parser_phrase_support_string_from_properties(properties),
144+
INVERTED_INDEX_PARSER_PHRASE_SUPPORT_YES);
145+
146+
properties.clear();
147+
properties[INVERTED_INDEX_NORMALIZER_NAME_KEY] = "lowercase";
148+
EXPECT_EQ(get_parser_phrase_support_string_from_properties(properties),
149+
INVERTED_INDEX_PARSER_PHRASE_SUPPORT_NO);
150+
151+
properties.clear();
152+
properties[INVERTED_INDEX_PARSER_KEY_ALIAS] = INVERTED_INDEX_PARSER_NONE;
153+
EXPECT_EQ(get_parser_phrase_support_string_from_properties(properties),
154+
INVERTED_INDEX_PARSER_PHRASE_SUPPORT_NO);
155+
133156
// Test with phrase support key present
134157
properties[INVERTED_INDEX_PARSER_PHRASE_SUPPORT_KEY] = INVERTED_INDEX_PARSER_PHRASE_SUPPORT_YES;
135158
EXPECT_EQ(get_parser_phrase_support_string_from_properties(properties),

‎fe/fe-catalog/src/main/java/org/apache/doris/catalog/Index.java‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -75,10 +75,9 @@ public Index(long indexId, String indexName, List<String> columns,
7575
this.comment = comment;
7676
if (indexType == IndexType.INVERTED) {
7777
if (this.properties != null && !this.properties.isEmpty()) {
78-
if (this.properties.containsKey(InvertedIndexProperties.INVERTED_INDEX_PARSER_KEY)
79-
|| this.properties.containsKey(InvertedIndexProperties.INVERTED_INDEX_PARSER_KEY_ALIAS)
80-
|| this.properties.containsKey(InvertedIndexProperties.INVERTED_INDEX_ANALYZER_NAME_KEY)
81-
|| this.properties.containsKey(InvertedIndexProperties.INVERTED_INDEX_NORMALIZER_NAME_KEY)) {
78+
if (!InvertedIndexProperties.INVERTED_INDEX_PARSER_NONE.equalsIgnoreCase(getInvertedIndexParser())
79+
|| StringUtils.isNotEmpty(this.properties.get(
80+
InvertedIndexProperties.INVERTED_INDEX_ANALYZER_NAME_KEY))) {
8281
String supportPhraseKey = InvertedIndexProperties
8382
.INVERTED_INDEX_SUPPORT_PHRASE_KEY;
8483
if (!this.properties.containsKey(supportPhraseKey)) {

‎fe/fe-core/src/test/java/org/apache/doris/catalog/IndexTest.java‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,9 +24,36 @@
2424

2525
import java.util.ArrayList;
2626
import java.util.List;
27+
import java.util.Map;
2728

2829
public class IndexTest {
2930

31+
@Test
32+
public void testInvertedIndexPhraseDefaultsFollowTokenization() {
33+
Index exact = new Index(1, "exact", List.of("col"), IndexType.INVERTED, Map.of(), "");
34+
Assertions.assertFalse(exact.getProperties().containsKey("support_phrase"));
35+
36+
Index keyword = new Index(2, "keyword", List.of("col"), IndexType.INVERTED,
37+
Map.of("parser", "none"), "");
38+
Assertions.assertFalse(keyword.getProperties().containsKey("support_phrase"));
39+
40+
Index normalized = new Index(3, "normalized", List.of("col"), IndexType.INVERTED,
41+
Map.of("normalizer", "lowercase"), "");
42+
Assertions.assertFalse(normalized.getProperties().containsKey("support_phrase"));
43+
44+
Index parsed = new Index(4, "parsed", List.of("col"), IndexType.INVERTED,
45+
Map.of("parser", "english"), "");
46+
Assertions.assertEquals("true", parsed.getProperties().get("support_phrase"));
47+
48+
Index analyzed = new Index(5, "analyzed", List.of("col"), IndexType.INVERTED,
49+
Map.of("analyzer", "english"), "");
50+
Assertions.assertEquals("true", analyzed.getProperties().get("support_phrase"));
51+
52+
Index disabled = new Index(6, "disabled", List.of("col"), IndexType.INVERTED,
53+
Map.of("parser", "english", "support_phrase", "false"), "");
54+
Assertions.assertEquals("false", disabled.getProperties().get("support_phrase"));
55+
}
56+
3057
@Test
3158
public void testGetColumnUniqueIds() {
3259
// Create test columns with unique IDs

0 commit comments

Comments
 (0)