Skip to content

Commit 1072387

Browse files
Tanya-Wpull[bot]
authored andcommitted
[fix](match_phrase) Fix the inconsistent query result for 'match_phrase' after creating index without support_phrase property (#20258)
if create inverted index without support_phrase property, remaining the match_phrase condition to filter by match function.
1 parent 8922096 commit 1072387

11 files changed

Lines changed: 410 additions & 4 deletions

File tree

‎be/src/common/status.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -265,6 +265,7 @@ E(INVERTED_INDEX_FILE_NOT_FOUND, -6003);
265265
E(INVERTED_INDEX_FILE_HIT_LIMIT, -6004);
266266
E(INVERTED_INDEX_NO_TERMS, -6005);
267267
E(INVERTED_INDEX_RENAME_FILE_FAILED, -6006);
268+
E(INVERTED_INDEX_EVALUATE_SKIPPED, -6007);
268269
#undef E
269270
} // namespace ErrorCode
270271

@@ -294,6 +295,7 @@ constexpr bool capture_stacktrace() {
294295
&& code != ErrorCode::INVERTED_INDEX_FILE_NOT_FOUND
295296
&& code != ErrorCode::INVERTED_INDEX_FILE_HIT_LIMIT
296297
&& code != ErrorCode::INVERTED_INDEX_NO_TERMS
298+
&& code != ErrorCode::INVERTED_INDEX_EVALUATE_SKIPPED
297299
&& code != ErrorCode::META_KEY_NOT_FOUND
298300
&& code != ErrorCode::PUSH_VERSION_ALREADY_EXIST
299301
&& code != ErrorCode::TRANSACTION_NOT_EXIST

‎be/src/olap/match_predicate.cpp‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121

2222
#include "exec/olap_utils.h"
2323
#include "olap/field.h"
24+
#include "olap/inverted_index_parser.h"
2425
#include "olap/olap_common.h"
2526
#include "olap/rowset/segment_v2/inverted_index_cache.h"
2627
#include "olap/rowset/segment_v2/inverted_index_reader.h"
@@ -43,6 +44,10 @@ Status MatchPredicate::evaluate(const Schema& schema, InvertedIndexIterator* ite
4344
if (iterator == nullptr) {
4445
return Status::OK();
4546
}
47+
if (_skip_evaluate(iterator)) {
48+
LOG(INFO) << "match predicate evaluate skipped.";
49+
return Status::Error<ErrorCode::INVERTED_INDEX_EVALUATE_SKIPPED>();
50+
}
4651
auto column_desc = schema.column(_column_id);
4752
roaring::Roaring roaring;
4853
auto inverted_index_query_type = _to_inverted_index_query_type(_match_type);
@@ -111,4 +116,14 @@ InvertedIndexQueryType MatchPredicate::_to_inverted_index_query_type(MatchType m
111116
return ret;
112117
}
113118

119+
bool MatchPredicate::_skip_evaluate(InvertedIndexIterator* iterator) const {
120+
if (_match_type == MatchType::MATCH_PHRASE &&
121+
iterator->get_inverted_index_reader_type() == InvertedIndexReaderType::FULLTEXT &&
122+
get_parser_phrase_support_string_from_properties(iterator->get_index_properties()) ==
123+
INVERTED_INDEX_PARSER_PHRASE_SUPPORT_NO) {
124+
return true;
125+
}
126+
return false;
127+
}
128+
114129
} // namespace doris

‎be/src/olap/match_predicate.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,7 @@ class MatchPredicate : public ColumnPredicate {
6464
std::string info = "MatchPredicate";
6565
return info;
6666
}
67+
bool _skip_evaluate(InvertedIndexIterator* iterator) const;
6768

6869
private:
6970
std::string _value;

‎be/src/olap/rowset/segment_v2/inverted_index_reader.cpp‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -902,5 +902,9 @@ InvertedIndexReaderType InvertedIndexIterator::get_inverted_index_reader_type()
902902
return _reader->type();
903903
}
904904

905+
const std::map<string, string>& InvertedIndexIterator::get_index_properties() const {
906+
return _reader->get_index_properties();
907+
}
908+
905909
} // namespace segment_v2
906910
} // namespace doris

‎be/src/olap/rowset/segment_v2/inverted_index_reader.h‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -99,6 +99,10 @@ class InvertedIndexReader {
9999

100100
uint32_t get_index_id() const { return _index_meta.index_id(); }
101101

102+
const std::map<string, string>& get_index_properties() const {
103+
return _index_meta.properties();
104+
}
105+
102106
static std::vector<std::wstring> get_analyse_result(const std::string& field_name,
103107
const std::string& value,
104108
InvertedIndexQueryType query_type,
@@ -236,6 +240,7 @@ class InvertedIndexIterator {
236240
}
237241

238242
InvertedIndexReaderType get_inverted_index_reader_type() const;
243+
const std::map<string, string>& get_index_properties() const;
239244

240245
private:
241246
OlapReaderStatistics* _stats;

‎be/src/olap/rowset/segment_v2/segment_iterator.cpp‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -805,6 +805,7 @@ Status SegmentIterator::_apply_inverted_index_on_column_predicate(
805805
if (!res.ok()) {
806806
if (res.code() == ErrorCode::INVERTED_INDEX_FILE_NOT_FOUND ||
807807
res.code() == ErrorCode::INVERTED_INDEX_FILE_HIT_LIMIT ||
808+
res.code() == ErrorCode::INVERTED_INDEX_EVALUATE_SKIPPED ||
808809
(res.code() == ErrorCode::INVERTED_INDEX_NO_TERMS &&
809810
need_remaining_after_evaluate)) {
810811
// 1. INVERTED_INDEX_FILE_NOT_FOUND means index file has not been built,

‎be/src/vec/functions/match.cpp‎

Lines changed: 15 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -74,6 +74,11 @@ Status FunctionMatchAny::execute_match(const std::string& column_name,
7474
doris::segment_v2::InvertedIndexReader::get_analyse_result(
7575
column_name, match_query_str,
7676
doris::segment_v2::InvertedIndexQueryType::MATCH_ANY_QUERY, inverted_index_ctx);
77+
if (query_tokens.empty()) {
78+
LOG(WARNING) << "invalid input query_str: " << match_query_str
79+
<< ", please check your query sql";
80+
return Status::Error<ErrorCode::INVERTED_INDEX_NO_TERMS>();
81+
}
7782
for (int i = 0; i < input_rows_count; i++) {
7883
const auto& str_ref = datas->get_data_at(i);
7984
std::vector<std::wstring> data_tokens =
@@ -109,7 +114,11 @@ Status FunctionMatchAll::execute_match(const std::string& column_name,
109114
doris::segment_v2::InvertedIndexReader::get_analyse_result(
110115
column_name, match_query_str,
111116
doris::segment_v2::InvertedIndexQueryType::MATCH_ALL_QUERY, inverted_index_ctx);
112-
117+
if (query_tokens.empty()) {
118+
LOG(WARNING) << "invalid input query_str: " << match_query_str
119+
<< ", please check your query sql";
120+
return Status::Error<ErrorCode::INVERTED_INDEX_NO_TERMS>();
121+
}
113122
for (int i = 0; i < input_rows_count; i++) {
114123
const auto& str_ref = datas->get_data_at(i);
115124
std::vector<std::wstring> data_tokens =
@@ -152,7 +161,11 @@ Status FunctionMatchPhrase::execute_match(const std::string& column_name,
152161
column_name, match_query_str,
153162
doris::segment_v2::InvertedIndexQueryType::MATCH_PHRASE_QUERY,
154163
inverted_index_ctx);
155-
164+
if (query_tokens.empty()) {
165+
LOG(WARNING) << "invalid input query_str: " << match_query_str
166+
<< ", please check your query sql";
167+
return Status::Error<ErrorCode::INVERTED_INDEX_NO_TERMS>();
168+
}
156169
for (int i = 0; i < input_rows_count; i++) {
157170
const auto& str_ref = datas->get_data_at(i);
158171
std::vector<std::wstring> data_tokens =
Lines changed: 129 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,129 @@
1+
-- This file is automatically generated. You should know what you did if you want to edit this
2+
-- !sql --
3+
zhang san grade 5 zhang yi chen san buy dancing book 10 2017-10-01T00:00 tall:120cm, weight: 35kg, hobbies: sing, dancing Like cultural and recreational activities Class activists
4+
zhang san yi grade 5 zhang yi chen san buy 11 2017-10-01T00:00 tall:120cm, weight: 35kg, hobbies: reading book A quiet little boy learn makes me happy
5+
6+
-- !sql --
7+
li si grade 4 li er wan jiu 9 2018-10-01T00:00 tall:100cm, weight: 30kg, hobbies: playing ball A naughty boy i just want go outside
8+
9+
-- !sql --
10+
11+
-- !sql --
12+
san zhang grade 5 10 2017-10-01T00:00 tall:100cm, weight: 30kg, hobbies:
13+
14+
-- !sql --
15+
16+
-- !sql --
17+
san zhang grade 5 10 2017-10-01T00:00 tall:100cm, weight: 30kg, hobbies:
18+
zhang san grade 5 zhang yi chen san buy dancing book 10 2017-10-01T00:00 tall:120cm, weight: 35kg, hobbies: sing, dancing Like cultural and recreational activities Class activists
19+
zhang san yi grade 5 zhang yi chen san buy 11 2017-10-01T00:00 tall:120cm, weight: 35kg, hobbies: reading book A quiet little boy learn makes me happy
20+
21+
-- !sql --
22+
san zhang grade 5 10 2017-10-01T00:00 tall:100cm, weight: 30kg, hobbies:
23+
zhang san grade 5 zhang yi chen san buy dancing book 10 2017-10-01T00:00 tall:120cm, weight: 35kg, hobbies: sing, dancing Like cultural and recreational activities Class activists
24+
zhang san yi grade 5 zhang yi chen san buy 11 2017-10-01T00:00 tall:120cm, weight: 35kg, hobbies: reading book A quiet little boy learn makes me happy
25+
26+
-- !sql --
27+
28+
-- !sql --
29+
san zhang grade 5 10 2017-10-01T00:00 tall:100cm, weight: 30kg, hobbies:
30+
zhang san grade 5 zhang yi chen san buy dancing book 10 2017-10-01T00:00 tall:120cm, weight: 35kg, hobbies: sing, dancing Like cultural and recreational activities Class activists
31+
zhang san yi grade 5 zhang yi chen san buy 11 2017-10-01T00:00 tall:120cm, weight: 35kg, hobbies: reading book A quiet little boy learn makes me happy
32+
33+
-- !sql --
34+
35+
-- !sql --
36+
37+
-- !sql --
38+
zhang san grade 5 zhang yi chen san buy dancing book 10 2017-10-01T00:00 tall:120cm, weight: 35kg, hobbies: sing, dancing Like cultural and recreational activities Class activists
39+
zhang san yi grade 5 zhang yi chen san buy 11 2017-10-01T00:00 tall:120cm, weight: 35kg, hobbies: reading book A quiet little boy learn makes me happy
40+
41+
-- !sql --
42+
43+
-- !sql --
44+
li si grade 4 li er wan jiu 9 2018-10-01T00:00 tall:100cm, weight: 30kg, hobbies: playing ball A naughty boy i just want go outside
45+
san zhang grade 5 10 2017-10-01T00:00 tall:100cm, weight: 30kg, hobbies:
46+
47+
-- !sql --
48+
49+
-- !sql --
50+
li sisi grade 6 li ba li liuliu 11 2016-10-01T00:00 tall:150cm, weight: 40kg, hobbies: sing, dancing, running good at handiwork and beaty
51+
zhang san grade 5 zhang yi chen san buy dancing book 10 2017-10-01T00:00 tall:120cm, weight: 35kg, hobbies: sing, dancing Like cultural and recreational activities Class activists
52+
53+
-- !sql --
54+
zhang san yi grade 5 zhang yi chen san buy 11 2017-10-01T00:00 tall:120cm, weight: 35kg, hobbies: reading book A quiet little boy learn makes me happy
55+
56+
-- !sql --
57+
li si grade 4 li er wan jiu 9 2018-10-01T00:00 tall:100cm, weight: 30kg, hobbies: playing ball A naughty boy i just want go outside
58+
59+
-- !sql --
60+
61+
-- !sql --
62+
li si grade 4 li er wan jiu 9 2018-10-01T00:00 tall:100cm, weight: 30kg, hobbies: playing ball A naughty boy i just want go outside
63+
64+
-- !sql --
65+
66+
-- !sql --
67+
zhang san grade 5 zhang yi chen san buy dancing book 10 2017-10-01T00:00 tall:120cm, weight: 35kg, hobbies: sing, dancing Like cultural and recreational activities Class activists
68+
zhang san yi grade 5 zhang yi chen san buy 11 2017-10-01T00:00 tall:120cm, weight: 35kg, hobbies: reading book A quiet little boy learn makes me happy
69+
70+
-- !sql --
71+
li si grade 4 li er wan jiu 9 2018-10-01T00:00 tall:100cm, weight: 30kg, hobbies: playing ball A naughty boy i just want go outside
72+
73+
-- !sql --
74+
75+
-- !sql --
76+
san zhang grade 5 10 2017-10-01T00:00 tall:100cm, weight: 30kg, hobbies:
77+
78+
-- !sql --
79+
80+
-- !sql --
81+
san zhang grade 5 10 2017-10-01T00:00 tall:100cm, weight: 30kg, hobbies:
82+
zhang san grade 5 zhang yi chen san buy dancing book 10 2017-10-01T00:00 tall:120cm, weight: 35kg, hobbies: sing, dancing Like cultural and recreational activities Class activists
83+
zhang san yi grade 5 zhang yi chen san buy 11 2017-10-01T00:00 tall:120cm, weight: 35kg, hobbies: reading book A quiet little boy learn makes me happy
84+
85+
-- !sql --
86+
san zhang grade 5 10 2017-10-01T00:00 tall:100cm, weight: 30kg, hobbies:
87+
zhang san grade 5 zhang yi chen san buy dancing book 10 2017-10-01T00:00 tall:120cm, weight: 35kg, hobbies: sing, dancing Like cultural and recreational activities Class activists
88+
zhang san yi grade 5 zhang yi chen san buy 11 2017-10-01T00:00 tall:120cm, weight: 35kg, hobbies: reading book A quiet little boy learn makes me happy
89+
90+
-- !sql --
91+
92+
-- !sql --
93+
san zhang grade 5 10 2017-10-01T00:00 tall:100cm, weight: 30kg, hobbies:
94+
zhang san grade 5 zhang yi chen san buy dancing book 10 2017-10-01T00:00 tall:120cm, weight: 35kg, hobbies: sing, dancing Like cultural and recreational activities Class activists
95+
zhang san yi grade 5 zhang yi chen san buy 11 2017-10-01T00:00 tall:120cm, weight: 35kg, hobbies: reading book A quiet little boy learn makes me happy
96+
97+
-- !sql --
98+
99+
-- !sql --
100+
101+
-- !sql --
102+
zhang san grade 5 zhang yi chen san buy dancing book 10 2017-10-01T00:00 tall:120cm, weight: 35kg, hobbies: sing, dancing Like cultural and recreational activities Class activists
103+
zhang san yi grade 5 zhang yi chen san buy 11 2017-10-01T00:00 tall:120cm, weight: 35kg, hobbies: reading book A quiet little boy learn makes me happy
104+
105+
-- !sql --
106+
107+
-- !sql --
108+
li si grade 4 li er wan jiu 9 2018-10-01T00:00 tall:100cm, weight: 30kg, hobbies: playing ball A naughty boy i just want go outside
109+
san zhang grade 5 10 2017-10-01T00:00 tall:100cm, weight: 30kg, hobbies:
110+
111+
-- !sql --
112+
113+
-- !sql --
114+
li sisi grade 6 li ba li liuliu 11 2016-10-01T00:00 tall:150cm, weight: 40kg, hobbies: sing, dancing, running good at handiwork and beaty
115+
zhang san grade 5 zhang yi chen san buy dancing book 10 2017-10-01T00:00 tall:120cm, weight: 35kg, hobbies: sing, dancing Like cultural and recreational activities Class activists
116+
117+
-- !sql --
118+
zhang san yi grade 5 zhang yi chen san buy 11 2017-10-01T00:00 tall:120cm, weight: 35kg, hobbies: reading book A quiet little boy learn makes me happy
119+
120+
-- !sql --
121+
li si grade 4 li er wan jiu 9 2018-10-01T00:00 tall:100cm, weight: 30kg, hobbies: playing ball A naughty boy i just want go outside
122+
123+
-- !sql --
124+
125+
-- !sql --
126+
li si grade 4 li er wan jiu 9 2018-10-01T00:00 tall:100cm, weight: 30kg, hobbies: playing ball A naughty boy i just want go outside
127+
128+
-- !sql --
129+

regression-test/data/inverted_index_p0/test_index_match_term_and_phrase_select.out renamed to regression-test/data/inverted_index_p0/test_index_match_select.out

File renamed without changes.

0 commit comments

Comments
 (0)