Skip to content

Commit b24f0ea

Browse files
airborne12airborne12
committed
[feat](inverted-index) Backport IK custom analyzers to branch-4.1
### What problem does this PR solve? Related PR: #67918 Problem Summary: Custom inverted-index analyzers on branch-4.1 cannot use IK smart or max-word tokenization. Backport the tokenizer factories, source-byte offset tracking, analyzer identities, and saved-policy resolution from the merged PR. Adapt the FE and BE changes to the 4.1 layout and interfaces while preserving its NGram validation limits. ### Release note Custom inverted-index analyzers support ik_smart and ik_max_word tokenizers. ### Check List (For Author) - Test - [x] Regression test: the three added analyzer suites passed on an isolated ASAN cluster. - [x] Unit Test: 437 BE tests, 34 PolicyValidatorTests, and 62 AnalyzerIdentityBuilderTest tests passed. - [x] ASAN BE and BE Java extensions built. The local BE build used an ignored compatibility shim for unavailable Lance symbols; Lance behavior was not tested. - [x] FE build passed, including Checkstyle. - [x] Clang-format 16 checked all 79 changed C++ files; clang-tidy found no changed-line warnings after target-specific fixes. - Behavior changed: - [x] Yes. Custom IK tokenizers, analyzer selection, and source offsets work across DDL, indexing, and MATCH queries. - Does this need documentation? - [x] No. Existing custom-analyzer syntax is reused. (cherry picked from commit ef72b93) Conflicts: - FE analyzer code is in fe-core on branch-4.1, and the BE reader and test APIs differ. - branch-4.1 lacks max_ngram_diff, so its existing NGram bounds remain in use.
1 parent 388e935 commit b24f0ea

102 files changed

Lines changed: 13111 additions & 821 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎be/src/exprs/function/function_search.cpp‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -253,6 +253,31 @@ Status FunctionSearch::evaluate_inverted_index_with_search_param(
253253
const IndexExecContext* index_exec_ctx,
254254
const std::unordered_map<std::string, int>& field_name_to_column_id,
255255
const std::shared_ptr<IndexQueryContext>& index_query_context) const {
256+
// VSearchExpr enters here directly, outside IFunction::execute() and its exception boundary,
257+
// so this is where a failure inside the search, such as an analyzer whose first token
258+
// stream throws, has to become a Status.
259+
try {
260+
return evaluate_inverted_index_with_search_param_unguarded(
261+
search_param, data_type_with_names, std::move(iterators), num_rows, bitmap_result,
262+
enable_cache, index_exec_ctx, field_name_to_column_id, index_query_context);
263+
} catch (const CLuceneError& e) {
264+
return Status::Error<ErrorCode::INVERTED_INDEX_ANALYZER_ERROR>("search failed: {}",
265+
e.what());
266+
} catch (const Exception& e) {
267+
return e.to_status();
268+
}
269+
}
270+
271+
// The search body stays together so the public entry point can convert thrown exceptions.
272+
// NOLINTNEXTLINE(readability-function-cognitive-complexity,readability-function-size)
273+
Status FunctionSearch::evaluate_inverted_index_with_search_param_unguarded(
274+
const TSearchParam& search_param,
275+
const std::unordered_map<std::string, IndexFieldNameAndTypePair>& data_type_with_names,
276+
std::unordered_map<std::string, IndexIterator*> iterators, uint32_t num_rows,
277+
InvertedIndexResultBitmap& bitmap_result, bool enable_cache,
278+
const IndexExecContext* index_exec_ctx,
279+
const std::unordered_map<std::string, int>& field_name_to_column_id,
280+
const std::shared_ptr<IndexQueryContext>& index_query_context) const {
256281
const bool is_nested_query = search_param.root.clause_type == "NESTED";
257282
if (is_nested_query && !is_nested_group_search_supported()) {
258283
return Status::NotSupported(

‎be/src/exprs/function/function_search.h‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -93,6 +93,16 @@ class FunctionSearch : public IFunction {
9393
const std::unordered_map<std::string, int>& field_name_to_column_id,
9494
const std::shared_ptr<IndexQueryContext>& index_query_context = nullptr) const;
9595

96+
// The body of the overload above; it may throw, so only that overload calls it.
97+
Status evaluate_inverted_index_with_search_param_unguarded(
98+
const TSearchParam& search_param,
99+
const std::unordered_map<std::string, IndexFieldNameAndTypePair>& data_type_with_names,
100+
std::unordered_map<std::string, IndexIterator*> iterators, uint32_t num_rows,
101+
InvertedIndexResultBitmap& bitmap_result, bool enable_cache,
102+
const IndexExecContext* index_exec_ctx,
103+
const std::unordered_map<std::string, int>& field_name_to_column_id,
104+
const std::shared_ptr<IndexQueryContext>& index_query_context) const;
105+
96106
// Public methods for testing
97107
enum class ClauseTypeCategory {
98108
NON_TOKENIZED, // TERM, PREFIX, WILDCARD, REGEXP, RANGE, LIST - no tokenization, use EQUAL_QUERY

‎be/src/exprs/function/match.cpp‎

Lines changed: 7 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -63,14 +63,7 @@ Status FunctionMatchBase::evaluate_inverted_index(
6363
}
6464
const std::string& function_name = get_name();
6565

66-
if (function_name == MATCH_PHRASE_FUNCTION || function_name == MATCH_PHRASE_PREFIX_FUNCTION ||
67-
function_name == MATCH_PHRASE_EDGE_FUNCTION) {
68-
auto reader = iter->get_reader(InvertedIndexReaderType::FULLTEXT);
69-
if (reader && !segment_v2::IndexReaderHelper::is_support_phrase(reader)) {
70-
return Status::Error<ErrorCode::INDEX_INVALID_PARAMETERS>(
71-
"phrase queries require setting support_phrase = true");
72-
}
73-
}
66+
// support_phrase is checked once the analyzer has selected the reader that runs the query.
7467
Field param_value;
7568
arguments[0].column->get(0, param_value);
7669
if (param_value.is_null()) {
@@ -199,10 +192,8 @@ std::vector<segment_v2::TermInfo> FunctionMatchBase::analyse_query_str_token(
199192
VLOG_DEBUG << "begin to run " << get_name() << ", parser_type: "
200193
<< inverted_index_parser_type_to_string(analyzer_ctx->parser_type);
201194

202-
// Decision is based on parser_type (from index properties):
203-
// - PARSER_NONE: no tokenization (keyword/exact match)
204-
// - Other parsers: tokenize using the analyzer
205-
if (!analyzer_ctx->should_tokenize()) {
195+
// A named analyzer also requires analysis when the parser is none.
196+
if (!analyzer_ctx->requires_analysis()) {
206197
// Keyword index: all strings (including empty) are valid tokens for exact match.
207198
// Empty string is a valid value in keyword index and should be matchable.
208199
query_tokens.emplace_back(match_query_str);
@@ -235,15 +226,14 @@ inline std::vector<segment_v2::TermInfo> FunctionMatchBase::analyse_data_token(
235226
return data_tokens;
236227
}
237228

238-
// Determine tokenization strategy based on parser_type
239-
const bool should_tokenize =
240-
analyzer_ctx->should_tokenize() && analyzer_ctx->analyzer != nullptr;
229+
const bool requires_analysis =
230+
analyzer_ctx->requires_analysis() && analyzer_ctx->analyzer != nullptr;
241231

242232
if (array_offsets) {
243233
for (auto next_src_array_offset = (*array_offsets)[current_block_row_idx];
244234
current_src_array_offset < next_src_array_offset; ++current_src_array_offset) {
245235
const auto& str_ref = string_col->get_data_at(current_src_array_offset);
246-
if (!should_tokenize) {
236+
if (!requires_analysis) {
247237
data_tokens.emplace_back(str_ref.to_string());
248238
continue;
249239
}
@@ -256,7 +246,7 @@ inline std::vector<segment_v2::TermInfo> FunctionMatchBase::analyse_data_token(
256246
}
257247
} else {
258248
const auto& str_ref = string_col->get_data_at(current_block_row_idx);
259-
if (!should_tokenize) {
249+
if (!requires_analysis) {
260250
data_tokens.emplace_back(str_ref.to_string());
261251
} else {
262252
auto reader = doris::segment_v2::inverted_index::InvertedIndexAnalyzer::create_reader(

‎be/src/exprs/function/variant_inverted_index_search.cpp‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -64,8 +64,45 @@ void add_search_binding_diagnostic(const std::shared_ptr<IndexQueryContext>& con
6464
}
6565
}
6666

67+
InvertedIndexAnalyzerCtxSPtr build_analyzer_context_unsafe(
68+
const std::map<std::string, std::string>& properties, const std::string& analyzer_key) {
69+
InvertedIndexAnalyzerConfig config;
70+
config.analyzer_name = get_analyzer_name_from_properties(properties);
71+
config.parser_type = get_inverted_index_parser_type_from_string(
72+
get_parser_string_from_properties(properties));
73+
config.parser_mode = get_parser_mode_string_from_properties(properties);
74+
config.lower_case = get_parser_lowercase_from_properties(properties);
75+
config.stop_words = get_parser_stopwords_from_properties(properties);
76+
config.char_filter_map = get_parser_char_filter_map_from_properties(properties);
77+
78+
auto analyzer_context = std::make_shared<InvertedIndexAnalyzerCtx>();
79+
analyzer_context->analyzer_key = analyzer_key;
80+
analyzer_context->analyzer_name = config.analyzer_name;
81+
analyzer_context->parser_type = config.parser_type;
82+
analyzer_context->char_filter_map = config.char_filter_map;
83+
if (analyzer_context->requires_analysis()) {
84+
analyzer_context->analyzer_provider =
85+
inverted_index::InvertedIndexAnalyzer::create_analyzer_provider(&config);
86+
}
87+
return analyzer_context;
88+
}
89+
6790
} // namespace
6891

92+
// Replayed components can collide across policy families, so building the provider throws.
93+
Result<InvertedIndexAnalyzerCtxSPtr> build_search_analyzer_context(
94+
const std::map<std::string, std::string>& properties, const std::string& analyzer_key) {
95+
try {
96+
return build_analyzer_context_unsafe(properties, analyzer_key);
97+
} catch (const CLuceneError& error) {
98+
return ResultError(Status::Error<ErrorCode::INVERTED_INDEX_ANALYZER_ERROR>(
99+
"Build search analyzer failed: {}", error.what()));
100+
} catch (const Exception& error) {
101+
return ResultError(Status::Error<ErrorCode::INVERTED_INDEX_ANALYZER_ERROR>(
102+
"Build search analyzer failed: {}", error.what()));
103+
}
104+
}
105+
69106
FieldReaderResolver::FieldReaderResolver(
70107
const std::unordered_map<std::string, IndexFieldNameAndTypePair>& data_type_with_names,
71108
const std::unordered_map<std::string, IndexIterator*>& iterators,

‎be/src/exprs/function/variant_inverted_index_search.h‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -90,6 +90,11 @@ struct FieldReaderBinding {
9090
}
9191
};
9292

93+
// Build the analyzer context a SEARCH binding executes with, converting a failure to build the
94+
// analyzer provider into a Status instead of letting the exception escape a Status-returning caller.
95+
Result<InvertedIndexAnalyzerCtxSPtr> build_search_analyzer_context(
96+
const std::map<std::string, std::string>& properties, const std::string& analyzer_key);
97+
9398
class FieldReaderResolver {
9499
public:
95100
FieldReaderResolver(

‎be/src/exprs/vmatch_predicate.cpp‎

Lines changed: 39 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -58,11 +58,14 @@ namespace doris {
5858
using namespace doris::segment_v2;
5959

6060
VMatchPredicate::VMatchPredicate(const TExprNode& node) : VExpr(node) {
61-
// Step 1: Create configuration (stack-allocated temporary, follows SRP)
61+
const auto resolved = AnalyzerConfigParser::parse(
62+
node.match_predicate.analyzer_name, node.match_predicate.parser_type,
63+
node.match_predicate.parser_mode, node.match_predicate.parser_lowercase,
64+
node.match_predicate.char_filter_map);
65+
6266
InvertedIndexAnalyzerConfig config;
63-
config.analyzer_name = node.match_predicate.analyzer_name;
64-
config.parser_type =
65-
get_inverted_index_parser_type_from_string(node.match_predicate.parser_type);
67+
config.analyzer_name = resolved.provider_name;
68+
config.parser_type = resolved.parser_type;
6669
config.parser_mode = node.match_predicate.parser_mode;
6770
config.char_filter_map = node.match_predicate.char_filter_map;
6871
if (node.match_predicate.parser_lowercase) {
@@ -74,18 +77,40 @@ VMatchPredicate::VMatchPredicate(const TExprNode& node) : VExpr(node) {
7477
{ config.lower_case = ""; })
7578
config.stop_words = node.match_predicate.parser_stopwords;
7679

77-
// Step 2: Use config to create analyzer (factory method).
78-
// Always create analyzer based on parser_type for slow path (tables without index).
79-
// For index path, FullTextIndexReader will check analyzer_name to decide whether
80-
// to use this analyzer or fallback to index's own analyzer.
81-
_analyzer = inverted_index::InvertedIndexAnalyzer::create_analyzer(&config);
82-
83-
// Step 3: Create runtime context (only extract runtime-needed info)
8480
_analyzer_ctx = std::make_shared<InvertedIndexAnalyzerCtx>();
85-
_analyzer_ctx->analyzer_name = config.analyzer_name;
86-
_analyzer_ctx->parser_type = config.parser_type;
81+
_analyzer_ctx->analyzer_key = resolved.analyzer_key;
82+
_analyzer_ctx->analyzer_name = resolved.provider_name;
83+
_analyzer_ctx->parser_type = resolved.parser_type;
84+
85+
if (_analyzer_ctx->requires_analysis()) {
86+
std::string bound_name;
87+
std::string legacy_name;
88+
_analyzer_provider = inverted_index::InvertedIndexAnalyzer::create_analyzer_provider(
89+
&config, &bound_name, &legacy_name);
90+
_analyzer = _analyzer_provider->get_analyzer();
91+
if (!legacy_name.empty() && legacy_name != bound_name) {
92+
_analyzer_ctx->legacy_analyzer_key =
93+
AnalyzerConfigParser::parse(legacy_name, node.match_predicate.parser_type,
94+
node.match_predicate.parser_mode,
95+
node.match_predicate.parser_lowercase,
96+
node.match_predicate.char_filter_map)
97+
.analyzer_key;
98+
}
99+
if (bound_name != resolved.provider_name) {
100+
// Reader selection and query tokenization must use the same policy binding.
101+
_analyzer_ctx->analyzer_key =
102+
AnalyzerConfigParser::parse(bound_name, node.match_predicate.parser_type,
103+
node.match_predicate.parser_mode,
104+
node.match_predicate.parser_lowercase,
105+
node.match_predicate.char_filter_map)
106+
.analyzer_key;
107+
_analyzer_ctx->analyzer_name = std::move(bound_name);
108+
}
109+
}
110+
87111
_analyzer_ctx->char_filter_map = std::move(config.char_filter_map);
88112
_analyzer_ctx->analyzer = _analyzer;
113+
_analyzer_ctx->analyzer_provider = _analyzer_provider;
89114
}
90115

91116
VMatchPredicate::~VMatchPredicate() = default;
@@ -153,7 +178,7 @@ Status VMatchPredicate::evaluate_inverted_index(VExprContext* context, uint32_t
153178
}
154179

155180
const std::string& VMatchPredicate::get_analyzer_key() const {
156-
return _analyzer_ctx->analyzer_name;
181+
return _analyzer_ctx->analyzer_key;
157182
}
158183

159184
Status VMatchPredicate::execute_column(VExprContext* context, const Block* block,

‎be/src/exprs/vmatch_predicate.h‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,9 @@ class VMatchPredicate final : public VExpr {
5757
const std::string& expr_name() const override;
5858
const std::string& function_name() const;
5959
[[nodiscard]] const std::string& get_analyzer_key() const override;
60+
[[nodiscard]] const InvertedIndexAnalyzerCtx* query_analyzer_ctx() const {
61+
return _analyzer_ctx.get();
62+
}
6063

6164
std::string debug_string() const override;
6265

@@ -69,6 +72,7 @@ class VMatchPredicate final : public VExpr {
6972

7073
// Lifecycle management: holds ownership of the analyzer
7174
std::shared_ptr<lucene::analysis::Analyzer> _analyzer;
75+
segment_v2::inverted_index::AnalyzerProviderPtr _analyzer_provider;
7276

7377
// Runtime context: holds raw pointer to analyzer and necessary runtime info
7478
InvertedIndexAnalyzerCtxSPtr _analyzer_ctx;

0 commit comments

Comments
 (0)