Skip to content

Commit 511a8a0

Browse files
authored
[fix](be) Normalize SNII prefix syntax before analysis (apache#66874)
### What problem does this PR solve? Issue Number: None Related PR: apache#66052 Problem Summary: `SEARCH ... PREFIX` values retain their trailing DSL `*` marker in FE. SNII classifies a custom keyword analyzer as analyzed, but that analyzer preserves the marker as a literal byte. Consequently, a predicate such as `fail*` sent `fail*` to the phrase-prefix reader instead of the indexed prefix `fail` and returned no rows, while V3 matched the expected documents. This PR strips exactly the trailing DSL marker before SNII phrase-prefix analysis. Non-analyzed keyword fields continue to use `WILDCARD_QUERY` with the marker intact, ordinary WILDCARD clauses are unchanged, and the V3 path is unchanged. The change is query-only: it does not alter index writes or the on-disk format, so existing SNII indexes are fixed after upgrade without rebuilding. TDD evidence: - RED: a real custom keyword tokenizer/analyzer preserved `fail*`; `FunctionSearch::build_leaf_query` sent `fail*` and produced an empty bitmap instead of `{0, 2}`. - GREEN: the same path sends `fail`, retains `MATCH_PHRASE_PREFIX_QUERY`, and produces `{0, 2}`. - All 98 `FunctionSearchTest` cases pass. - `./build.sh --be -j 192` passes with ASAN. - `build-support/check-format.sh` passes. ### Release note Fix SEARCH PREFIX queries on SNII indexes that use a custom keyword analyzer.
1 parent 74b4418 commit 511a8a0

2 files changed

Lines changed: 91 additions & 17 deletions

File tree

‎be/src/exprs/function/function_search.cpp‎

Lines changed: 17 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -860,14 +860,10 @@ Status FunctionSearch::build_leaf_query(const TSearchClause& clause,
860860
!inverted_index::InvertedIndexAnalyzer::should_analyzer(
861861
binding.index_properties)) {
862862
// FE keeps the trailing '*' in the PREFIX value unstripped (SearchDslParser.java).
863-
// On an analysed field the tokenizer drops it, leaving a clean single prefix term,
864-
// so the default clause_type_to_query_type mapping (MATCH_PHRASE_PREFIX_QUERY) is
865-
// correct as-is. On a keyword (non-analysed) field the whole string -- '*' included
866-
// -- becomes one literal term (InvertedIndexAnalyzer::get_analyse_result), so
867-
// MATCH_PHRASE_PREFIX_QUERY would search for a term that can never exist. Route
868-
// those to WILDCARD_QUERY instead, exactly like the CLucene path's
869-
// WildcardQuery(value) for PREFIX (function_search.cpp:1075-1076): the reader
870-
// forwards a WILDCARD_QUERY value unanalysed, so the trailing '*' works the same way.
863+
// A non-analysed keyword field needs that marker for WILDCARD_QUERY, matching the
864+
// CLucene path's WildcardQuery(value) for PREFIX (function_search.cpp:1075-1076).
865+
// Analysed fields stay on MATCH_PHRASE_PREFIX_QUERY; its reader input is normalized
866+
// below because a custom keyword tokenizer preserves '*' as a literal byte.
871867
snii_query_type = InvertedIndexQueryType::WILDCARD_QUERY;
872868
}
873869

@@ -903,12 +899,17 @@ Status FunctionSearch::build_leaf_query(const TSearchClause& clause,
903899
if (clause_type == "WILDCARD" && value == "*") {
904900
data_bitmap->addRange(0, num_rows);
905901
} else {
906-
// Wildcard patterns carry the analyzer's lower_case semantics; every other clause
907-
// passes its value through untouched, since the reader analyses it.
908-
std::string pattern =
909-
clause_type == "WILDCARD"
910-
? normalize_wildcard_pattern(value, binding.index_properties)
911-
: value;
902+
// Wildcard patterns carry the analyzer's lower_case semantics. Phrase-prefix analysis
903+
// consumes the prefix text, not the DSL's trailing '*' syntax marker. This must happen
904+
// before analysis because custom keyword tokenizers preserve the marker as a literal.
905+
std::string pattern = value;
906+
if (clause_type == "WILDCARD") {
907+
pattern = normalize_wildcard_pattern(value, binding.index_properties);
908+
} else if (snii_query_type == InvertedIndexQueryType::MATCH_PHRASE_PREFIX_QUERY) {
909+
DORIS_CHECK(clause_type == "PREFIX");
910+
DORIS_CHECK(pattern.ends_with('*'));
911+
pattern.pop_back();
912+
}
912913
Field query_value = Field::create_field<TYPE_STRING>(pattern);
913914
const bool raw_pattern_query =
914915
snii_query_type == InvertedIndexQueryType::WILDCARD_QUERY ||
@@ -925,10 +926,9 @@ Status FunctionSearch::build_leaf_query(const TSearchClause& clause,
925926
if (reader_context != context) {
926927
context->merge_reader_outputs(*reader_context);
927928
}
928-
// Restore the pre-normalization value for WILDCARD so the trace still shows what the
929-
// caller actually asked for, not just what was sent to the reader.
929+
// Preserve the caller's pre-normalization value in the trace.
930930
std::string log_suffix =
931-
clause_type == "WILDCARD" ? (" (original='" + value + "')") : std::string();
931+
pattern != value ? (" (original='" + value + "')") : std::string();
932932
VLOG_DEBUG << "search: SNII clause processed, type=" << clause_type
933933
<< ", field=" << field_name << ", value='" << pattern << "'" << log_suffix;
934934
}

‎be/test/exprs/function/function_search_test.cpp‎

Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@
4242
#include "runtime/index_policy/index_policy_mgr.h"
4343
#include "storage/index/index_file_reader.h"
4444
#include "storage/index/index_iterator.h"
45+
#include "storage/index/inverted/analyzer/analyzer.h"
4546
#include "storage/index/inverted/inverted_index_iterator.h"
4647
#include "storage/index/inverted/inverted_index_parser.h"
4748
#include "storage/index/inverted/query_v2/collect/doc_set_collector.h"
@@ -2824,6 +2825,79 @@ TEST_F(FunctionSearchTest, TestSniiNativeKeywordPrefixRoutesToWildcardQuery) {
28242825
expect_bitmap_eq(collect_docs(scorer), {0, 2});
28252826
}
28262827

2828+
TEST_F(FunctionSearchTest, TestSniiNativeCustomKeywordPrefixStripsDslSuffixBeforeAnalysis) {
2829+
auto* exec_env = ExecEnv::GetInstance();
2830+
auto* previous_policy_mgr = exec_env->index_policy_mgr();
2831+
IndexPolicyMgr scoped_policy_mgr;
2832+
exec_env->_index_policy_mgr = &scoped_policy_mgr;
2833+
DEFER(exec_env->_index_policy_mgr = previous_policy_mgr);
2834+
2835+
TIndexPolicy tokenizer;
2836+
tokenizer.id = 910030;
2837+
tokenizer.name = "function_search_keyword_tokenizer";
2838+
tokenizer.type = TIndexPolicyType::TOKENIZER;
2839+
tokenizer.properties["type"] = "keyword";
2840+
2841+
TIndexPolicy analyzer;
2842+
analyzer.id = 910031;
2843+
analyzer.name = "function_search_keyword_analyzer";
2844+
analyzer.type = TIndexPolicyType::ANALYZER;
2845+
analyzer.properties["tokenizer"] = tokenizer.name;
2846+
scoped_policy_mgr.apply_policy_changes({tokenizer, analyzer}, {});
2847+
2848+
std::map<std::string, std::string> properties {
2849+
{INVERTED_INDEX_ANALYZER_NAME_KEY, analyzer.name}};
2850+
ASSERT_TRUE(inverted_index::InvertedIndexAnalyzer::should_analyzer(properties));
2851+
auto raw_terms = inverted_index::InvertedIndexAnalyzer::get_analyse_result(
2852+
"fail*", properties, inverted_index::AnalysisPurpose::kPhrasePrefixQuery);
2853+
ASSERT_EQ(1, raw_terms.size());
2854+
EXPECT_EQ("fail*", raw_terms[0].get_single_term());
2855+
2856+
OlapReaderStatistics stats;
2857+
auto context = std::make_shared<IndexQueryContext>();
2858+
context->stats = &stats;
2859+
auto index_meta = make_test_inverted_index(24, properties);
2860+
auto index_file_reader = std::make_shared<RejectingCluceneIndexFileReader>();
2861+
auto reader =
2862+
std::make_shared<RecordingNativeInvertedIndexReader>(&index_meta, index_file_reader);
2863+
reader->set_query_result("fail", make_bitmap({0, 2}));
2864+
segment_v2::InvertedIndexIterator iterator;
2865+
iterator.add_reader(segment_v2::InvertedIndexReaderType::FULLTEXT, reader);
2866+
2867+
std::unordered_map<std::string, IndexFieldNameAndTypePair> data_type_with_names;
2868+
data_type_with_names.emplace(
2869+
"body", IndexFieldNameAndTypePair {"body", std::make_shared<DataTypeString>()});
2870+
std::unordered_map<std::string, IndexIterator*> iterators;
2871+
iterators["body"] = &iterator;
2872+
TSearchFieldBinding field_binding;
2873+
field_binding.field_name = "body";
2874+
field_binding.index_properties = properties;
2875+
field_binding.__isset.index_properties = true;
2876+
FieldReaderResolver resolver(data_type_with_names, iterators, context, {field_binding});
2877+
2878+
auto clause = make_leaf_clause("PREFIX", "fail*");
2879+
inverted_index::query_v2::QueryPtr query;
2880+
std::string binding_key;
2881+
auto status = function_search->build_leaf_query(clause, context, resolver, &query, &binding_key,
2882+
"OR", 0, 4);
2883+
2884+
ASSERT_TRUE(status.ok()) << status.to_string();
2885+
ASSERT_NE(nullptr, query);
2886+
EXPECT_EQ(1, reader->query_calls);
2887+
EXPECT_EQ(InvertedIndexQueryType::MATCH_PHRASE_PREFIX_QUERY, reader->last_query_type);
2888+
EXPECT_EQ("fail", reader->last_query_value);
2889+
2890+
auto weight = query->weight(false);
2891+
ASSERT_NE(nullptr, weight);
2892+
inverted_index::query_v2::QueryExecutionContext exec_ctx;
2893+
exec_ctx.segment_num_rows = 4;
2894+
auto scorer = weight->scorer(exec_ctx, binding_key);
2895+
ASSERT_NE(nullptr, scorer);
2896+
expect_bitmap_eq(collect_docs(scorer), {0, 2});
2897+
2898+
scoped_policy_mgr.apply_policy_changes({}, {tokenizer.id, analyzer.id});
2899+
}
2900+
28272901
// Shared wiring for the SNII native SEARCH scoring tests: one fake SNII reader bound to field
28282902
// "body" behind a standard analyzer, plus the resolver build_leaf_query needs. The resolver keeps
28292903
// references to the maps, so they must be owned by something that outlives it.

0 commit comments

Comments
 (0)