Skip to content

Commit 74b4418

Browse files
authored
[fix](be) Validate SNII analyzer context before query cache (apache#66869)
### What problem does this PR solve? Issue Number: None Related PR: apache#66052 Problem Summary: SEARCH selected the correct SNII reader but passed no analyzer context to native queries. A cold CommonGrams query therefore bypassed while a warm raw-query cache entry could be returned before the immutable segment analyzer contract was validated. This PR builds the analyzer context from the selected reader properties, passes it to analyzed native queries, and defers analyzed raw-cache lookup until segment admission. Analyzer-independent wildcard and regexp queries keep their early cache path. ### Release note Fix SNII SEARCH queries on CommonGrams indexes so cold and cached queries validate the same analyzer contract.
1 parent ce6c03a commit 74b4418

7 files changed

Lines changed: 240 additions & 19 deletions

File tree

‎be/src/exprs/function/function_search.cpp‎

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -773,7 +773,12 @@ Status FunctionSearch::build_leaf_query(const TSearchClause& clause,
773773
};
774774

775775
FieldReaderBinding binding;
776-
RETURN_IF_ERROR(resolver.resolve(field_name, query_type, &binding));
776+
const bool require_analyzer_context = clause_type != "WILDCARD" && clause_type != "REGEXP";
777+
if (require_analyzer_context) {
778+
RETURN_IF_ERROR(resolver.resolve_with_analyzer_context(field_name, query_type, &binding));
779+
} else {
780+
RETURN_IF_ERROR(resolver.resolve(field_name, query_type, &binding));
781+
}
777782

778783
if (!binding.is_bound()) {
779784
LOG(INFO) << "search: No inverted index for field '" << field_name
@@ -905,9 +910,12 @@ Status FunctionSearch::build_leaf_query(const TSearchClause& clause,
905910
? normalize_wildcard_pattern(value, binding.index_properties)
906911
: value;
907912
Field query_value = Field::create_field<TYPE_STRING>(pattern);
908-
RETURN_IF_ERROR(binding.inverted_reader->query(reader_context,
909-
binding.stored_field_name, query_value,
910-
snii_query_type, data_bitmap, nullptr));
913+
const bool raw_pattern_query =
914+
snii_query_type == InvertedIndexQueryType::WILDCARD_QUERY ||
915+
snii_query_type == InvertedIndexQueryType::MATCH_REGEXP_QUERY;
916+
RETURN_IF_ERROR(binding.inverted_reader->query(
917+
reader_context, binding.stored_field_name, query_value, snii_query_type,
918+
data_bitmap, raw_pattern_query ? nullptr : binding.analyzer_context.get()));
911919
// Reply-direction fields land on the copy the reader was given, so they have to be
912920
// folded back. Today this is unreachable rather than load-bearing: the count-only
913921
// fast path requires the scan to have no score runtime, while the similarity that

‎be/src/exprs/function/variant_inverted_index_search.cpp‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,29 @@ void add_search_binding_diagnostic(const std::shared_ptr<IndexQueryContext>& con
6565
}
6666
}
6767

68+
InvertedIndexAnalyzerCtxSPtr build_analyzer_context(
69+
const std::map<std::string, std::string>& properties, const std::string& analyzer_key) {
70+
InvertedIndexAnalyzerConfig config;
71+
config.analyzer_name = get_analyzer_name_from_properties(properties);
72+
config.parser_type = get_inverted_index_parser_type_from_string(
73+
get_parser_string_from_properties(properties));
74+
config.parser_mode = get_parser_mode_string_from_properties(properties);
75+
config.lower_case = get_parser_lowercase_from_properties(properties);
76+
config.stop_words = get_parser_stopwords_from_properties(properties);
77+
config.char_filter_map = get_parser_char_filter_map_from_properties(properties);
78+
79+
auto analyzer_context = std::make_shared<InvertedIndexAnalyzerCtx>();
80+
analyzer_context->analyzer_key = analyzer_key;
81+
analyzer_context->analyzer_name = config.analyzer_name;
82+
analyzer_context->parser_type = config.parser_type;
83+
analyzer_context->char_filter_map = config.char_filter_map;
84+
if (analyzer_context->requires_analysis()) {
85+
analyzer_context->analyzer_provider =
86+
inverted_index::InvertedIndexAnalyzer::create_analyzer_provider(&config);
87+
}
88+
return analyzer_context;
89+
}
90+
6891
} // namespace
6992

7093
FieldReaderResolver::FieldReaderResolver(
@@ -384,6 +407,20 @@ Status FieldReaderResolver::resolve(const std::string& field_name,
384407
return Status::OK();
385408
}
386409

410+
Status FieldReaderResolver::resolve_with_analyzer_context(const std::string& field_name,
411+
InvertedIndexQueryType query_type,
412+
FieldReaderBinding* binding) {
413+
RETURN_IF_ERROR(resolve(field_name, query_type, binding));
414+
if (!binding->use_snii_native_reader() || binding->analyzer_context != nullptr) {
415+
return Status::OK();
416+
}
417+
418+
binding->analyzer_context =
419+
build_analyzer_context(binding->index_properties, binding->analyzer_key);
420+
_cache.at(binding->binding_key).analyzer_context = binding->analyzer_context;
421+
return Status::OK();
422+
}
423+
387424
segment_v2::IndexIterator* VariantSearchNullBitmapAdapter::iterator_for(
388425
const query_v2::Scorer& /*scorer*/, const std::string& logical_field) const {
389426
if (logical_field.empty()) {

‎be/src/exprs/function/variant_inverted_index_search.h‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -86,6 +86,7 @@ struct FieldReaderBinding {
8686
std::map<std::string, std::string> index_properties;
8787
std::string binding_key;
8888
std::string analyzer_key;
89+
InvertedIndexAnalyzerCtxSPtr analyzer_context;
8990
SearchFieldBindingState state = SearchFieldBindingState::MISSING_IN_SEGMENT;
9091
SearchFieldExecutionMode execution_mode = SearchFieldExecutionMode::UNBOUND;
9192

@@ -112,6 +113,10 @@ class FieldReaderResolver {
112113
Status resolve(const std::string& field_name, InvertedIndexQueryType query_type,
113114
FieldReaderBinding* binding);
114115

116+
Status resolve_with_analyzer_context(const std::string& field_name,
117+
InvertedIndexQueryType query_type,
118+
FieldReaderBinding* binding);
119+
115120
bool is_variant_subcolumn(const std::string& field_name) const {
116121
return _variant_subcolumn_fields.count(field_name) > 0;
117122
}

‎be/src/storage/index/snii/snii_index_reader.cpp‎

Lines changed: 11 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -642,6 +642,12 @@ Status SniiIndexReader::_query(const IndexQueryContextPtr& context, const std::s
642642
const bool common_grams_query_eligible = common_grams_phrase_shape && !actual_similarity;
643643
const bool raw_pattern_query = query_type == InvertedIndexQueryType::MATCH_REGEXP_QUERY ||
644644
query_type == InvertedIndexQueryType::WILDCARD_QUERY;
645+
// A physical keyword-lane index has no analyzer contract for the open below to validate:
646+
// SniiIndexColumnWriter::init() refuses a CommonGrams metadata seed whenever should_analyzer()
647+
// is false, so such a segment can never carry gram terms. Key this on the writer-side
648+
// predicate itself rather than on the reader type it happens to select.
649+
const bool keyword_lane_query =
650+
!inverted_index::InvertedIndexAnalyzer::should_analyzer(_index_meta.properties());
645651
// Lucene-style CommonGrams: the plan decision is local to the segment and query. Snapshot the
646652
// switch once so this query's plan and cache identity use the same mode.
647653
const bool common_grams_query_plan_enabled = config::enable_common_grams_query_plan;
@@ -656,11 +662,11 @@ Status SniiIndexReader::_query(const IndexQueryContextPtr& context, const std::s
656662
ctx->has_complete_common_grams_identity();
657663
};
658664
const bool safety_requires_plain = !common_grams_query_plan_enabled;
659-
// The raw cache key cannot prove whether the immutable segment analyzer has CommonGrams until
660-
// its metadata is open. Delay every eligible forced-plain lookup, then restore ordinary cache
661-
// access below only for a segment that cannot contain gram terms.
662-
const bool initial_force_plain = common_grams_query_eligible && safety_requires_plain;
663-
const bool initial_allow_result_cache = !actual_similarity && !initial_force_plain;
665+
// An analyzed raw query can only share a cached result after the immutable segment analyzer
666+
// contract has been validated below. Patterns and the keyword lane are analyzer-independent
667+
// and can still use the cache before opening the logical reader.
668+
const bool initial_allow_result_cache =
669+
!actual_similarity && (raw_pattern_query || keyword_lane_query);
664670
const bool defer_result_cache_lookup = !actual_similarity && !initial_allow_result_cache;
665671
const InvertedIndexRawQuerySemantic raw_semantic {
666672
.raw_query_bytes = search_str,

‎be/test/exprs/function/function_search_test.cpp‎

Lines changed: 70 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2346,7 +2346,8 @@ TEST_F(FunctionSearchTest, TestBuildLeafQueryExecutesSelectedSniiWildcardReader)
23462346
{INVERTED_INDEX_PARSER_KEY, INVERTED_INDEX_PARSER_ENGLISH},
23472347
{INVERTED_INDEX_PARSER_LOWERCASE_KEY, INVERTED_INDEX_PARSER_TRUE}};
23482348
std::map<std::string, std::string> selected_properties {
2349-
{INVERTED_INDEX_PARSER_KEY, INVERTED_INDEX_PARSER_STANDARD},
2349+
{INVERTED_INDEX_ANALYZER_NAME_KEY, "unregistered_search_wildcard_analyzer"},
2350+
{INVERTED_INDEX_PARSER_KEY, INVERTED_INDEX_PARSER_NONE},
23502351
{INVERTED_INDEX_PARSER_LOWERCASE_KEY, INVERTED_INDEX_PARSER_TRUE}};
23512352
auto decoy_meta = make_test_inverted_index(15, decoy_properties);
23522353
auto selected_meta = make_test_inverted_index(16, selected_properties);
@@ -2530,6 +2531,74 @@ TEST_F(FunctionSearchTest, TestSniiNativeForwardsTermClauseAsEqualQuery) {
25302531
expect_bitmap_eq(collect_docs(scorer), {0, 2});
25312532
}
25322533

2534+
TEST_F(FunctionSearchTest, TestSniiNativePassesSelectedAnalyzerContext) {
2535+
auto* exec_env = ExecEnv::GetInstance();
2536+
auto* previous_policy_mgr = exec_env->index_policy_mgr();
2537+
IndexPolicyMgr scoped_policy_mgr;
2538+
exec_env->_index_policy_mgr = &scoped_policy_mgr;
2539+
DEFER(exec_env->_index_policy_mgr = previous_policy_mgr);
2540+
2541+
TIndexPolicy tokenizer;
2542+
tokenizer.id = 910030;
2543+
tokenizer.name = "function_search_context_tokenizer";
2544+
tokenizer.type = TIndexPolicyType::TOKENIZER;
2545+
tokenizer.properties["type"] = "char_group";
2546+
tokenizer.properties["tokenize_on_chars"] = "[whitespace]";
2547+
2548+
TIndexPolicy common_grams;
2549+
common_grams.id = 910031;
2550+
common_grams.name = "function_search_context_common_grams";
2551+
common_grams.type = TIndexPolicyType::TOKEN_FILTER;
2552+
common_grams.properties["type"] = "common_grams";
2553+
2554+
TIndexPolicy analyzer;
2555+
analyzer.id = 910032;
2556+
analyzer.name = "function_search_context_analyzer";
2557+
analyzer.type = TIndexPolicyType::ANALYZER;
2558+
analyzer.properties["tokenizer"] = tokenizer.name;
2559+
analyzer.properties["token_filter"] = "lowercase," + common_grams.name;
2560+
scoped_policy_mgr.apply_policy_changes({tokenizer, common_grams, analyzer}, {});
2561+
2562+
auto context = std::make_shared<IndexQueryContext>();
2563+
std::map<std::string, std::string> properties {
2564+
{INVERTED_INDEX_ANALYZER_NAME_KEY, analyzer.name},
2565+
{INVERTED_INDEX_PARSER_KEY, INVERTED_INDEX_PARSER_NONE}};
2566+
auto index_meta = make_test_inverted_index(45, properties);
2567+
auto index_file_reader = std::make_shared<RejectingCluceneIndexFileReader>();
2568+
auto reader =
2569+
std::make_shared<RecordingNativeInvertedIndexReader>(&index_meta, index_file_reader);
2570+
reader->set_query_result("running quickly", make_bitmap({1}));
2571+
segment_v2::InvertedIndexIterator iterator;
2572+
iterator.add_reader(segment_v2::InvertedIndexReaderType::FULLTEXT, reader);
2573+
2574+
std::unordered_map<std::string, IndexFieldNameAndTypePair> data_type_with_names;
2575+
data_type_with_names.emplace(
2576+
"body", IndexFieldNameAndTypePair {"body", std::make_shared<DataTypeString>()});
2577+
std::unordered_map<std::string, IndexIterator*> iterators;
2578+
iterators["body"] = &iterator;
2579+
TSearchFieldBinding field_binding;
2580+
field_binding.field_name = "body";
2581+
field_binding.index_properties = properties;
2582+
field_binding.__isset.index_properties = true;
2583+
FieldReaderResolver resolver(data_type_with_names, iterators, context, {field_binding});
2584+
2585+
inverted_index::query_v2::QueryPtr query;
2586+
std::string binding_key;
2587+
auto status =
2588+
function_search->build_leaf_query(make_leaf_clause("MATCH", "running quickly"), context,
2589+
resolver, &query, &binding_key, "OR", 0, 3);
2590+
2591+
ASSERT_TRUE(status.ok()) << status;
2592+
ASSERT_NE(reader->last_analyzer_ctx, nullptr);
2593+
EXPECT_EQ(reader->last_analyzer_ctx->analyzer_key, analyzer.name);
2594+
EXPECT_EQ(reader->last_analyzer_ctx->analyzer_name, analyzer.name);
2595+
EXPECT_EQ(reader->last_analyzer_ctx->parser_type, InvertedIndexParserType::PARSER_NONE);
2596+
EXPECT_TRUE(reader->last_analyzer_ctx->requires_analysis());
2597+
ASSERT_NE(reader->last_analyzer_ctx->analyzer_provider, nullptr);
2598+
EXPECT_TRUE(reader->last_analyzer_ctx->analyzer_provider->uses_common_grams());
2599+
EXPECT_FALSE(reader->last_analyzer_ctx->analyzer_provider->base_analyzer_fingerprint().empty());
2600+
}
2601+
25332602
// default_operator "and" maps a multi-token TERM clause onto MATCH_ALL_QUERY instead of the
25342603
// default EQUAL_QUERY (which is an OR of terms) -- SNII has no boolean query tree to build, so
25352604
// this is expressed entirely as which query type gets forwarded to the reader.

‎be/test/storage/index/inverted/inverted_index_reader_analysis_purpose_test.cpp‎

Lines changed: 10 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -407,10 +407,10 @@ class InvertedIndexReaderAnalysisPurposeTest : public testing::Test {
407407
}
408408

409409
template <typename Reader, typename Provider>
410-
void expect_raw_cache_hit_before_analysis(const std::shared_ptr<Reader>& reader,
411-
const std::shared_ptr<IndexFileReader>& file_reader,
412-
const std::shared_ptr<Provider>& provider,
413-
bool common_grams_query_plan_enabled) {
410+
void expect_raw_cache_hit_after_segment_admission(
411+
const std::shared_ptr<Reader>& reader,
412+
const std::shared_ptr<IndexFileReader>& file_reader,
413+
const std::shared_ptr<Provider>& provider, bool common_grams_query_plan_enabled) {
414414
QueryExecutionContext execution(/*scoring=*/false);
415415
InvertedIndexAnalyzerCtx analyzer_ctx;
416416
analyzer_ctx.parser_type = InvertedIndexParserType::PARSER_ENGLISH;
@@ -447,7 +447,7 @@ class InvertedIndexReaderAnalysisPurposeTest : public testing::Test {
447447
EXPECT_EQ(execution.stats.inverted_index_query_cache_miss, 0);
448448
EXPECT_EQ(execution.stats.inverted_index_query_cache_lookup, 1);
449449
EXPECT_EQ(execution.stats.inverted_index_query_cache_insert, 0);
450-
EXPECT_EQ(execution.stats.inverted_index_searcher_cache_hit, 0);
450+
EXPECT_EQ(execution.stats.inverted_index_searcher_cache_hit, 1);
451451
EXPECT_EQ(execution.stats.inverted_index_searcher_cache_miss, 0);
452452
}
453453

@@ -562,6 +562,7 @@ TEST(InvertedIndexAnalyzerCtxTest, UsesProviderCommonGramsIdentityWithoutCopying
562562

563563
TEST_F(InvertedIndexReaderAnalysisPurposeTest,
564564
SniiRawCacheLookupIsIndependentOfRequestAnalyzerIdentity) {
565+
preload_legacy_searcher_cache_entries();
565566
const bool original = config::enable_common_grams_query_plan;
566567
Defer restore([original] {
567568
EXPECT_TRUE(config::set_config("enable_common_grams_query_plan",
@@ -577,14 +578,14 @@ TEST_F(InvertedIndexReaderAnalysisPurposeTest,
577578
.common_grams_fingerprint = "grams:complete"};
578579
const inverted_index::CommonGramsQueryIdentity empty_identity;
579580
for (const auto& identity : {complete_identity, empty_identity}) {
580-
expect_raw_cache_hit_before_analysis(
581+
expect_raw_cache_hit_after_segment_admission(
581582
_snii_reader, _snii_file_reader,
582583
std::make_shared<IdentityFailingAnalyzerProvider>(identity),
583584
config::enable_common_grams_query_plan);
584585
}
585-
expect_raw_cache_hit_before_analysis(_snii_reader, _snii_file_reader,
586-
std::make_shared<RecordingFailingAnalyzerProvider>(),
587-
config::enable_common_grams_query_plan);
586+
expect_raw_cache_hit_after_segment_admission(
587+
_snii_reader, _snii_file_reader, std::make_shared<RecordingFailingAnalyzerProvider>(),
588+
config::enable_common_grams_query_plan);
588589
}
589590

590591
TEST_F(InvertedIndexReaderAnalysisPurposeTest, DisabledResultCacheDoesNotLookupCountOrInsert) {

0 commit comments

Comments
 (0)