@@ -114,41 +114,33 @@ static std::string extract_segment_prefix(
114114
115115namespace {
116116
117- // A SEARCH result is cached as two entries: its TRUE rows, and its UNKNOWN rows under the
118- // signature with a "__null" suffix.
117+ // Keep two-valued SEARCH results separate from older SQL UNKNOWN cache entries.
119118InvertedIndexQueryCache::CacheKey dsl_cache_key (const std::string& seg_prefix,
120119 const std::string& signature) {
121- return {seg_prefix, " __search_dsl__ " , InvertedIndexQueryType::SEARCH_DSL_QUERY , signature};
120+ return {seg_prefix, " __search_dsl_2vl__ " , InvertedIndexQueryType::SEARCH_DSL_QUERY , signature};
122121}
123122
124- // Either entry can be evicted, or not yet inserted, without the other, and TRUE rows found
125- // without their UNKNOWN rows are no result.
126123bool lookup_dsl_result (InvertedIndexQueryCache* cache, const std::string& seg_prefix,
127124 const std::string& signature, InvertedIndexResultBitmap* result) {
128125 InvertedIndexQueryCacheHandle rows_handle;
129- InvertedIndexQueryCacheHandle nulls_handle;
130- if (!cache->lookup (dsl_cache_key (seg_prefix, signature), &rows_handle) ||
131- !cache->lookup (dsl_cache_key (seg_prefix, signature + " __null" ), &nulls_handle)) {
126+ if (!cache->lookup (dsl_cache_key (seg_prefix, signature), &rows_handle)) {
132127 return false ;
133128 }
134129 auto rows = rows_handle.get_bitmap ();
135- auto nulls = nulls_handle.get_bitmap ();
136- if (rows == nullptr || nulls == nullptr ) {
130+ if (rows == nullptr ) {
137131 return false ;
138132 }
139- *result = InvertedIndexResultBitmap (std::move (rows), std::move (nulls));
133+ // SQL bitmap composition can mutate its input. The cache owns an independent copy.
134+ *result = InvertedIndexResultBitmap (std::make_shared<roaring::Roaring>(*rows),
135+ std::make_shared<roaring::Roaring>());
140136 return true ;
141137}
142138
143- // The UNKNOWN rows go in first, so a search that finds the TRUE rows finds them as well.
144139void insert_dsl_result (InvertedIndexQueryCache* cache, const std::string& seg_prefix,
145140 const std::string& signature, const InvertedIndexResultBitmap& result) {
146- auto nulls = result.get_null_bitmap () != nullptr ? result.get_null_bitmap ()
147- : std::make_shared<roaring::Roaring>();
148- InvertedIndexQueryCacheHandle nulls_handle;
149- cache->insert (dsl_cache_key (seg_prefix, signature + " __null" ), std::move (nulls), &nulls_handle);
150141 InvertedIndexQueryCacheHandle rows_handle;
151- cache->insert (dsl_cache_key (seg_prefix, signature), result.get_data_bitmap (), &rows_handle);
142+ cache->insert (dsl_cache_key (seg_prefix, signature),
143+ std::make_shared<roaring::Roaring>(*result.get_data_bitmap ()), &rows_handle);
152144}
153145
154146bool is_nested_group_search_supported () {
@@ -172,7 +164,7 @@ class SearchFieldCatalog final : public logical::FieldCatalog {
172164 if (!binding.is_bound ()) {
173165 LOG (INFO ) << " search: No inverted index for field '" << field
174166 << " ' in this segment, query_type=" << static_cast <int >(query_type)
175- << " , returning UNKNOWN bitmap" ;
167+ << " , returning non-matching bitmap" ;
176168 *out = logical::FieldProps {};
177169 return Status::OK ();
178170 }
@@ -244,28 +236,23 @@ Status compile_node(const logical::Node& node, const SearchLeafContext& ctx,
244236 FieldReaderResolver& resolver, query_v2::QueryPtr* out,
245237 std::string* binding_key);
246238
247- // Marks the rows where the field is NULL as UNKNOWN for leaves that would misreport them: an array
248- // index can list a NULL array's leftover values, and a term threshold counts its terms two-valued.
249- Status mask_null_rows (const FieldReaderBinding& binding, const FieldReaderResolver& resolver,
250- const std::string& field, bool threshold, query_v2::QueryPtr* query) {
251- const bool array = binding.column_type != nullptr &&
252- remove_nullable (binding.column_type )->get_primitive_type () == TYPE_ARRAY ;
253- if (!array && !threshold) {
254- return Status::OK ();
255- }
239+ // Physical NULL postings can overlap a NULL ARRAY's leftover values. Exclude them
240+ // before Boolean composition without introducing SQL UNKNOWN or changing scores.
241+ Status mask_null_rows (const FieldReaderResolver& resolver, const std::string& field,
242+ query_v2::QueryPtr* query) {
256243 IndexIterator* iterator = resolver.get_iterator (field);
257- if (iterator == nullptr ) {
258- return Status::OK ();
259- }
260- auto has_null = iterator->has_null ();
261- if (!has_null.has_value () || !has_null.value ()) {
244+ DORIS_CHECK (iterator != nullptr );
245+ const bool has_null = DORIS_TRY (iterator->has_null ());
246+ if (!has_null) {
262247 return Status::OK ();
263248 }
264249 InvertedIndexQueryCacheHandle handle;
265250 RETURN_IF_ERROR (iterator->read_null_bitmap (&handle));
266251 auto nulls = handle.get_bitmap ();
267- if (nulls != nullptr && !nulls->isEmpty ()) {
268- *query = std::make_shared<query_v2::NullMaskedQuery>(std::move (*query), std::move (nulls));
252+ DORIS_CHECK (nulls != nullptr );
253+ if (!nulls->isEmpty ()) {
254+ *query = std::make_shared<query_v2::NullMaskedQuery>(std::move (*query), std::move (nulls),
255+ /* nulls_are_false=*/ true );
269256 }
270257 return Status::OK ();
271258}
@@ -277,8 +264,7 @@ bool has_required_clause(const Clauses& clauses) {
277264 clauses, [](const auto & clause) { return clause.first == logical::Occur::kMust ; });
278265}
279266
280- // Without scoring, optional clauses beside a required one change neither the rows nor their
281- // UNKNOWN state, so they are left out.
267+ // Without scoring, optional clauses beside a required one do not change the match set.
282268Clauses matching_clauses (const logical::Bool& boolean, bool scoring) {
283269 if (scoring || boolean.op != logical::BoolOp::kOccur || boolean.min_should_match != 0 ||
284270 !has_required_clause (boolean.clauses )) {
@@ -367,23 +353,30 @@ Status compile_node(const logical::Node& node, const SearchLeafContext& ctx,
367353 if (binding_key != nullptr ) {
368354 *binding_key = field->binding ;
369355 }
370- if (node.as <logical::Unknown >() != nullptr ) {
371- *out = make_unknown_leaf_query (ctx. num_rows );
356+ if (node.as <logical::Empty >() != nullptr ) {
357+ *out = std::make_shared<query_v2::BitSetQuery>(std::make_shared<roaring::Roaring>() );
372358 return resolver.map_leaf_query (field->name , out);
373359 }
374360 const FieldReaderBinding* binding = resolver.find_binding (field->binding );
375361 if (binding == nullptr || binding->leaf_compiler == nullptr ) {
376362 return Status::InternalError (" search: field '{}' has no compiler for binding '{}'" ,
377363 field->name , field->binding );
378364 }
365+ SearchLeafContext leaf_ctx = ctx;
366+ RETURN_IF_ERROR (resolver.leaf_num_rows (field->name , &leaf_ctx.num_rows ));
379367 const auto * set = node.as <logical::TermSet>();
380368 const bool threshold = set != nullptr && set->min_should_match > 0 ;
381- if (threshold) {
382- RETURN_IF_ERROR (compile_term_threshold (*set, ctx, *binding->leaf_compiler , out));
369+ if (node.as <logical::Exists>() != nullptr ) {
370+ // Exists also works for BKD and uses the NestedGroup leaf's element domain.
371+ auto matches = std::make_shared<roaring::Roaring>();
372+ matches->addRange (0 , leaf_ctx.num_rows );
373+ *out = std::make_shared<query_v2::BitSetQuery>(std::move (matches));
374+ } else if (threshold) {
375+ RETURN_IF_ERROR (compile_term_threshold (*set, leaf_ctx, *binding->leaf_compiler , out));
383376 } else {
384- RETURN_IF_ERROR (binding->leaf_compiler ->compile (node, ctx , out));
377+ RETURN_IF_ERROR (binding->leaf_compiler ->compile (node, leaf_ctx , out));
385378 }
386- RETURN_IF_ERROR (mask_null_rows (*binding, resolver, field->name , threshold , out));
379+ RETURN_IF_ERROR (mask_null_rows (resolver, field->name , out));
387380 return resolver.map_leaf_query (field->name , out);
388381}
389382
@@ -453,22 +446,14 @@ Status FunctionSearch::_evaluate_search_param(
453446 " NESTED query requires NestedGroup support, which is unavailable in this build" );
454447 }
455448
456- if (!is_nested_query && (iterators.empty () || data_type_with_names.empty ())) {
457- LOG (INFO ) << " No indexed columns or iterators available, returning empty result, dsl:"
458- << search_param.original_dsl ;
459- bitmap_result = InvertedIndexResultBitmap (std::make_shared<roaring::Roaring>(),
460- std::make_shared<roaring::Roaring>());
461- return Status::OK ();
462- }
463-
464449 // Track overall query time (equivalent to inverted_index_query_timer in MATCH path).
465450 // Must be declared before the DSL cache lookup so that cache-hit fast paths are
466451 // also covered by the timer.
467452 int64_t query_timer_dummy = 0 ;
468453 OlapReaderStatistics* outer_stats = index_query_context ? index_query_context->stats : nullptr ;
469454 SCOPED_RAW_TIMER (outer_stats ? &outer_stats->inverted_index_query_timer : &query_timer_dummy);
470455
471- // DSL result cache only stores bitmap/null bitmap. It does not store BM25 scores,
456+ // DSL result cache only stores the two-valued match bitmap. It does not store BM25 scores,
472457 // so score() queries must execute scorers to populate CollectionSimilarity.
473458 const bool enable_scoring =
474459 index_query_context != nullptr && index_query_context->collection_similarity != nullptr ;
@@ -560,7 +545,7 @@ Status FunctionSearch::_evaluate_search_param(
560545 return Status::OK ();
561546 }
562547
563- VariantSearchNullBitmapAdapter null_resolver (resolver) ;
548+ SearchNullBitmapResolver null_resolver;
564549 query_v2::QueryExecutionContext exec_ctx =
565550 build_variant_search_query_execution_context (num_rows, resolver, &null_resolver);
566551
@@ -580,13 +565,17 @@ Status FunctionSearch::_evaluate_search_param(
580565 }
581566
582567 std::shared_ptr<roaring::Roaring> roaring = std::make_shared<roaring::Roaring>();
583- // A three-valued root reports the rows it leaves UNKNOWN (TRUE OR NULL is TRUE, FALSE OR
584- // NULL is NULL); a lucene-style Boolean root is two-valued: every row it does not match is
585- // FALSE.
586- std::shared_ptr<roaring::Roaring> null_bitmap = std::make_shared<roaring::Roaring>();
587- const bool three_valued =
588- exec_ctx.null_resolver != nullptr && search_param.root .clause_type != " OCCUR_BOOLEAN" ;
589- const bool top_k_path = enable_scoring && !is_asc && top_k > 0 ;
568+ // Keyword readers can lack BM25 length statistics. Preserve their original
569+ // scores and all matches for the SQL sort when pruning cannot order them.
570+ const bool can_prune_scores = std::none_of (
571+ resolver.binding_cache ().begin (), resolver.binding_cache ().end (),
572+ [](const auto & entry) {
573+ const auto & binding = entry.second ;
574+ DORIS_CHECK (binding.inverted_reader != nullptr );
575+ return binding.inverted_reader ->type () == InvertedIndexReaderType::STRING_TYPE ;
576+ });
577+ const bool top_k_path = enable_scoring && !is_asc && top_k > 0 && top_k < num_rows &&
578+ top_k <= query_v2::TopKCollector::kMaxK && can_prune_scores;
590579 exec_ctx.top_k = top_k_path;
591580 {
592581 int64_t exec_dummy = 0 ;
@@ -595,33 +584,24 @@ Status FunctionSearch::_evaluate_search_param(
595584 bool use_wand = index_query_context->runtime_state != nullptr &&
596585 index_query_context->runtime_state ->query_options ()
597586 .enable_inverted_index_wand_query ;
598- query_v2::collect_multi_segment_top_k (
599- weight, exec_ctx, root_binding_key, top_k, roaring,
600- index_query_context->collection_similarity , use_wand ,
601- index_query_context-> delete_bitmap , three_valued ? null_bitmap. get () : nullptr );
587+ query_v2::collect_multi_segment_top_k (weight, exec_ctx, root_binding_key, top_k,
588+ roaring,
589+ index_query_context->collection_similarity ,
590+ use_wand, index_query_context-> delete_bitmap );
602591 } else {
603- // The scorers listing the rows report the UNKNOWN ones as they go.
604592 query_v2::collect_multi_segment_doc_set (
605593 weight, exec_ctx, root_binding_key, roaring,
606594 index_query_context ? index_query_context->collection_similarity : nullptr ,
607- enable_scoring, three_valued ? null_bitmap. get () : nullptr );
595+ enable_scoring);
608596 }
609597 }
610598
611599 VLOG_DEBUG << " search: Query completed, matched " << roaring->cardinality () << " documents" ;
612600
613- VLOG_TRACE << " search: Before mask - true_bitmap=" << roaring->cardinality ()
614- << " , null_bitmap=" << null_bitmap->cardinality ();
615-
616- // Create result and mask out NULLs (SQL WHERE clause semantics: only TRUE rows)
617- bitmap_result = InvertedIndexResultBitmap (std::move (roaring), std::move (null_bitmap));
618- bitmap_result.mask_out_null ();
619-
620- VLOG_TRACE << " search: After mask - result_bitmap="
621- << bitmap_result.get_data_bitmap ()->cardinality ();
601+ // SEARCH never returns UNKNOWN; outer SQL composes this non-nullable match set.
602+ bitmap_result =
603+ InvertedIndexResultBitmap (std::move (roaring), std::make_shared<roaring::Roaring>());
622604
623- // The cached result keeps its UNKNOWN rows, so NOT, OR and AND over a cache hit stay
624- // three-valued.
625605 if (dsl_cache && cache_usable) {
626606 insert_dsl_result (dsl_cache, seg_prefix, dsl_sig, bitmap_result);
627607 }
0 commit comments