Skip to content

Commit f48b3e8

Browse files
committed
[fix](be) Make index_tool MATCH_ALL run and fail when it cannot read postings
### What problem does this PR solve? Issue Number: None Related PR: #68758 Problem Summary: `index_tool --operation=term_query --pred_type=match_all` had two problems. The query builds a RuntimeState, whose constructor registers memory trackers, but the tool never initialized them, so the command crashed before reading the index; master has the same code. Past that, MATCH_ALL runs its terms through run_clucene_leaf, which returns a Status instead of throwing. When the postings could not be read, search() printed the error, then the count of the incomplete result, and the tool exited 0. term_query now initializes the memory trackers, as meta_tool does. search() returns the query status and prints no count when the query fails, and term_query prints the error and exits with -1, as it does when CLucene throws. The MATCH_ALL setup moves into match_all(), which keeps search() within the function size limit. ### Release note None ### Check List (For Author) - Test: Manual test. On the V1 fixture x86_new_1744025019684.idx, MATCH_ALL of `gif` on `request` crashed before the change and now reports 37337 rows, the same as MATCH. On a copy whose _0.frq length is set to zero, so the dictionary opens and the postings read fails, the tool with only the tracker change printed `Term queried count:0` and exited 0; with the whole change it prints the read error and exits 255. - Behavior changed: No - Does this need documentation: No
1 parent 802d64a commit f48b3e8

1 file changed

Lines changed: 49 additions & 28 deletions

File tree

‎be/src/tools/index_tool.cpp‎

Lines changed: 49 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@
4545
#pragma clang diagnostic pop
4646
#endif
4747
#include "io/fs/local_file_system.h"
48+
#include "runtime/exec_env.h"
4849
#include "runtime/runtime_state.h"
4950
#include "runtime/thread_context.h"
5051
#include "storage/index/index_compound_reader.h"
@@ -124,8 +125,31 @@ std::vector<std::string> split(const std::string& s, char delimiter) {
124125
return tokens;
125126
}
126127

127-
void search(lucene::store::Directory* dir, std::string& field, std::string& token,
128-
std::string& pred) {
128+
// Finds the rows of `field` that hold every term of `token`, whose terms are separated by '|'.
129+
doris::Status match_all(const FulltextIndexSearcherPtr& searcher, const std::wstring& field,
130+
const std::string& token, const std::shared_ptr<roaring::Roaring>& result) {
131+
doris::OlapReaderStatistics stats;
132+
doris::RuntimeState runtime_state;
133+
doris::TQueryOptions query_options;
134+
query_options.inverted_index_max_expansions = 50;
135+
runtime_state.set_query_options(query_options);
136+
doris::io::IOContext io_ctx;
137+
138+
IndexQueryContextPtr context = std::make_shared<IndexQueryContext>();
139+
context->runtime_state = &runtime_state;
140+
context->stats = &stats;
141+
context->io_ctx = &io_ctx;
142+
143+
const doris::index_query::logical::Node leaf {
144+
.value = doris::index_query::logical::TermSet {.field = {},
145+
.terms = split(token, '|'),
146+
.require_all = true,
147+
.min_should_match = 0}};
148+
return run_clucene_leaf(context, field, leaf, nullptr, /*scoring=*/false, searcher, result);
149+
}
150+
151+
doris::Status search(lucene::store::Directory* dir, std::string& field, std::string& token,
152+
std::string& pred) {
129153
lucene::index::IndexReader* reader = lucene::index::IndexReader::open(dir);
130154

131155
lucene::index::IndexReader* newreader = reader->reopen();
@@ -172,31 +196,10 @@ void search(lucene::store::Directory* dir, std::string& field, std::string& toke
172196
}
173197

174198
int32_t total = 0;
199+
doris::Status status;
175200
if (pred == "match_all") {
176-
doris::OlapReaderStatistics stats;
177-
doris::RuntimeState runtime_state;
178-
doris::TQueryOptions query_options;
179-
query_options.inverted_index_max_expansions = 50;
180-
runtime_state.set_query_options(query_options);
181-
doris::io::IOContext io_ctx;
182-
183-
IndexQueryContextPtr context = std::make_shared<IndexQueryContext>();
184-
context->runtime_state = &runtime_state;
185-
context->stats = &stats;
186-
context->io_ctx = &io_ctx;
187-
188-
const doris::index_query::logical::Node leaf {
189-
.value = doris::index_query::logical::TermSet {.field = {},
190-
.terms = split(token, '|'),
191-
.require_all = true,
192-
.min_should_match = 0}};
193201
auto result = std::make_shared<roaring::Roaring>();
194-
const doris::Status status = run_clucene_leaf(context, field_ws, leaf, nullptr,
195-
/*scoring=*/false, s, result);
196-
if (!status.ok()) {
197-
std::cerr << "match_all failed: " << status << std::endl;
198-
}
199-
202+
status = match_all(s, field_ws, token, result);
200203
total += result->cardinality();
201204
} else {
202205
roaring::Roaring result;
@@ -209,11 +212,15 @@ void search(lucene::store::Directory* dir, std::string& field, std::string& toke
209212
});
210213
total += result.cardinality();
211214
}
212-
std::cout << "Term queried count:" << total << std::endl;
215+
// A failed query may have an incomplete result, so it prints no count.
216+
if (status.ok()) {
217+
std::cout << "Term queried count:" << total << std::endl;
218+
}
213219

214220
s->close();
215221
reader->close();
216222
_CLLDELETE(reader);
223+
return status;
217224
}
218225

219226
void check_terms_stats(lucene::store::Directory* dir) {
@@ -347,6 +354,8 @@ int main(int argc, char** argv) {
347354
std::cerr << "invalid params for term_query " << std::endl;
348355
return -1;
349356
}
357+
// MATCH_ALL builds a RuntimeState, which registers memory trackers.
358+
doris::ExecEnv::GetInstance()->init_mem_tracker();
350359
auto fs = doris::io::global_local_filesystem();
351360
std::unique_ptr<DorisCompoundReader> reader;
352361
try {
@@ -376,8 +385,14 @@ int main(int argc, char** argv) {
376385
std::cout << "Search " << FLAGS_column_name << ":" << FLAGS_term << " from "
377386
<< file_str << std::endl;
378387
std::cout << "==================================" << std::endl;
379-
search(reader.get(), FLAGS_column_name, FLAGS_term, FLAGS_pred_type);
388+
const doris::Status st = search(reader.get(), FLAGS_column_name, FLAGS_term,
389+
FLAGS_pred_type);
380390
reader->close();
391+
if (!st.ok()) {
392+
std::cerr << "error occurred when search file: " << f.file_name
393+
<< ", error:" << st << std::endl;
394+
return -1;
395+
}
381396
} catch (CLuceneError& err) {
382397
std::cerr << "error occurred when search file: " << f.file_name
383398
<< ", error:" << err.what() << std::endl;
@@ -399,8 +414,14 @@ int main(int argc, char** argv) {
399414
<< FLAGS_idx_file_name << std::endl;
400415
std::cout << "==================================" << std::endl;
401416
try {
402-
search(reader.get(), FLAGS_column_name, FLAGS_term, FLAGS_pred_type);
417+
const doris::Status st =
418+
search(reader.get(), FLAGS_column_name, FLAGS_term, FLAGS_pred_type);
403419
reader->close();
420+
if (!st.ok()) {
421+
std::cerr << "error occurred when search file: " << FLAGS_idx_file_name
422+
<< ", error:" << st << std::endl;
423+
return -1;
424+
}
404425
} catch (CLuceneError& err) {
405426
std::cerr << "error occurred when search file: " << FLAGS_idx_file_name
406427
<< ", error:" << err.what() << std::endl;

0 commit comments

Comments
 (0)