Skip to content

Commit e9a14d7

Browse files
authored
[opt](index compaction)Add dual write inverted index file switch (#42280)
## Proposed changes To check correctness of index file produced by index compaction process, we add a switch `dual_write_inverted_index_enable` (default is `false`). When both `inverted_index_compaction_enable` and `dual_write_inverted_index_enable` are `true`, Doris will produce index file through both normal compaction process and index compaction process, and compares both index files, which are theoretically identical. Doris will log FATAL and crash after the check failed. We add this feature only for test, *DO NOT* use it in production.
1 parent 89286af commit e9a14d7

6 files changed

Lines changed: 163 additions & 3 deletions

File tree

‎be/src/common/config.cpp‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1042,6 +1042,8 @@ DEFINE_Int32(max_depth_in_bkd_tree, "32");
10421042
DEFINE_mBool(inverted_index_compaction_enable, "false");
10431043
// Only for debug, do not use in production
10441044
DEFINE_mBool(debug_inverted_index_compaction, "false");
1045+
// Only for debug, do not use in production
1046+
DEFINE_mBool(dual_write_inverted_index_enable, "false");
10451047
// index by RAM directory
10461048
DEFINE_mBool(inverted_index_ram_dir_enable, "true");
10471049
// use num_broadcast_buffer blocks as buffer to do broadcast

‎be/src/common/config.h‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1090,7 +1090,11 @@ DECLARE_Int32(max_depth_in_bkd_tree);
10901090
// index compaction
10911091
DECLARE_mBool(inverted_index_compaction_enable);
10921092
// Only for debug, do not use in production
1093+
// Debug switch for collecting intermediate data in inverted index compaction
10931094
DECLARE_mBool(debug_inverted_index_compaction);
1095+
// Only for debug, do not use in production
1096+
// Debug switch for writing inverted index both in compaction process and index compaction process
1097+
DECLARE_mBool(dual_write_inverted_index_enable);
10941098
// index by RAM directory
10951099
DECLARE_mBool(inverted_index_ram_dir_enable);
10961100
// use num_broadcast_buffer blocks as buffer to do broadcast

‎be/src/olap/compaction.cpp‎

Lines changed: 150 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
#include "olap/compaction.h"
1919

2020
#include <fmt/format.h>
21+
#include <gen_cpp/olap_common.pb.h>
2122
#include <gen_cpp/olap_file.pb.h>
2223
#include <glog/logging.h>
2324

@@ -35,6 +36,8 @@
3536
#include <shared_mutex>
3637
#include <utility>
3738

39+
#include "CLucene/config/repl_wchar.h"
40+
#include "CLucene/index/Terms.h"
3841
#include "cloud/cloud_meta_mgr.h"
3942
#include "cloud/cloud_storage_engine.h"
4043
#include "common/config.h"
@@ -619,6 +622,7 @@ Status Compaction::do_inverted_index_compaction() {
619622
// Some columns have already been indexed
620623
// key: seg_id, value: inverted index file size
621624
std::unordered_map<int, int64_t> compacted_idx_file_size;
625+
auto tmp_file_dir = ExecEnv::GetInstance()->get_tmp_file_dirs()->get_tmp_file_dir();
622626
for (int seg_id = 0; seg_id < dest_segment_num; ++seg_id) {
623627
std::string index_path_prefix {
624628
InvertedIndexDescriptor::get_index_file_path_prefix(ctx.segment_path(seg_id))};
@@ -644,6 +648,12 @@ Status Compaction::do_inverted_index_compaction() {
644648
}
645649
compacted_idx_file_size[seg_id] = fsize;
646650
}
651+
// if dual_write_inverted_index_enable is true, we need to write inverted index to tmp dir
652+
if (config::dual_write_inverted_index_enable) {
653+
auto tmp_index_path_prefix =
654+
tmp_file_dir / (dest_rowset_id.to_string() + "_" + std::to_string(seg_id));
655+
index_path_prefix = tmp_index_path_prefix;
656+
}
647657
auto inverted_index_file_writer = std::make_unique<InvertedIndexFileWriter>(
648658
ctx.fs(), index_path_prefix, ctx.rowset_id.to_string(), seg_id,
649659
_cur_tablet_schema->get_inverted_index_storage_format());
@@ -667,7 +677,6 @@ Status Compaction::do_inverted_index_compaction() {
667677
}
668678

669679
// use tmp file dir to store index files
670-
auto tmp_file_dir = ExecEnv::GetInstance()->get_tmp_file_dirs()->get_tmp_file_dir();
671680
auto index_tmp_path = tmp_file_dir / dest_rowset_id.to_string();
672681
LOG(INFO) << "start index compaction"
673682
<< ". tablet=" << _tablet->tablet_id() << ", source index size=" << src_segment_num
@@ -754,6 +763,70 @@ Status Compaction::do_inverted_index_compaction() {
754763
return status;
755764
}
756765

766+
// check idx file correctness only when dual_write_inverted_index_enable is true
767+
if (config::dual_write_inverted_index_enable) {
768+
for (auto&& column_uniq_id : ctx.columns_to_do_index_compaction) {
769+
auto col = _cur_tablet_schema->column_by_uid(column_uniq_id);
770+
const auto* index_meta = _cur_tablet_schema->get_inverted_index(col);
771+
for (int dest_segment_id = 0; dest_segment_id < dest_segment_num; dest_segment_id++) {
772+
// create index file reader for normal compaction index file
773+
std::string index_path_prefix {InvertedIndexDescriptor::get_index_file_path_prefix(
774+
ctx.segment_path(dest_segment_id))};
775+
io::Path cfs_path;
776+
if (_cur_tablet_schema->get_inverted_index_storage_format() !=
777+
doris::InvertedIndexStorageFormatPB::V1) {
778+
cfs_path = InvertedIndexDescriptor::get_index_file_path_v2(index_path_prefix);
779+
} else {
780+
cfs_path = InvertedIndexDescriptor::get_index_file_path_v1(
781+
index_path_prefix, index_meta->index_id(),
782+
index_meta->get_index_suffix());
783+
}
784+
auto inverted_index_file_reader = std::make_unique<InvertedIndexFileReader>(
785+
ctx.fs(), index_path_prefix,
786+
_cur_tablet_schema->get_inverted_index_storage_format());
787+
bool open_idx_file_cache = false;
788+
auto st = inverted_index_file_reader->init(config::inverted_index_read_buffer_size,
789+
open_idx_file_cache);
790+
if (!st.ok()) {
791+
LOG(FATAL) << "inverted_index_file_reader init failed in index compaction "
792+
"correctness check, error:"
793+
<< st;
794+
}
795+
auto index_reader = DORIS_TRY(inverted_index_file_reader->open(index_meta));
796+
797+
// create index file reader for tmp index compaction index file
798+
auto tmp_index_path_prefix = tmp_file_dir / (dest_rowset_id.to_string() + "_" +
799+
std::to_string(dest_segment_id));
800+
auto tmp_inverted_index_file_reader = std::make_unique<InvertedIndexFileReader>(
801+
doris::io::global_local_filesystem(), tmp_index_path_prefix,
802+
_cur_tablet_schema->get_inverted_index_storage_format());
803+
st = tmp_inverted_index_file_reader->init(config::inverted_index_read_buffer_size,
804+
open_idx_file_cache);
805+
if (!st.ok()) {
806+
LOG(FATAL) << "tmp_inverted_index_file_reader init failed in index compaction "
807+
"correctness check, error:"
808+
<< st;
809+
}
810+
auto tmp_index_reader = DORIS_TRY(tmp_inverted_index_file_reader->open(index_meta));
811+
812+
st = check_idx_file_correctness(*index_reader, *tmp_index_reader);
813+
if (!st.ok()) {
814+
LOG(FATAL) << "index compaction correctness check failed"
815+
<< ", tablet=" << _tablet->tablet_id() << ", index_path=" << cfs_path
816+
<< ", tmp_index_path="
817+
<< (tmp_index_path_prefix.string() + "_" +
818+
std::to_string(index_meta->index_id()) + ".idx")
819+
<< ", error=" << st.msg();
820+
}
821+
LOG(INFO) << "index compaction correctness check succeed"
822+
<< ", tablet=" << _tablet->tablet_id() << ", index_path=" << cfs_path
823+
<< ", tmp_index_path="
824+
<< (tmp_index_path_prefix.string() + "_" +
825+
std::to_string(index_meta->index_id()) + ".idx");
826+
}
827+
}
828+
}
829+
757830
// index compaction should update total disk size and index disk size
758831
_output_rowset->rowset_meta()->set_data_disk_size(_output_rowset->data_disk_size() +
759832
inverted_index_file_size);
@@ -776,6 +849,82 @@ Status Compaction::do_inverted_index_compaction() {
776849
return Status::OK();
777850
}
778851

852+
Status Compaction::check_idx_file_correctness(DorisCompoundReader& index_reader,
853+
DorisCompoundReader& tmp_index_reader) {
854+
lucene::index::IndexReader* idx_reader = lucene::index::IndexReader::open(&index_reader);
855+
lucene::index::IndexReader* tmp_idx_reader =
856+
lucene::index::IndexReader::open(&tmp_index_reader);
857+
858+
// compare numDocs
859+
if (idx_reader->numDocs() != tmp_idx_reader->numDocs()) {
860+
return Status::InternalError(
861+
"index compaction correctness check failed, numDocs not equal, idx_numDocs={}, "
862+
"tmp_idx_numDocs={}",
863+
idx_reader->numDocs(), tmp_idx_reader->numDocs());
864+
}
865+
866+
lucene::index::TermEnum* term_enum = idx_reader->terms();
867+
lucene::index::TermEnum* tmp_term_enum = tmp_idx_reader->terms();
868+
869+
// iterate TermEnum
870+
while (term_enum->next() && tmp_term_enum->next()) {
871+
std::string token = lucene_wcstoutf8string(term_enum->term(false)->text(),
872+
term_enum->term(false)->textLength());
873+
std::string field = lucene_wcstoutf8string(term_enum->term(false)->field(),
874+
lenOfString(term_enum->term(false)->field()));
875+
std::string tmp_token = lucene_wcstoutf8string(tmp_term_enum->term(false)->text(),
876+
tmp_term_enum->term(false)->textLength());
877+
std::string tmp_field =
878+
lucene_wcstoutf8string(tmp_term_enum->term(false)->field(),
879+
lenOfString(tmp_term_enum->term(false)->field()));
880+
// compare token and field
881+
if (field != tmp_field) {
882+
return Status::InternalError(
883+
"index compaction correctness check failed, fields not equal, field={}, "
884+
"tmp_field={}",
885+
field, field);
886+
}
887+
if (token != tmp_token) {
888+
return Status::InternalError(
889+
"index compaction correctness check failed, tokens not equal, token={}, "
890+
"tmp_token={}",
891+
token, tmp_token);
892+
}
893+
894+
// get term's docId and freq
895+
lucene::index::TermDocs* term_docs = idx_reader->termDocs(term_enum->term());
896+
lucene::index::TermDocs* tmp_term_docs = tmp_idx_reader->termDocs(tmp_term_enum->term());
897+
898+
// compare term's docId and freq
899+
while (term_docs->next() && tmp_term_docs->next()) {
900+
if (term_docs->doc() != tmp_term_docs->doc() ||
901+
term_docs->freq() != tmp_term_docs->freq()) {
902+
return Status::InternalError(
903+
"index compaction correctness check failed, docId or freq not equal, "
904+
"docId={}, tmp_docId={}, freq={}, tmp_freq={}",
905+
term_docs->doc(), tmp_term_docs->doc(), term_docs->freq(),
906+
tmp_term_docs->freq());
907+
}
908+
}
909+
910+
// check if there are remaining docs
911+
if (term_docs->next() || tmp_term_docs->next()) {
912+
return Status::InternalError(
913+
"index compaction correctness check failed, number of docs not equal for "
914+
"term={}, tmp_term={}",
915+
token, tmp_token);
916+
}
917+
}
918+
919+
// check if there are remaining terms
920+
if (term_enum->next() || tmp_term_enum->next()) {
921+
return Status::InternalError(
922+
"index compaction correctness check failed, number of terms not equal");
923+
}
924+
925+
return Status::OK();
926+
}
927+
779928
void Compaction::construct_index_compaction_columns(RowsetWriterContext& ctx) {
780929
for (const auto& index : _cur_tablet_schema->indexes()) {
781930
if (index.index_type() != IndexType::INVERTED) {

‎be/src/olap/compaction.h‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,9 @@ class Compaction {
6969

7070
Status do_inverted_index_compaction();
7171

72+
Status check_idx_file_correctness(DorisCompoundReader& index_reader,
73+
DorisCompoundReader& tmp_index_reader);
74+
7275
void construct_index_compaction_columns(RowsetWriterContext& ctx);
7376

7477
virtual Status construct_output_rowset_writer(RowsetWriterContext& ctx) = 0;

‎be/src/olap/rowset/segment_v2/segment_writer.cpp‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -208,7 +208,8 @@ Status SegmentWriter::_create_column_writer(uint32_t cid, const TabletColumn& co
208208

209209
opts.need_bitmap_index = column.has_bitmap_index();
210210
bool skip_inverted_index = false;
211-
if (_opts.rowset_ctx != nullptr) {
211+
// if dual_write_inverted_index_enable is true, do not skip write inverted index on index compaction columns
212+
if (_opts.rowset_ctx != nullptr && !config::dual_write_inverted_index_enable) {
212213
// skip write inverted index for index compaction column
213214
skip_inverted_index =
214215
_opts.rowset_ctx->columns_to_do_index_compaction.count(column.unique_id()) > 0;

‎be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -201,7 +201,8 @@ Status VerticalSegmentWriter::_create_column_writer(uint32_t cid, const TabletCo
201201

202202
opts.need_bitmap_index = column.has_bitmap_index();
203203
bool skip_inverted_index = false;
204-
if (_opts.rowset_ctx != nullptr) {
204+
// if dual_write_inverted_index_enable is true, do not skip write inverted index on index compaction columns
205+
if (_opts.rowset_ctx != nullptr && !config::dual_write_inverted_index_enable) {
205206
// skip write inverted index for index compaction column
206207
skip_inverted_index =
207208
_opts.rowset_ctx->columns_to_do_index_compaction.contains(column.unique_id());

0 commit comments

Comments
 (0)