1818#include " olap/compaction.h"
1919
2020#include < fmt/format.h>
21+ #include < gen_cpp/olap_common.pb.h>
2122#include < gen_cpp/olap_file.pb.h>
2223#include < glog/logging.h>
2324
3536#include < shared_mutex>
3637#include < utility>
3738
39+ #include " CLucene/config/repl_wchar.h"
40+ #include " CLucene/index/Terms.h"
3841#include " cloud/cloud_meta_mgr.h"
3942#include " cloud/cloud_storage_engine.h"
4043#include " common/config.h"
@@ -619,6 +622,7 @@ Status Compaction::do_inverted_index_compaction() {
619622 // Some columns have already been indexed
620623 // key: seg_id, value: inverted index file size
621624 std::unordered_map<int , int64_t > compacted_idx_file_size;
625+ auto tmp_file_dir = ExecEnv::GetInstance ()->get_tmp_file_dirs ()->get_tmp_file_dir ();
622626 for (int seg_id = 0 ; seg_id < dest_segment_num; ++seg_id) {
623627 std::string index_path_prefix {
624628 InvertedIndexDescriptor::get_index_file_path_prefix (ctx.segment_path (seg_id))};
@@ -644,6 +648,12 @@ Status Compaction::do_inverted_index_compaction() {
644648 }
645649 compacted_idx_file_size[seg_id] = fsize;
646650 }
651+ // if dual_write_inverted_index_enable is true, we need to write inverted index to tmp dir
652+ if (config::dual_write_inverted_index_enable) {
653+ auto tmp_index_path_prefix =
654+ tmp_file_dir / (dest_rowset_id.to_string () + " _" + std::to_string (seg_id));
655+ index_path_prefix = tmp_index_path_prefix;
656+ }
647657 auto inverted_index_file_writer = std::make_unique<InvertedIndexFileWriter>(
648658 ctx.fs (), index_path_prefix, ctx.rowset_id .to_string (), seg_id,
649659 _cur_tablet_schema->get_inverted_index_storage_format ());
@@ -667,7 +677,6 @@ Status Compaction::do_inverted_index_compaction() {
667677 }
668678
669679 // use tmp file dir to store index files
670- auto tmp_file_dir = ExecEnv::GetInstance ()->get_tmp_file_dirs ()->get_tmp_file_dir ();
671680 auto index_tmp_path = tmp_file_dir / dest_rowset_id.to_string ();
672681 LOG (INFO ) << " start index compaction"
673682 << " . tablet=" << _tablet->tablet_id () << " , source index size=" << src_segment_num
@@ -754,6 +763,70 @@ Status Compaction::do_inverted_index_compaction() {
754763 return status;
755764 }
756765
766+ // check idx file correctness only when dual_write_inverted_index_enable is true
767+ if (config::dual_write_inverted_index_enable) {
768+ for (auto && column_uniq_id : ctx.columns_to_do_index_compaction ) {
769+ auto col = _cur_tablet_schema->column_by_uid (column_uniq_id);
770+ const auto * index_meta = _cur_tablet_schema->get_inverted_index (col);
771+ for (int dest_segment_id = 0 ; dest_segment_id < dest_segment_num; dest_segment_id++) {
772+ // create index file reader for normal compaction index file
773+ std::string index_path_prefix {InvertedIndexDescriptor::get_index_file_path_prefix (
774+ ctx.segment_path (dest_segment_id))};
775+ io::Path cfs_path;
776+ if (_cur_tablet_schema->get_inverted_index_storage_format () !=
777+ doris::InvertedIndexStorageFormatPB::V1 ) {
778+ cfs_path = InvertedIndexDescriptor::get_index_file_path_v2 (index_path_prefix);
779+ } else {
780+ cfs_path = InvertedIndexDescriptor::get_index_file_path_v1 (
781+ index_path_prefix, index_meta->index_id (),
782+ index_meta->get_index_suffix ());
783+ }
784+ auto inverted_index_file_reader = std::make_unique<InvertedIndexFileReader>(
785+ ctx.fs (), index_path_prefix,
786+ _cur_tablet_schema->get_inverted_index_storage_format ());
787+ bool open_idx_file_cache = false ;
788+ auto st = inverted_index_file_reader->init (config::inverted_index_read_buffer_size,
789+ open_idx_file_cache);
790+ if (!st.ok ()) {
791+ LOG (FATAL ) << " inverted_index_file_reader init failed in index compaction "
792+ " correctness check, error:"
793+ << st;
794+ }
795+ auto index_reader = DORIS_TRY (inverted_index_file_reader->open (index_meta));
796+
797+ // create index file reader for tmp index compaction index file
798+ auto tmp_index_path_prefix = tmp_file_dir / (dest_rowset_id.to_string () + " _" +
799+ std::to_string (dest_segment_id));
800+ auto tmp_inverted_index_file_reader = std::make_unique<InvertedIndexFileReader>(
801+ doris::io::global_local_filesystem (), tmp_index_path_prefix,
802+ _cur_tablet_schema->get_inverted_index_storage_format ());
803+ st = tmp_inverted_index_file_reader->init (config::inverted_index_read_buffer_size,
804+ open_idx_file_cache);
805+ if (!st.ok ()) {
806+ LOG (FATAL ) << " tmp_inverted_index_file_reader init failed in index compaction "
807+ " correctness check, error:"
808+ << st;
809+ }
810+ auto tmp_index_reader = DORIS_TRY (tmp_inverted_index_file_reader->open (index_meta));
811+
812+ st = check_idx_file_correctness (*index_reader, *tmp_index_reader);
813+ if (!st.ok ()) {
814+ LOG (FATAL ) << " index compaction correctness check failed"
815+ << " , tablet=" << _tablet->tablet_id () << " , index_path=" << cfs_path
816+ << " , tmp_index_path="
817+ << (tmp_index_path_prefix.string () + " _" +
818+ std::to_string (index_meta->index_id ()) + " .idx" )
819+ << " , error=" << st.msg ();
820+ }
821+ LOG (INFO ) << " index compaction correctness check succeed"
822+ << " , tablet=" << _tablet->tablet_id () << " , index_path=" << cfs_path
823+ << " , tmp_index_path="
824+ << (tmp_index_path_prefix.string () + " _" +
825+ std::to_string (index_meta->index_id ()) + " .idx" );
826+ }
827+ }
828+ }
829+
757830 // index compaction should update total disk size and index disk size
758831 _output_rowset->rowset_meta ()->set_data_disk_size (_output_rowset->data_disk_size () +
759832 inverted_index_file_size);
@@ -776,6 +849,82 @@ Status Compaction::do_inverted_index_compaction() {
776849 return Status::OK ();
777850}
778851
852+ Status Compaction::check_idx_file_correctness (DorisCompoundReader& index_reader,
853+ DorisCompoundReader& tmp_index_reader) {
854+ lucene::index::IndexReader* idx_reader = lucene::index::IndexReader::open (&index_reader);
855+ lucene::index::IndexReader* tmp_idx_reader =
856+ lucene::index::IndexReader::open (&tmp_index_reader);
857+
858+ // compare numDocs
859+ if (idx_reader->numDocs () != tmp_idx_reader->numDocs ()) {
860+ return Status::InternalError (
861+ " index compaction correctness check failed, numDocs not equal, idx_numDocs={}, "
862+ " tmp_idx_numDocs={}" ,
863+ idx_reader->numDocs (), tmp_idx_reader->numDocs ());
864+ }
865+
866+ lucene::index::TermEnum* term_enum = idx_reader->terms ();
867+ lucene::index::TermEnum* tmp_term_enum = tmp_idx_reader->terms ();
868+
869+ // iterate TermEnum
870+ while (term_enum->next () && tmp_term_enum->next ()) {
871+ std::string token = lucene_wcstoutf8string (term_enum->term (false )->text (),
872+ term_enum->term (false )->textLength ());
873+ std::string field = lucene_wcstoutf8string (term_enum->term (false )->field (),
874+ lenOfString (term_enum->term (false )->field ()));
875+ std::string tmp_token = lucene_wcstoutf8string (tmp_term_enum->term (false )->text (),
876+ tmp_term_enum->term (false )->textLength ());
877+ std::string tmp_field =
878+ lucene_wcstoutf8string (tmp_term_enum->term (false )->field (),
879+ lenOfString (tmp_term_enum->term (false )->field ()));
880+ // compare token and field
881+ if (field != tmp_field) {
882+ return Status::InternalError (
883+ " index compaction correctness check failed, fields not equal, field={}, "
884+ " tmp_field={}" ,
885+ field, field);
886+ }
887+ if (token != tmp_token) {
888+ return Status::InternalError (
889+ " index compaction correctness check failed, tokens not equal, token={}, "
890+ " tmp_token={}" ,
891+ token, tmp_token);
892+ }
893+
894+ // get term's docId and freq
895+ lucene::index::TermDocs* term_docs = idx_reader->termDocs (term_enum->term ());
896+ lucene::index::TermDocs* tmp_term_docs = tmp_idx_reader->termDocs (tmp_term_enum->term ());
897+
898+ // compare term's docId and freq
899+ while (term_docs->next () && tmp_term_docs->next ()) {
900+ if (term_docs->doc () != tmp_term_docs->doc () ||
901+ term_docs->freq () != tmp_term_docs->freq ()) {
902+ return Status::InternalError (
903+ " index compaction correctness check failed, docId or freq not equal, "
904+ " docId={}, tmp_docId={}, freq={}, tmp_freq={}" ,
905+ term_docs->doc (), tmp_term_docs->doc (), term_docs->freq (),
906+ tmp_term_docs->freq ());
907+ }
908+ }
909+
910+ // check if there are remaining docs
911+ if (term_docs->next () || tmp_term_docs->next ()) {
912+ return Status::InternalError (
913+ " index compaction correctness check failed, number of docs not equal for "
914+ " term={}, tmp_term={}" ,
915+ token, tmp_token);
916+ }
917+ }
918+
919+ // check if there are remaining terms
920+ if (term_enum->next () || tmp_term_enum->next ()) {
921+ return Status::InternalError (
922+ " index compaction correctness check failed, number of terms not equal" );
923+ }
924+
925+ return Status::OK ();
926+ }
927+
779928void Compaction::construct_index_compaction_columns (RowsetWriterContext& ctx) {
780929 for (const auto & index : _cur_tablet_schema->indexes ()) {
781930 if (index.index_type () != IndexType::INVERTED ) {
0 commit comments