@@ -117,7 +117,7 @@ std::string tablet_path_prefix(int64_t tablet_id) {
117117 if (ret != 0 ) {
118118 return -1 ;
119119 }
120- return txn->get (key, &val);
120+ return txn->get (key, &val, true );
121121}
122122
123123// 0 for success, negative for error
@@ -128,7 +128,7 @@ static int txn_get(TxnKv* txn_kv, std::string_view begin, std::string_view end,
128128 if (ret != 0 ) {
129129 return -1 ;
130130 }
131- return txn->get (begin, end, &it);
131+ return txn->get (begin, end, &it, true );
132132}
133133
134134// return 0 for success otherwise error
@@ -577,10 +577,86 @@ static int decrypt_ak_sk_helper(const std::string& cipher_ak, const std::string&
577577 return 0 ;
578578}
579579
580+ class InstanceRecycler ::InvertedIndexIdCache {
581+ public:
582+ InvertedIndexIdCache (std::string instance_id, std::shared_ptr<TxnKv> txn_kv)
583+ : instance_id_(std::move(instance_id)), txn_kv_(std::move(txn_kv)) {}
584+
585+ // Return 0 if success, 1 if schema kv not found, negative for error
586+ int get (int64_t index_id, int32_t schema_version, std::vector<int64_t >& res) {
587+ {
588+ std::lock_guard lock (mtx_);
589+ if (schemas_without_inverted_index_.count ({index_id, schema_version})) {
590+ return 0 ;
591+ }
592+ if (auto it = inverted_index_id_map_.find ({index_id, schema_version});
593+ it != inverted_index_id_map_.end ()) {
594+ res = it->second ;
595+ return 0 ;
596+ }
597+ }
598+ // Get schema from kv
599+ // TODO(plat1ko): Single flight
600+ std::string schema_key, schema_val;
601+ meta_schema_key ({instance_id_, index_id, schema_version}, &schema_key);
602+ int ret = txn_get (txn_kv_.get (), schema_key, schema_val);
603+ if (ret != 0 ) {
604+ LOG (WARNING ) << " failed to get schema, ret=" << ret;
605+ return ret;
606+ }
607+ doris::TabletSchemaPB schema;
608+ if (!schema.ParseFromString (schema_val)) {
609+ LOG (WARNING ) << " malformed schema value, key=" << hex (schema_key);
610+ return -1 ;
611+ }
612+ if (schema.index_size () > 0 ) {
613+ res.reserve (schema.index_size ());
614+ for (auto & i : schema.index ()) {
615+ res.push_back (i.index_id ());
616+ }
617+ }
618+ insert (index_id, schema_version, res);
619+ return 0 ;
620+ }
621+
622+ // Empty `ids` means this schema has no inverted index
623+ void insert (int64_t index_id, int32_t schema_version, const std::vector<int64_t >& ids) {
624+ if (ids.empty ()) {
625+ TEST_SYNC_POINT_CALLBACK (" InvertedIndexIdCache::insert1" , nullptr );
626+ std::lock_guard lock (mtx_);
627+ schemas_without_inverted_index_.emplace (index_id, schema_version);
628+ } else {
629+ TEST_SYNC_POINT_CALLBACK (" InvertedIndexIdCache::insert2" , nullptr );
630+ std::lock_guard lock (mtx_);
631+ inverted_index_id_map_.try_emplace ({index_id, schema_version}, ids);
632+ }
633+ }
634+
635+ private:
636+ std::string instance_id_;
637+ std::shared_ptr<TxnKv> txn_kv_;
638+
639+ std::mutex mtx_;
640+ using Key = std::pair<int64_t , int32_t >; // <index_id, schema_version>
641+ struct HashOfKey {
642+ size_t operator ()(const Key& key) const {
643+ size_t seed = 0 ;
644+ seed = std::hash<int64_t > {}(key.first );
645+ seed = std::hash<int32_t > {}(key.second );
646+ return seed;
647+ }
648+ };
649+ // <index_id, schema_version> -> inverted_index_ids
650+ std::unordered_map<Key, std::vector<int64_t >, HashOfKey> inverted_index_id_map_;
651+ // Store <index_id, schema_version> of schema which doesn't have inverted index
652+ std::unordered_set<Key, HashOfKey> schemas_without_inverted_index_;
653+ };
654+
580655InstanceRecycler::InstanceRecycler (std::shared_ptr<TxnKv> txn_kv, const InstanceInfoPB& instance)
581656 : txn_kv_(std::move(txn_kv)),
582657 instance_id_ (instance.instance_id()),
583- instance_info_(instance) {}
658+ instance_info_(instance),
659+ inverted_index_id_cache_(std::make_unique<InvertedIndexIdCache>(instance_id_, txn_kv_)) {}
584660
585661InstanceRecycler::~InstanceRecycler () = default ;
586662
@@ -980,6 +1056,13 @@ int InstanceRecycler::recycle_tablets(int64_t table_id, int64_t index_id, int64_
9801056 }
9811057 txn->remove (stats_key_begin, stats_key_end);
9821058 txn->remove (job_key_begin, job_key_end);
1059+ std::string schema_key_begin, schema_key_end;
1060+ if (partition_id <= 0 ) {
1061+ // Delete schema kv of this index
1062+ meta_schema_key ({instance_id_, index_id, 0 }, &schema_key_begin);
1063+ meta_schema_key ({instance_id_, index_id + 1 , 0 }, &schema_key_end);
1064+ txn->remove (schema_key_begin, schema_key_end);
1065+ }
9831066 if (txn->commit () != 0 ) {
9841067 LOG (WARNING ) << " failed to delete tablet job or stats key, instance_id=" << instance_id_;
9851068 return -1 ;
@@ -991,6 +1074,10 @@ int InstanceRecycler::recycle_tablets(int64_t table_id, int64_t index_id, int64_
9911074int InstanceRecycler::delete_rowset_data (const doris::RowsetMetaPB& rs_meta_pb) {
9921075 int64_t num_segments = rs_meta_pb.num_segments ();
9931076 if (num_segments <= 0 ) return 0 ;
1077+ if (!rs_meta_pb.has_tablet_schema ()) {
1078+ return delete_rowset_data (rs_meta_pb.resource_id (), rs_meta_pb.tablet_id (),
1079+ rs_meta_pb.rowset_id_v2 ());
1080+ }
9941081 auto it = accessor_map_.find (rs_meta_pb.resource_id ());
9951082 if (it == accessor_map_.end ()) {
9961083 LOG_WARNING (" instance has no such resource id" )
@@ -1007,7 +1094,6 @@ int InstanceRecycler::delete_rowset_data(const doris::RowsetMetaPB& rs_meta_pb)
10071094 for (auto & index_pb : rs_meta_pb.tablet_schema ().index ()) {
10081095 index_ids.push_back (index_pb.index_id ());
10091096 }
1010-
10111097 std::vector<std::string> file_paths;
10121098 file_paths.reserve (num_segments * (1 + index_ids.size ()));
10131099 for (int64_t i = 0 ; i < num_segments; ++i) {
@@ -1020,14 +1106,22 @@ int InstanceRecycler::delete_rowset_data(const doris::RowsetMetaPB& rs_meta_pb)
10201106}
10211107
10221108int InstanceRecycler::delete_rowset_data (const std::vector<doris::RowsetMetaPB>& rowsets) {
1109+ int ret = 0 ;
10231110 // resource_id -> file_paths
10241111 std::map<std::string, std::vector<std::string>> resource_file_paths;
10251112 for (auto & rs : rowsets) {
1113+ {
1114+ std::lock_guard lock (recycled_tablets_mtx_);
1115+ if (recycled_tablets_.count (rs.tablet_id ())) {
1116+ continue ; // Rowset data has already been deleted
1117+ }
1118+ }
10261119 auto it = accessor_map_.find (rs.resource_id ());
10271120 if (it == accessor_map_.end ()) [[unlikely]] { // impossible
10281121 LOG_WARNING (" instance has no such resource id" )
10291122 .tag (" instance_id" , instance_id_)
10301123 .tag (" resource_id" , rs.resource_id ());
1124+ ret = -1 ;
10311125 continue ;
10321126 }
10331127 auto & file_paths = resource_file_paths[rs.resource_id ()];
@@ -1037,9 +1131,39 @@ int InstanceRecycler::delete_rowset_data(const std::vector<doris::RowsetMetaPB>&
10371131 if (num_segments <= 0 ) continue ;
10381132 // process inverted indexes
10391133 std::vector<int64_t > index_ids;
1040- index_ids.reserve (rs.tablet_schema ().index ().size ());
1041- for (auto & index_pb : rs.tablet_schema ().index ()) {
1042- index_ids.push_back (index_pb.index_id ());
1134+ if (rs.has_tablet_schema ()) {
1135+ index_ids.reserve (rs.tablet_schema ().index ().size ());
1136+ for (auto & index_pb : rs.tablet_schema ().index ()) {
1137+ index_ids.push_back (index_pb.index_id ());
1138+ }
1139+ } else { // Detached schema
1140+ if (!rs.has_index_id () || !rs.has_schema_version ()) {
1141+ LOG (WARNING ) << " rowset must have either schema or schema_version and index_id, "
1142+ " instance_id="
1143+ << instance_id_ << " tablet_id=" << tablet_id
1144+ << " rowset_id=" << rowset_id;
1145+ ret = -1 ;
1146+ continue ;
1147+ }
1148+ int get_ret =
1149+ inverted_index_id_cache_->get (rs.index_id (), rs.schema_version (), index_ids);
1150+ if (get_ret != 0 ) {
1151+ if (get_ret == 1 ) { // Schema kv not found
1152+ // Check tablet existence
1153+ std::string tablet_idx_key, tablet_idx_val;
1154+ meta_tablet_idx_key ({instance_id_, tablet_id}, &tablet_idx_key);
1155+ if (txn_get (txn_kv_.get (), tablet_idx_key, tablet_idx_val) == 1 ) {
1156+ // Tablet has been recycled, rowset data has already been deleted
1157+ std::lock_guard lock (recycled_tablets_mtx_);
1158+ recycled_tablets_.insert (tablet_id);
1159+ continue ;
1160+ }
1161+ }
1162+ LOG (WARNING ) << " failed to get schema kv for rowset, instance_id=" << instance_id_
1163+ << " tablet_id=" << tablet_id << " rowset_id=" << rowset_id;
1164+ ret = -1 ;
1165+ continue ;
1166+ }
10431167 }
10441168 for (int64_t i = 0 ; i < num_segments; ++i) {
10451169 file_paths.push_back (segment_path (tablet_id, rowset_id, i));
@@ -1048,7 +1172,6 @@ int InstanceRecycler::delete_rowset_data(const std::vector<doris::RowsetMetaPB>&
10481172 }
10491173 }
10501174 }
1051- int ret = 0 ;
10521175 for (auto & [resource_id, file_paths] : resource_file_paths) {
10531176 auto & accessor = accessor_map_[resource_id];
10541177 DCHECK (accessor);
@@ -1112,6 +1235,9 @@ int InstanceRecycler::recycle_tablet(int64_t tablet_id) {
11121235 LOG (WARNING ) << " failed to delete rowset data of tablet " << tablet_id
11131236 << " s3_path=" << accessor->path ();
11141237 ret = -1 ;
1238+ } else {
1239+ std::lock_guard lock (recycled_tablets_mtx_);
1240+ recycled_tablets_.insert (tablet_id);
11151241 }
11161242 }
11171243 return ret;
0 commit comments