Skip to content

Commit 671c458

Browse files
authored
Fix possible null tablet schema introduced by apache#1562 in delete_rowset_data (apache#1580)
After detaching TabletSchemaPB from rowset meta (apache#1562), the recycler may dereference the null tablet schema in RowsetMetaPB. This commit accomplishes apache#1562 apache#1526.
1 parent cd92407 commit 671c458

5 files changed

Lines changed: 392 additions & 104 deletions

File tree

‎cloud/src/meta-service/keys.cpp‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -236,10 +236,10 @@ void meta_tablet_idx_key(const MetaTabletIdxKeyInfo& in, std::string* out) {
236236
}
237237

238238
void meta_schema_key(const MetaSchemaKeyInfo& in, std::string* out) {
239-
encode_prefix(in, out); // 0x01 "meta" ${instance_id}
240-
encode_bytes(META_KEY_INFIX_TABLET_IDX, out); // "schema"
241-
encode_int64(std::get<1>(in), out); // index_id
242-
encode_int64(std::get<2>(in), out); // schema_version
239+
encode_prefix(in, out); // 0x01 "meta" ${instance_id}
240+
encode_bytes(META_KEY_INFIX_SCHEMA, out); // "schema"
241+
encode_int64(std::get<1>(in), out); // index_id
242+
encode_int64(std::get<2>(in), out); // schema_version
243243
}
244244

245245
//==============================================================================

‎cloud/src/meta-service/meta_service.cpp‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2201,7 +2201,9 @@ void MetaServiceImpl::prepare_rowset(::google::protobuf::RpcController* controll
22012201
int64_t now = duration_cast<seconds>(system_clock::now().time_since_epoch()).count();
22022202
prepare_rowset.set_creation_time(now);
22032203
prepare_rowset.set_expiration(request->rowset_meta().txn_expiration());
2204-
prepare_rowset.mutable_rowset_meta()->CopyFrom(request->rowset_meta());
2204+
// Schema is useless for PREPARE type recycle rowset, set it to null to reduce storage space
2205+
rowset_meta.set_allocated_tablet_schema(nullptr);
2206+
prepare_rowset.mutable_rowset_meta()->CopyFrom(rowset_meta);
22052207
prepare_rowset.set_type(RecycleRowsetPB::PREPARE);
22062208
prepare_rowset.SerializeToString(&prepare_val);
22072209

‎cloud/src/recycler/recycler.cpp‎

Lines changed: 134 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -117,7 +117,7 @@ std::string tablet_path_prefix(int64_t tablet_id) {
117117
if (ret != 0) {
118118
return -1;
119119
}
120-
return txn->get(key, &val);
120+
return txn->get(key, &val, true);
121121
}
122122

123123
// 0 for success, negative for error
@@ -128,7 +128,7 @@ static int txn_get(TxnKv* txn_kv, std::string_view begin, std::string_view end,
128128
if (ret != 0) {
129129
return -1;
130130
}
131-
return txn->get(begin, end, &it);
131+
return txn->get(begin, end, &it, true);
132132
}
133133

134134
// return 0 for success otherwise error
@@ -577,10 +577,86 @@ static int decrypt_ak_sk_helper(const std::string& cipher_ak, const std::string&
577577
return 0;
578578
}
579579

580+
class InstanceRecycler::InvertedIndexIdCache {
581+
public:
582+
InvertedIndexIdCache(std::string instance_id, std::shared_ptr<TxnKv> txn_kv)
583+
: instance_id_(std::move(instance_id)), txn_kv_(std::move(txn_kv)) {}
584+
585+
// Return 0 if success, 1 if schema kv not found, negative for error
586+
int get(int64_t index_id, int32_t schema_version, std::vector<int64_t>& res) {
587+
{
588+
std::lock_guard lock(mtx_);
589+
if (schemas_without_inverted_index_.count({index_id, schema_version})) {
590+
return 0;
591+
}
592+
if (auto it = inverted_index_id_map_.find({index_id, schema_version});
593+
it != inverted_index_id_map_.end()) {
594+
res = it->second;
595+
return 0;
596+
}
597+
}
598+
// Get schema from kv
599+
// TODO(plat1ko): Single flight
600+
std::string schema_key, schema_val;
601+
meta_schema_key({instance_id_, index_id, schema_version}, &schema_key);
602+
int ret = txn_get(txn_kv_.get(), schema_key, schema_val);
603+
if (ret != 0) {
604+
LOG(WARNING) << "failed to get schema, ret=" << ret;
605+
return ret;
606+
}
607+
doris::TabletSchemaPB schema;
608+
if (!schema.ParseFromString(schema_val)) {
609+
LOG(WARNING) << "malformed schema value, key=" << hex(schema_key);
610+
return -1;
611+
}
612+
if (schema.index_size() > 0) {
613+
res.reserve(schema.index_size());
614+
for (auto& i : schema.index()) {
615+
res.push_back(i.index_id());
616+
}
617+
}
618+
insert(index_id, schema_version, res);
619+
return 0;
620+
}
621+
622+
// Empty `ids` means this schema has no inverted index
623+
void insert(int64_t index_id, int32_t schema_version, const std::vector<int64_t>& ids) {
624+
if (ids.empty()) {
625+
TEST_SYNC_POINT_CALLBACK("InvertedIndexIdCache::insert1", nullptr);
626+
std::lock_guard lock(mtx_);
627+
schemas_without_inverted_index_.emplace(index_id, schema_version);
628+
} else {
629+
TEST_SYNC_POINT_CALLBACK("InvertedIndexIdCache::insert2", nullptr);
630+
std::lock_guard lock(mtx_);
631+
inverted_index_id_map_.try_emplace({index_id, schema_version}, ids);
632+
}
633+
}
634+
635+
private:
636+
std::string instance_id_;
637+
std::shared_ptr<TxnKv> txn_kv_;
638+
639+
std::mutex mtx_;
640+
using Key = std::pair<int64_t, int32_t>; // <index_id, schema_version>
641+
struct HashOfKey {
642+
size_t operator()(const Key& key) const {
643+
size_t seed = 0;
644+
seed = std::hash<int64_t> {}(key.first);
645+
seed = std::hash<int32_t> {}(key.second);
646+
return seed;
647+
}
648+
};
649+
// <index_id, schema_version> -> inverted_index_ids
650+
std::unordered_map<Key, std::vector<int64_t>, HashOfKey> inverted_index_id_map_;
651+
// Store <index_id, schema_version> of schema which doesn't have inverted index
652+
std::unordered_set<Key, HashOfKey> schemas_without_inverted_index_;
653+
};
654+
580655
InstanceRecycler::InstanceRecycler(std::shared_ptr<TxnKv> txn_kv, const InstanceInfoPB& instance)
581656
: txn_kv_(std::move(txn_kv)),
582657
instance_id_(instance.instance_id()),
583-
instance_info_(instance) {}
658+
instance_info_(instance),
659+
inverted_index_id_cache_(std::make_unique<InvertedIndexIdCache>(instance_id_, txn_kv_)) {}
584660

585661
InstanceRecycler::~InstanceRecycler() = default;
586662

@@ -980,6 +1056,13 @@ int InstanceRecycler::recycle_tablets(int64_t table_id, int64_t index_id, int64_
9801056
}
9811057
txn->remove(stats_key_begin, stats_key_end);
9821058
txn->remove(job_key_begin, job_key_end);
1059+
std::string schema_key_begin, schema_key_end;
1060+
if (partition_id <= 0) {
1061+
// Delete schema kv of this index
1062+
meta_schema_key({instance_id_, index_id, 0}, &schema_key_begin);
1063+
meta_schema_key({instance_id_, index_id + 1, 0}, &schema_key_end);
1064+
txn->remove(schema_key_begin, schema_key_end);
1065+
}
9831066
if (txn->commit() != 0) {
9841067
LOG(WARNING) << "failed to delete tablet job or stats key, instance_id=" << instance_id_;
9851068
return -1;
@@ -991,6 +1074,10 @@ int InstanceRecycler::recycle_tablets(int64_t table_id, int64_t index_id, int64_
9911074
int InstanceRecycler::delete_rowset_data(const doris::RowsetMetaPB& rs_meta_pb) {
9921075
int64_t num_segments = rs_meta_pb.num_segments();
9931076
if (num_segments <= 0) return 0;
1077+
if (!rs_meta_pb.has_tablet_schema()) {
1078+
return delete_rowset_data(rs_meta_pb.resource_id(), rs_meta_pb.tablet_id(),
1079+
rs_meta_pb.rowset_id_v2());
1080+
}
9941081
auto it = accessor_map_.find(rs_meta_pb.resource_id());
9951082
if (it == accessor_map_.end()) {
9961083
LOG_WARNING("instance has no such resource id")
@@ -1007,7 +1094,6 @@ int InstanceRecycler::delete_rowset_data(const doris::RowsetMetaPB& rs_meta_pb)
10071094
for (auto& index_pb : rs_meta_pb.tablet_schema().index()) {
10081095
index_ids.push_back(index_pb.index_id());
10091096
}
1010-
10111097
std::vector<std::string> file_paths;
10121098
file_paths.reserve(num_segments * (1 + index_ids.size()));
10131099
for (int64_t i = 0; i < num_segments; ++i) {
@@ -1020,14 +1106,22 @@ int InstanceRecycler::delete_rowset_data(const doris::RowsetMetaPB& rs_meta_pb)
10201106
}
10211107

10221108
int InstanceRecycler::delete_rowset_data(const std::vector<doris::RowsetMetaPB>& rowsets) {
1109+
int ret = 0;
10231110
// resource_id -> file_paths
10241111
std::map<std::string, std::vector<std::string>> resource_file_paths;
10251112
for (auto& rs : rowsets) {
1113+
{
1114+
std::lock_guard lock(recycled_tablets_mtx_);
1115+
if (recycled_tablets_.count(rs.tablet_id())) {
1116+
continue; // Rowset data has already been deleted
1117+
}
1118+
}
10261119
auto it = accessor_map_.find(rs.resource_id());
10271120
if (it == accessor_map_.end()) [[unlikely]] { // impossible
10281121
LOG_WARNING("instance has no such resource id")
10291122
.tag("instance_id", instance_id_)
10301123
.tag("resource_id", rs.resource_id());
1124+
ret = -1;
10311125
continue;
10321126
}
10331127
auto& file_paths = resource_file_paths[rs.resource_id()];
@@ -1037,9 +1131,39 @@ int InstanceRecycler::delete_rowset_data(const std::vector<doris::RowsetMetaPB>&
10371131
if (num_segments <= 0) continue;
10381132
// process inverted indexes
10391133
std::vector<int64_t> index_ids;
1040-
index_ids.reserve(rs.tablet_schema().index().size());
1041-
for (auto& index_pb : rs.tablet_schema().index()) {
1042-
index_ids.push_back(index_pb.index_id());
1134+
if (rs.has_tablet_schema()) {
1135+
index_ids.reserve(rs.tablet_schema().index().size());
1136+
for (auto& index_pb : rs.tablet_schema().index()) {
1137+
index_ids.push_back(index_pb.index_id());
1138+
}
1139+
} else { // Detached schema
1140+
if (!rs.has_index_id() || !rs.has_schema_version()) {
1141+
LOG(WARNING) << "rowset must have either schema or schema_version and index_id, "
1142+
"instance_id="
1143+
<< instance_id_ << " tablet_id=" << tablet_id
1144+
<< " rowset_id=" << rowset_id;
1145+
ret = -1;
1146+
continue;
1147+
}
1148+
int get_ret =
1149+
inverted_index_id_cache_->get(rs.index_id(), rs.schema_version(), index_ids);
1150+
if (get_ret != 0) {
1151+
if (get_ret == 1) { // Schema kv not found
1152+
// Check tablet existence
1153+
std::string tablet_idx_key, tablet_idx_val;
1154+
meta_tablet_idx_key({instance_id_, tablet_id}, &tablet_idx_key);
1155+
if (txn_get(txn_kv_.get(), tablet_idx_key, tablet_idx_val) == 1) {
1156+
// Tablet has been recycled, rowset data has already been deleted
1157+
std::lock_guard lock(recycled_tablets_mtx_);
1158+
recycled_tablets_.insert(tablet_id);
1159+
continue;
1160+
}
1161+
}
1162+
LOG(WARNING) << "failed to get schema kv for rowset, instance_id=" << instance_id_
1163+
<< " tablet_id=" << tablet_id << " rowset_id=" << rowset_id;
1164+
ret = -1;
1165+
continue;
1166+
}
10431167
}
10441168
for (int64_t i = 0; i < num_segments; ++i) {
10451169
file_paths.push_back(segment_path(tablet_id, rowset_id, i));
@@ -1048,7 +1172,6 @@ int InstanceRecycler::delete_rowset_data(const std::vector<doris::RowsetMetaPB>&
10481172
}
10491173
}
10501174
}
1051-
int ret = 0;
10521175
for (auto& [resource_id, file_paths] : resource_file_paths) {
10531176
auto& accessor = accessor_map_[resource_id];
10541177
DCHECK(accessor);
@@ -1112,6 +1235,9 @@ int InstanceRecycler::recycle_tablet(int64_t tablet_id) {
11121235
LOG(WARNING) << "failed to delete rowset data of tablet " << tablet_id
11131236
<< " s3_path=" << accessor->path();
11141237
ret = -1;
1238+
} else {
1239+
std::lock_guard lock(recycled_tablets_mtx_);
1240+
recycled_tablets_.insert(tablet_id);
11151241
}
11161242
}
11171243
return ret;

‎cloud/src/recycler/recycler.h‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -155,6 +155,13 @@ class InstanceRecycler {
155155
std::string instance_id_;
156156
InstanceInfoPB instance_info_;
157157
std::unordered_map<std::string, std::shared_ptr<ObjStoreAccessor>> accessor_map_;
158+
159+
class InvertedIndexIdCache;
160+
std::unique_ptr<InvertedIndexIdCache> inverted_index_id_cache_;
161+
162+
std::mutex recycled_tablets_mtx_;
163+
// Store recycled tablets, we can skip deleting rowset data of these tablets because these data has already been deleted.
164+
std::unordered_set<int64_t> recycled_tablets_;
158165
};
159166

160167
} // namespace selectdb

0 commit comments

Comments
 (0)