Skip to content

Commit dede454

Browse files
committed
[fix](cloud) Skip stale delete bitmap aggregation
### What problem does this PR solve? Problem Summary: Delayed compaction aggregation can overwrite newer delete bitmaps and make deleted rows visible. Validate the compaction output range in every MS transaction, stopping stale requests before writes or cleanup. Preserve non-OK error checks and add unit coverage for concurrent replacement, same-range replacement, and transaction splitting, plus a two-BE Docker case. ### Release note Fix deleted rows becoming visible after stale compaction bitmap aggregation. ### Check List (For Author) - Test: ASAN MetaServiceTest.* passed all 109 tests on the development machine. Clang-tidy, formatting, and diff checks passed. The Docker case reproduced the bug before the fix; it has not been rerun after the fix. - Behavior changed: Yes. Stale aggregation is skipped; errors stop processing. - Does this need documentation: No
1 parent 37c97fe commit dede454

3 files changed

Lines changed: 389 additions & 24 deletions

File tree

‎cloud/src/meta-service/meta_service.cpp‎

Lines changed: 83 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -3920,7 +3920,51 @@ static bool check_delete_bitmap_point_delete(MetaServiceCode& code, std::string&
39203920
return true;
39213921
}
39223922

3923-
void _write_delete_bitmap_kvs(MetaServiceCode& code, std::string& msg, std::stringstream& ss,
3923+
// False stops the whole request, including cleanup. An obsolete aggregation is a successful no-op.
3924+
static bool check_pre_rowset_aggregation(MetaServiceCode& code, std::string& msg, Transaction* txn,
3925+
const UpdateDeleteBitmapRequest* request,
3926+
const std::string& instance_id) {
3927+
if (!request->without_lock() ||
3928+
request->lock_id() != COMPACTION_WITHOUT_LOCK_DELETE_BITMAP_LOCK_ID ||
3929+
!request->has_pre_rowset_agg_start_version() ||
3930+
!request->has_pre_rowset_agg_end_version()) {
3931+
return true;
3932+
}
3933+
3934+
auto tablet_id = request->tablet_id();
3935+
auto start = request->pre_rowset_agg_start_version();
3936+
auto end = request->pre_rowset_agg_end_version();
3937+
auto key = meta_rowset_key({instance_id, tablet_id, end});
3938+
std::string value;
3939+
// Compaction removes/replaces this live key even with versioned metadata enabled.
3940+
// A non-snapshot read conflicts with a compaction that invalidates the output rowset
3941+
// before this transaction commits. Historical versioned rowsets cannot establish liveness.
3942+
auto err = txn->get(key, &value);
3943+
if (err != TxnErrorCode::TXN_OK && err != TxnErrorCode::TXN_KEY_NOT_FOUND) {
3944+
code = cast_as<ErrCategory::READ>(err);
3945+
msg = fmt::format("failed to get aggregation output rowset, tablet_id={}, key={}, err={}",
3946+
tablet_id, hex(key), err);
3947+
return false;
3948+
}
3949+
RowsetMetaCloudPB rowset;
3950+
if (err == TxnErrorCode::TXN_OK && !rowset.ParseFromString(value)) {
3951+
code = MetaServiceCode::PROTOBUF_PARSE_ERR;
3952+
msg = fmt::format("malformed aggregation output rowset, tablet_id={}, key={}", tablet_id,
3953+
hex(key));
3954+
return false;
3955+
}
3956+
if (err == TxnErrorCode::TXN_KEY_NOT_FOUND || rowset.start_version() != start ||
3957+
rowset.end_version() != end) {
3958+
LOG(INFO) << "skip obsolete pre rowset delete bitmap aggregation, instance_id="
3959+
<< instance_id << " tablet_id=" << tablet_id << " start_version=" << start
3960+
<< " end_version=" << end;
3961+
return false;
3962+
}
3963+
TEST_SYNC_POINT_CALLBACK("update_delete_bitmap:check_pre_rowset_aggregation", request);
3964+
return true;
3965+
}
3966+
3967+
bool _write_delete_bitmap_kvs(MetaServiceCode& code, std::string& msg, std::stringstream& ss,
39243968
std::shared_ptr<TxnKv> txn_kv, std::unique_ptr<Transaction>& txn,
39253969
std::string& use_version, std::set<std::string>& non_exist_rowset_ids,
39263970
size_t i, std::string& log, KVStats& stats,
@@ -3954,7 +3998,7 @@ void _write_delete_bitmap_kvs(MetaServiceCode& code, std::string& msg, std::stri
39543998
<< " txn_size=" << txn->approximate_bytes();
39553999
msg = ss.str();
39564000
g_bvar_update_delete_bitmap_fail_counter << 1;
3957-
return;
4001+
return false;
39584002
}
39594003
stats.get_bytes += txn->get_bytes();
39604004
stats.put_bytes += txn->put_bytes();
@@ -3968,7 +4012,10 @@ void _write_delete_bitmap_kvs(MetaServiceCode& code, std::string& msg, std::stri
39684012
if (err != TxnErrorCode::TXN_OK) {
39694013
code = cast_as<ErrCategory::CREATE>(err);
39704014
msg = "failed to init txn";
3971-
return;
4015+
return false;
4016+
}
4017+
if (!check_pre_rowset_aggregation(code, msg, txn.get(), request, instance_id)) {
4018+
return false;
39724019
}
39734020
if (!without_lock) {
39744021
std::string lock_key = meta_delete_bitmap_update_lock_key({instance_id, table_id, -1});
@@ -3979,7 +4026,7 @@ void _write_delete_bitmap_kvs(MetaServiceCode& code, std::string& msg, std::stri
39794026
LOG(WARNING) << "failed to check delete bitmap lock, table_id=" << table_id
39804027
<< " request lock_id=" << request->lock_id()
39814028
<< " request initiator=" << request->initiator() << " msg " << msg;
3982-
return;
4029+
return false;
39834030
}
39844031
}
39854032
}
@@ -3988,7 +4035,7 @@ void _write_delete_bitmap_kvs(MetaServiceCode& code, std::string& msg, std::stri
39884035
if (config::enable_update_delete_bitmap_kv_check) {
39894036
if (!check_delete_bitmap_kv_exists(code, msg, txn, key, instance_id, tablet_id,
39904037
request->lock_id())) {
3991-
return;
4038+
return false;
39924039
}
39934040
}
39944041
#endif
@@ -4000,7 +4047,7 @@ void _write_delete_bitmap_kvs(MetaServiceCode& code, std::string& msg, std::stri
40004047
LOG(INFO) << "skip update delete bitmap, rowset_id=" << request->rowset_ids(i)
40014048
<< " version=" << request->pre_rowset_versions(i)
40024049
<< " tablet_id=" << tablet_id << " because the rowset does not exist";
4003-
return;
4050+
return true;
40044051
}
40054052

40064053
TxnErrorCode err = TxnErrorCode::TXN_OK;
@@ -4016,15 +4063,15 @@ void _write_delete_bitmap_kvs(MetaServiceCode& code, std::string& msg, std::stri
40164063
<< " err=" << err;
40174064
msg = ss.str();
40184065
code = cast_as<ErrCategory::READ>(err);
4019-
return;
4066+
return false;
40204067
}
40214068
if (err == TxnErrorCode::TXN_OK &&
40224069
!rs.ParseFromArray(rowset_val.data(), rowset_val.size())) {
40234070
code = MetaServiceCode::PROTOBUF_PARSE_ERR;
40244071
ss << "malformed rowset meta, unable to deserialize, tablet_id=" << tablet_id
40254072
<< " key=" << hex(rowset_key);
40264073
msg = ss.str();
4027-
return;
4074+
return false;
40284075
}
40294076
} else {
40304077
CloneChainReader reader(instance_id, resource_mgr);
@@ -4035,22 +4082,22 @@ void _write_delete_bitmap_kvs(MetaServiceCode& code, std::string& msg, std::stri
40354082
<< " version=" << request->pre_rowset_versions(i);
40364083
msg = ss.str();
40374084
code = cast_as<ErrCategory::READ>(err);
4038-
return;
4085+
return false;
40394086
}
40404087
}
40414088
if (err == TxnErrorCode::TXN_KEY_NOT_FOUND) {
40424089
non_exist_rowset_ids.emplace(request->rowset_ids(i));
40434090
LOG(INFO) << "skip update delete bitmap, rowset_id=" << request->rowset_ids(i)
40444091
<< " version=" << request->pre_rowset_versions(i)
40454092
<< " tablet_id=" << tablet_id << " because the rowset is not exist";
4046-
return;
4093+
return true;
40474094
}
40484095
if (rs.rowset_id_v2() != request->rowset_ids(i)) {
40494096
LOG(INFO) << "skip update delete bitmap, rowset_id=" << request->rowset_ids(i)
40504097
<< " version=" << request->pre_rowset_versions(i)
40514098
<< " tablet_id=" << tablet_id << " because the rowset is not exist";
40524099
non_exist_rowset_ids.emplace(request->rowset_ids(i));
4053-
return;
4100+
return true;
40544101
}
40554102
}
40564103
// remove first
@@ -4060,7 +4107,7 @@ void _write_delete_bitmap_kvs(MetaServiceCode& code, std::string& msg, std::stri
40604107
std::optional<uint16_t> max_blob_sequence;
40614108
if (!check_delete_bitmap_point_delete(code, msg, txn, key, request, i, point_delete,
40624109
delete_bitmap_exists, max_blob_sequence)) {
4063-
return;
4110+
return false;
40644111
}
40654112
if (point_delete) {
40664113
if (delete_bitmap_exists) {
@@ -4096,12 +4143,15 @@ void _write_delete_bitmap_kvs(MetaServiceCode& code, std::string& msg, std::stri
40964143
VLOG_DEBUG << "xxx update delete bitmap put delete_bitmap_key=" << hex(key)
40974144
<< " lock_id=" << request->lock_id() << " initiator=" << request->initiator()
40984145
<< " key_size: " << key.size() << " value_size: " << val.size();
4146+
return true;
40994147
}
41004148

41014149
static bool commit_pre_rowset_delete_bitmap_removal(
41024150
MetaServiceCode& code, std::string& msg, std::stringstream& ss,
41034151
const std::shared_ptr<TxnKv>& txn_kv, std::unique_ptr<Transaction>& txn, KVStats& stats,
4104-
UpdateDeleteBitmapTxnStats& txn_stats, int64_t tablet_id, const std::string& rowset_id) {
4152+
UpdateDeleteBitmapTxnStats& txn_stats, const UpdateDeleteBitmapRequest* request,
4153+
const std::string& instance_id, const std::string& rowset_id) {
4154+
auto tablet_id = request->tablet_id();
41054155
auto txn_size = txn->approximate_bytes();
41064156
LOG(INFO) << "commit delete bitmap point deletes before transaction size exceeds limit, "
41074157
"tablet_id="
@@ -4134,7 +4184,7 @@ static bool commit_pre_rowset_delete_bitmap_removal(
41344184
msg = "failed to init txn when removing pre rowsets delete bitmap";
41354185
return false;
41364186
}
4137-
return true;
4187+
return check_pre_rowset_aggregation(code, msg, txn.get(), request, instance_id);
41384188
}
41394189

41404190
static bool remove_pre_rowset_delete_bitmap(
@@ -4198,7 +4248,7 @@ static bool remove_pre_rowset_delete_bitmap(
41984248
if (txn_size > 0 &&
41994249
txn_size + key.size() * 3 > static_cast<size_t>(config::max_txn_commit_byte) &&
42004250
!commit_pre_rowset_delete_bitmap_removal(code, msg, ss, txn_kv, txn, stats,
4201-
txn_stats, tablet_id,
4251+
txn_stats, request, instance_id,
42024252
rowset_stats.rowset_id())) {
42034253
return false;
42044254
}
@@ -4298,6 +4348,10 @@ void MetaServiceImpl::update_delete_bitmap(google::protobuf::RpcController* cont
42984348
return;
42994349
}
43004350

4351+
if (!check_pre_rowset_aggregation(code, msg, txn.get(), request, instance_id)) {
4352+
return;
4353+
}
4354+
43014355
bool is_explicit_txn = (request->has_is_explicit_txn() && request->is_explicit_txn());
43024356
bool is_first_sub_txn = (is_explicit_txn && request->txn_id() == request->lock_id());
43034357
std::string log = ", update delete bitmap for tablet " + std::to_string(tablet_id);
@@ -4424,10 +4478,13 @@ void MetaServiceImpl::update_delete_bitmap(google::protobuf::RpcController* cont
44244478
auto& key = delete_bitmap_keys_v1[i];
44254479
auto& val = request->segment_delete_bitmaps(i);
44264480
bool is_versioned_read = is_version_read_enabled(instance_id);
4427-
_write_delete_bitmap_kvs(code, msg, ss, txn_kv_, txn, use_version, non_exist_rowset_ids, i,
4428-
log, stats, request, instance_id, key, val, txn_stats,
4429-
is_versioned_read, resource_mgr_.get());
4430-
if (code != MetaServiceCode::OK) return;
4481+
if (!_write_delete_bitmap_kvs(code, msg, ss, txn_kv_, txn, use_version,
4482+
non_exist_rowset_ids, i, log, stats, request, instance_id,
4483+
key, val, txn_stats, is_versioned_read,
4484+
resource_mgr_.get()) ||
4485+
code != MetaServiceCode::OK) {
4486+
return;
4487+
}
44314488
}
44324489
// write v2 kvs
44334490
for (size_t i = 0; i < delete_bitmap_keys_v2.size(); ++i) {
@@ -4443,10 +4500,13 @@ void MetaServiceImpl::update_delete_bitmap(google::protobuf::RpcController* cont
44434500
<< request->delete_bitmap_storages(i).delete_bitmap().rowset_ids_size()
44444501
<< ". key=" << hex(key) << ", value_size=" << val.size();
44454502
bool is_versioned_read = is_version_read_enabled(instance_id);
4446-
_write_delete_bitmap_kvs(code, msg, ss, txn_kv_, txn, use_version, non_exist_rowset_ids, i,
4447-
log, stats, request, instance_id, key, val, txn_stats,
4448-
is_versioned_read, resource_mgr_.get());
4449-
if (code != MetaServiceCode::OK) return;
4503+
if (!_write_delete_bitmap_kvs(code, msg, ss, txn_kv_, txn, use_version,
4504+
non_exist_rowset_ids, i, log, stats, request, instance_id,
4505+
key, val, txn_stats, is_versioned_read,
4506+
resource_mgr_.get()) ||
4507+
code != MetaServiceCode::OK) {
4508+
return;
4509+
}
44504510
}
44514511

44524512
if (!remove_pre_rowset_delete_bitmap(code, msg, ss, txn_kv_, txn, stats, request, instance_id,

0 commit comments

Comments
 (0)