Skip to content

Commit 1727203

Browse files
committed
[refactor](cloud) Add a dedicated RPC for TSO recovery
### What problem does this PR solve? Issue Number: None Related PR: #67820 Problem Summary: TSO recovery had added strict-check and batch modes to the existing transaction-conflict RPC, mixing recovery scans with table-scoped conflict checks. Add get_tso_recovery_transactions with its own request, response, client wrappers, handler and metrics. Restore check_txn_conflict and its messages to their original definitions. Preserve fixed-bound batch scanning, table-scoped waits and committed-TSO advancement. An unavailable new RPC or incomplete batch keeps recovery closed without falling back to the old conflict check. ### Release note Upgrade MetaService before FE to provide the dedicated TSO recovery RPC. Incremental-query waiting behavior and error codes remain unchanged. ### Check List (For Author) - Test: 108 FE unit tests; 9 ASAN MetaService/recovery/legacy-conflict/lazy-commit tests; test_committed_tso regression; live old/new RPC and three-FE failover verification; FE/MS product builds, FE Checkstyle, clang-format 16 and Cloud clang-tidy. - Behavior changed: Yes, the internal recovery RPC changes; SQL behavior is unchanged. - Does this need documentation: No
1 parent 8eaf6b3 commit 1727203

14 files changed

Lines changed: 284 additions & 229 deletions

File tree

‎cloud/src/common/bvars.cpp‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,7 @@ BvarLatencyRecorderWithTag g_bvar_ms_create_meta_sync_point("ms", "create_meta_s
4343
BvarLatencyRecorderWithTag g_bvar_ms_begin_sub_txn("ms", "begin_sub_txn");
4444
BvarLatencyRecorderWithTag g_bvar_ms_abort_sub_txn("ms", "abort_sub_txn");
4545
BvarLatencyRecorderWithTag g_bvar_ms_check_txn_conflict("ms", "check_txn_conflict");
46+
BvarLatencyRecorderWithTag g_bvar_ms_get_tso_recovery_transactions("ms", "get_tso_recovery_transactions");
4647
BvarLatencyRecorderWithTag g_bvar_ms_abort_txn_with_coordinator("ms", "abort_txn_with_coordinator");
4748
BvarLatencyRecorderWithTag g_bvar_ms_get_prepare_txn_by_coordinator("ms", "get_prepare_txn_by_coordinator");
4849
BvarLatencyRecorderWithTag g_bvar_ms_clean_txn_label("ms", "clean_txn_label");
@@ -500,6 +501,7 @@ mBvarInt64Adder g_bvar_rpc_kv_abort_txn_with_coordinator_get_counter("rpc_kv_abo
500501
mBvarInt64Adder g_bvar_rpc_kv_get_prepare_txn_by_coordinator_get_counter("rpc_kv_get_prepare_txn_by_coordinator_get_counter",{"instance_id"});
501502
// check_txn_conflict
502503
mBvarInt64Adder g_bvar_rpc_kv_check_txn_conflict_get_counter("rpc_kv_check_txn_conflict_get_counter",{"instance_id"});
504+
mBvarInt64Adder g_bvar_rpc_kv_get_tso_recovery_transactions_get_counter("rpc_kv_get_tso_recovery_transactions_get_counter",{"instance_id"});
503505
// clean_txn_label
504506
mBvarInt64Adder g_bvar_rpc_kv_clean_txn_label_get_counter("rpc_kv_clean_txn_label_get_counter",{"instance_id"});
505507
mBvarInt64Adder g_bvar_rpc_kv_clean_txn_label_put_counter("rpc_kv_clean_txn_label_put_counter",{"instance_id"});
@@ -710,6 +712,7 @@ mBvarInt64Adder g_bvar_rpc_kv_abort_txn_with_coordinator_get_bytes("rpc_kv_abort
710712
mBvarInt64Adder g_bvar_rpc_kv_get_prepare_txn_by_coordinator_get_bytes("rpc_kv_get_prepare_txn_by_coordinator_get_bytes",{"instance_id"});
711713
// check_txn_conflict
712714
mBvarInt64Adder g_bvar_rpc_kv_check_txn_conflict_get_bytes("rpc_kv_check_txn_conflict_get_bytes",{"instance_id"});
715+
mBvarInt64Adder g_bvar_rpc_kv_get_tso_recovery_transactions_get_bytes("rpc_kv_get_tso_recovery_transactions_get_bytes",{"instance_id"});
713716
// clean_txn_label
714717
mBvarInt64Adder g_bvar_rpc_kv_clean_txn_label_get_bytes("rpc_kv_clean_txn_label_get_bytes",{"instance_id"});
715718
mBvarInt64Adder g_bvar_rpc_kv_clean_txn_label_put_bytes("rpc_kv_clean_txn_label_put_bytes",{"instance_id"});

‎cloud/src/common/bvars.h‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -553,6 +553,7 @@ extern BvarLatencyRecorderWithTag g_bvar_ms_get_txn;
553553
extern BvarLatencyRecorderWithTag g_bvar_ms_get_current_max_txn_id;
554554
extern BvarLatencyRecorderWithTag g_bvar_ms_create_meta_sync_point;
555555
extern BvarLatencyRecorderWithTag g_bvar_ms_check_txn_conflict;
556+
extern BvarLatencyRecorderWithTag g_bvar_ms_get_tso_recovery_transactions;
556557
extern BvarLatencyRecorderWithTag g_bvar_ms_abort_txn_with_coordinator;
557558
extern BvarLatencyRecorderWithTag g_bvar_ms_get_prepare_txn_by_coordinator;
558559
extern BvarLatencyRecorderWithTag g_bvar_ms_begin_sub_txn;
@@ -907,6 +908,7 @@ extern mBvarInt64Adder g_bvar_rpc_kv_abort_sub_txn_put_counter;
907908
extern mBvarInt64Adder g_bvar_rpc_kv_abort_txn_with_coordinator_get_counter;
908909
extern mBvarInt64Adder g_bvar_rpc_kv_get_prepare_txn_by_coordinator_get_counter;
909910
extern mBvarInt64Adder g_bvar_rpc_kv_check_txn_conflict_get_counter;
911+
extern mBvarInt64Adder g_bvar_rpc_kv_get_tso_recovery_transactions_get_counter;
910912
extern mBvarInt64Adder g_bvar_rpc_kv_clean_txn_label_get_counter;
911913
extern mBvarInt64Adder g_bvar_rpc_kv_clean_txn_label_put_counter;
912914
extern mBvarInt64Adder g_bvar_rpc_kv_clean_txn_label_del_counter;
@@ -1050,6 +1052,7 @@ extern mBvarInt64Adder g_bvar_rpc_kv_abort_sub_txn_put_bytes;
10501052
extern mBvarInt64Adder g_bvar_rpc_kv_abort_txn_with_coordinator_get_bytes;
10511053
extern mBvarInt64Adder g_bvar_rpc_kv_get_prepare_txn_by_coordinator_get_bytes;
10521054
extern mBvarInt64Adder g_bvar_rpc_kv_check_txn_conflict_get_bytes;
1055+
extern mBvarInt64Adder g_bvar_rpc_kv_get_tso_recovery_transactions_get_bytes;
10531056
extern mBvarInt64Adder g_bvar_rpc_kv_clean_txn_label_get_bytes;
10541057
extern mBvarInt64Adder g_bvar_rpc_kv_clean_txn_label_put_bytes;
10551058
extern mBvarInt64Adder g_bvar_rpc_kv_clean_txn_label_del_bytes;

‎cloud/src/meta-service/meta_service.h‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -145,6 +145,11 @@ class MetaServiceImpl : public cloud::MetaService {
145145
CheckTxnConflictResponse* response,
146146
::google::protobuf::Closure* done) override;
147147

148+
void get_tso_recovery_transactions(::google::protobuf::RpcController* controller,
149+
const GetTsoRecoveryTransactionsRequest* request,
150+
GetTsoRecoveryTransactionsResponse* response,
151+
::google::protobuf::Closure* done) override;
152+
148153
void abort_txn_with_coordinator(::google::protobuf::RpcController* controller,
149154
const AbortTxnWithCoordinatorRequest* request,
150155
AbortTxnWithCoordinatorResponse* response,
@@ -619,6 +624,14 @@ class MetaServiceProxy final : public MetaService {
619624
call_impl(&cloud::MetaService::check_txn_conflict, controller, request, response, done);
620625
}
621626

627+
void get_tso_recovery_transactions(::google::protobuf::RpcController* controller,
628+
const GetTsoRecoveryTransactionsRequest* request,
629+
GetTsoRecoveryTransactionsResponse* response,
630+
::google::protobuf::Closure* done) override {
631+
call_impl(&cloud::MetaService::get_tso_recovery_transactions, controller, request, response,
632+
done);
633+
}
634+
622635
void abort_txn_with_coordinator(::google::protobuf::RpcController* controller,
623636
const AbortTxnWithCoordinatorRequest* request,
624637
AbortTxnWithCoordinatorResponse* response,

‎cloud/src/meta-service/meta_service_txn.cpp‎

Lines changed: 101 additions & 90 deletions
Original file line numberDiff line numberDiff line change
@@ -4724,17 +4724,111 @@ std::string get_txn_info_key_from_txn_running_key(std::string_view txn_running_k
47244724
return conflict_txn_info_key;
47254725
}
47264726

4727+
void MetaServiceImpl::get_tso_recovery_transactions(
4728+
::google::protobuf::RpcController* controller,
4729+
const GetTsoRecoveryTransactionsRequest* request,
4730+
GetTsoRecoveryTransactionsResponse* response, ::google::protobuf::Closure* done) {
4731+
RPC_PREPROCESS(get_tso_recovery_transactions, get);
4732+
if (request->end_txn_id() <= 0 || request->batch_size() <= 0 || request->batch_size() > 1000) {
4733+
code = MetaServiceCode::INVALID_ARGUMENT;
4734+
msg = "invalid TSO recovery transaction bound or batch size";
4735+
return;
4736+
}
4737+
instance_id = get_instance_id(resource_mgr_, request->cloud_unique_id());
4738+
if (instance_id.empty()) {
4739+
code = MetaServiceCode::INVALID_ARGUMENT;
4740+
msg = "cannot find instance_id for TSO recovery";
4741+
return;
4742+
}
4743+
RPC_RATE_LIMIT(get_tso_recovery_transactions)
4744+
// Keys sort by database first. Scan the instance and apply the fixed exclusive ID bound
4745+
// to each key; a batch containing only newer transactions does not finish the scan.
4746+
std::string begin_key = txn_running_key({instance_id, 0, 0});
4747+
std::string end_key = txn_running_key({instance_id, INT64_MAX, INT64_MAX});
4748+
end_key.push_back('\x00');
4749+
if (!request->start_key().empty()) {
4750+
if (request->start_key() < begin_key || request->start_key() >= end_key) {
4751+
code = MetaServiceCode::INVALID_ARGUMENT;
4752+
msg = "invalid TSO recovery start key";
4753+
return;
4754+
}
4755+
begin_key = request->start_key();
4756+
}
4757+
TxnErrorCode err = txn_kv_->create_txn(&txn);
4758+
if (err != TxnErrorCode::TXN_OK) {
4759+
code = cast_as<ErrCategory::CREATE>(err);
4760+
msg = "failed to create TSO recovery read transaction";
4761+
return;
4762+
}
4763+
std::unique_ptr<RangeGetIterator> it;
4764+
err = txn->get(begin_key, end_key, &it, true, request->batch_size());
4765+
if (err != TxnErrorCode::TXN_OK) {
4766+
code = cast_as<ErrCategory::READ>(err);
4767+
msg = "failed to get running transactions during TSO recovery";
4768+
return;
4769+
}
4770+
while (it->has_next()) {
4771+
auto [key, value] = it->next();
4772+
if (!it->has_next()) {
4773+
begin_key = key;
4774+
}
4775+
std::string_view encoded_key = key;
4776+
encoded_key.remove_prefix(1);
4777+
std::vector<std::tuple<std::variant<int64_t, std::string>, int, int>> fields;
4778+
if (decode_key(&encoded_key, &fields) != 0 || fields.size() != 5 ||
4779+
!std::holds_alternative<int64_t>(std::get<0>(fields[3])) ||
4780+
!std::holds_alternative<int64_t>(std::get<0>(fields[4]))) {
4781+
code = MetaServiceCode::UNDEFINED_ERR;
4782+
msg = "failed to decode running transaction key during TSO recovery";
4783+
return;
4784+
}
4785+
const auto db_id = std::get<int64_t>(std::get<0>(fields[3]));
4786+
const auto txn_id = std::get<int64_t>(std::get<0>(fields[4]));
4787+
if (txn_id >= request->end_txn_id()) {
4788+
continue;
4789+
}
4790+
// Running keys are removed atomically with real VISIBLE/ABORTED. Expired COMMITTED
4791+
// lazy transactions still block. Read the details in the same KV snapshot.
4792+
std::string info_val;
4793+
err = txn->get(txn_info_key({instance_id, db_id, txn_id}), &info_val, true);
4794+
if (err != TxnErrorCode::TXN_OK) {
4795+
code = cast_as<ErrCategory::READ>(err);
4796+
msg = fmt::format("failed to read TSO recovery transaction, db_id={}, txn_id={}", db_id,
4797+
txn_id);
4798+
return;
4799+
}
4800+
TxnInfoPB info;
4801+
if (!info.ParseFromString(info_val)) {
4802+
code = MetaServiceCode::PROTOBUF_PARSE_ERR;
4803+
msg = "failed to parse TSO recovery transaction";
4804+
return;
4805+
}
4806+
// Subtransactions update txn_info.table_ids; the running record can be older.
4807+
if (info.db_id() != db_id || info.txn_id() != txn_id || info.table_ids().empty()) {
4808+
code = MetaServiceCode::UNDEFINED_ERR;
4809+
msg = "invalid TSO recovery transaction identity or tables";
4810+
return;
4811+
}
4812+
// Recovery needs identities and visibility boundaries, not commit attachments.
4813+
auto* recovered = response->add_txn_infos();
4814+
recovered->set_db_id(db_id);
4815+
recovered->set_txn_id(txn_id);
4816+
recovered->mutable_table_ids()->CopyFrom(info.table_ids());
4817+
recovered->set_status(info.status());
4818+
if (info.has_commit_tso()) {
4819+
recovered->set_commit_tso(info.commit_tso());
4820+
}
4821+
}
4822+
begin_key.push_back('\x00');
4823+
response->set_next_start_key(it->more() ? begin_key : "");
4824+
}
4825+
47274826
void MetaServiceImpl::check_txn_conflict(::google::protobuf::RpcController* controller,
47284827
const CheckTxnConflictRequest* request,
47294828
CheckTxnConflictResponse* response,
47304829
::google::protobuf::Closure* done) {
47314830
RPC_PREPROCESS(check_txn_conflict, get);
4732-
const bool strict_recovery = request->strict_recovery_check();
4733-
const bool recovery_batch = request->has_recovery_batch_size();
4734-
if (!request->has_end_txn_id() || (strict_recovery && request->end_txn_id() <= 0) ||
4735-
(!strict_recovery && (!request->has_db_id() || request->table_ids_size() <= 0)) ||
4736-
(recovery_batch && (!strict_recovery || request->recovery_batch_size() <= 0 ||
4737-
request->recovery_batch_size() > 1000))) {
4831+
if (!request->has_db_id() || !request->has_end_txn_id() || (request->table_ids_size() <= 0)) {
47384832
code = MetaServiceCode::INVALID_ARGUMENT;
47394833
msg = "invalid db id, end txn id or table_ids.";
47404834
return;
@@ -4754,26 +4848,6 @@ void MetaServiceImpl::check_txn_conflict(::google::protobuf::RpcController* cont
47544848

47554849
std::string begin_running_key = txn_running_key({instance_id, db_id, 0});
47564850
std::string end_running_key = txn_running_key({instance_id, db_id, request->end_txn_id()});
4757-
if (strict_recovery) {
4758-
// Database and transaction IDs are non-negative. Include the entire instance and apply
4759-
// the exclusive transaction bound after decoding each key (keys sort by database first).
4760-
begin_running_key = txn_running_key({instance_id, 0, 0});
4761-
end_running_key = txn_running_key({instance_id, INT64_MAX, INT64_MAX});
4762-
end_running_key.push_back('\x00');
4763-
response->set_strict_recovery_check_applied(true);
4764-
}
4765-
if (recovery_batch) {
4766-
if (!request->recovery_start_key().empty()) {
4767-
if (request->recovery_start_key() < begin_running_key ||
4768-
request->recovery_start_key() >= end_running_key) {
4769-
code = MetaServiceCode::INVALID_ARGUMENT;
4770-
msg = "invalid TSO recovery start key";
4771-
return;
4772-
}
4773-
begin_running_key = request->recovery_start_key();
4774-
}
4775-
response->set_recovery_batch_applied(true);
4776-
}
47774851
LOG(INFO) << "begin_running_key:" << hex(begin_running_key)
47784852
<< " end_running_key:" << hex(end_running_key);
47794853

@@ -4792,8 +4866,7 @@ void MetaServiceImpl::check_txn_conflict(::google::protobuf::RpcController* cont
47924866
int total_iteration_cnt = 0;
47934867
bool finished = true;
47944868
while (it == nullptr /* may be not init */ || it->more()) {
4795-
err = txn->get(begin_running_key, end_running_key, &it, true,
4796-
recovery_batch ? request->recovery_batch_size() : 10000);
4869+
err = txn->get(begin_running_key, end_running_key, &it, true);
47974870
if (err != TxnErrorCode::TXN_OK) {
47984871
code = cast_as<ErrCategory::READ>(err);
47994872
ss << "failed to get txn running info. err=" << err;
@@ -4811,64 +4884,6 @@ void MetaServiceImpl::check_txn_conflict(::google::protobuf::RpcController* cont
48114884
while (it->has_next()) {
48124885
total_iteration_cnt++;
48134886
auto [k, v] = it->next();
4814-
if (strict_recovery) {
4815-
std::string_view encoded_key = k;
4816-
encoded_key.remove_prefix(1);
4817-
std::vector<std::tuple<std::variant<int64_t, std::string>, int, int>> fields;
4818-
if (decode_key(&encoded_key, &fields) != 0 || fields.size() != 5 ||
4819-
!std::holds_alternative<int64_t>(std::get<0>(fields[3])) ||
4820-
!std::holds_alternative<int64_t>(std::get<0>(fields[4]))) {
4821-
code = MetaServiceCode::UNDEFINED_ERR;
4822-
msg = "failed to decode running transaction key during TSO recovery";
4823-
return;
4824-
}
4825-
if (std::get<int64_t>(std::get<0>(fields[4])) < request->end_txn_id()) {
4826-
// A running key is removed atomically with real VISIBLE/ABORTED. In
4827-
// particular an expired COMMITTED lazy transaction must still block.
4828-
if (!recovery_batch) {
4829-
response->set_finished(false);
4830-
return;
4831-
}
4832-
const auto running_db_id = std::get<int64_t>(std::get<0>(fields[3]));
4833-
const auto running_txn_id = std::get<int64_t>(std::get<0>(fields[4]));
4834-
std::string info_val;
4835-
err = txn->get(txn_info_key({instance_id, running_db_id, running_txn_id}),
4836-
&info_val, true);
4837-
if (err != TxnErrorCode::TXN_OK) {
4838-
code = cast_as<ErrCategory::READ>(err);
4839-
msg = fmt::format(
4840-
"failed to read TSO recovery transaction, db_id={}, txn_id={}",
4841-
running_db_id, running_txn_id);
4842-
return;
4843-
}
4844-
TxnInfoPB info;
4845-
if (!info.ParseFromString(info_val)) {
4846-
code = MetaServiceCode::PROTOBUF_PARSE_ERR;
4847-
msg = "failed to parse TSO recovery transaction";
4848-
return;
4849-
}
4850-
// Subtransactions update txn_info.table_ids; the running record can be older.
4851-
if (info.db_id() != running_db_id || info.txn_id() != running_txn_id ||
4852-
info.table_ids().empty()) {
4853-
code = MetaServiceCode::UNDEFINED_ERR;
4854-
msg = "invalid TSO recovery transaction identity or tables";
4855-
return;
4856-
}
4857-
// Recovery needs identities and visibility boundaries, not commit attachments.
4858-
auto* recovered = response->add_conflict_txns();
4859-
recovered->set_db_id(running_db_id);
4860-
recovered->set_txn_id(running_txn_id);
4861-
recovered->mutable_table_ids()->CopyFrom(info.table_ids());
4862-
recovered->set_status(info.status());
4863-
if (info.has_commit_tso()) {
4864-
recovered->set_commit_tso(info.commit_tso());
4865-
}
4866-
}
4867-
if (!it->has_next()) {
4868-
begin_running_key = k;
4869-
}
4870-
continue;
4871-
}
48724887
LOG(INFO) << "check watermark conflict range_get txn_run_key=" << hex(k);
48734888
TxnRunningPB running_pb;
48744889
if (!running_pb.ParseFromArray(v.data(), v.size())) {
@@ -4928,10 +4943,6 @@ void MetaServiceImpl::check_txn_conflict(::google::protobuf::RpcController* cont
49284943
}
49294944
}
49304945
begin_running_key.push_back('\x00'); // Update to next smallest key for iteration
4931-
if (recovery_batch) {
4932-
response->set_next_recovery_key(it->more() ? begin_running_key : "");
4933-
return;
4934-
}
49354946
}
49364947
LOG(INFO) << "skip timeout txn count: " << skip_timeout_txn_cnt
49374948
<< " conflict txn count: " << response->conflict_txns_size()

0 commit comments

Comments
 (0)