Skip to content

Commit e53291d

Browse files
committed
[fix](binlog) Fence uncertain commit TSO attempts
### What problem does this PR solve? Issue Number: None Related PR: apache#67181, apache#67594 Problem Summary: Incremental reads track TSO-bearing commit attempts in FE memory. A commit RPC can finish without a conclusive response, leaving the attempt in the tracker indefinitely, while scanning Meta Service transactions during startup and periodic reconciliation adds complexity and can block unrelated reads. Fence an uncertain attempt in Meta Service before removing its exact TSO from the tracker, release definite responses immediately, and let incremental scans wait for pending lazy commits on their selected partitions. Disable transport retries for TSO-bearing commit RPCs so every uncertain delivery is fenced. Startup calibration advances the same per-instance fence, so stale commits from an old master are rejected without a recovery scan. ### Release note Bounded incremental reads now fence uncertain commit TSO attempts and no longer require startup or periodic transaction recovery scans. ### Check List (For Author) - Test: Unit Test - FE unit tests for TSO tracking, commit response handling, Meta Service retry behavior, and incremental scan version waits - Cloud unit tests for TSO fence monotonicity, fence-key encoding, and response-code compatibility - ./build.sh -j32 - ./build.sh --cloud -j32 - Cloud clang-tidy and build hygiene checks - Behavior changed: Yes. Uncertain TSO commit attempts are fenced before release, definite failures are released directly, and incremental scans wait for selected-partition pending lazy commits. - Does this need documentation: Yes. Connector-facing error-code documentation is included in the PR description.
1 parent 64d601f commit e53291d

22 files changed

Lines changed: 268 additions & 1228 deletions

‎cloud/src/common/bvars.cpp‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,6 @@ BvarLatencyRecorderWithTag g_bvar_ms_create_meta_sync_point("ms", "create_meta_s
4343
BvarLatencyRecorderWithTag g_bvar_ms_begin_sub_txn("ms", "begin_sub_txn");
4444
BvarLatencyRecorderWithTag g_bvar_ms_abort_sub_txn("ms", "abort_sub_txn");
4545
BvarLatencyRecorderWithTag g_bvar_ms_check_txn_conflict("ms", "check_txn_conflict");
46-
BvarLatencyRecorderWithTag g_bvar_ms_get_tso_recovery_transactions("ms", "get_tso_recovery_transactions");
4746
BvarLatencyRecorderWithTag g_bvar_ms_advance_tso_fence("ms", "advance_tso_fence");
4847
BvarLatencyRecorderWithTag g_bvar_ms_abort_txn_with_coordinator("ms", "abort_txn_with_coordinator");
4948
BvarLatencyRecorderWithTag g_bvar_ms_get_prepare_txn_by_coordinator("ms", "get_prepare_txn_by_coordinator");
@@ -502,7 +501,6 @@ mBvarInt64Adder g_bvar_rpc_kv_abort_txn_with_coordinator_get_counter("rpc_kv_abo
502501
mBvarInt64Adder g_bvar_rpc_kv_get_prepare_txn_by_coordinator_get_counter("rpc_kv_get_prepare_txn_by_coordinator_get_counter",{"instance_id"});
503502
// check_txn_conflict
504503
mBvarInt64Adder g_bvar_rpc_kv_check_txn_conflict_get_counter("rpc_kv_check_txn_conflict_get_counter",{"instance_id"});
505-
mBvarInt64Adder g_bvar_rpc_kv_get_tso_recovery_transactions_get_counter("rpc_kv_get_tso_recovery_transactions_get_counter",{"instance_id"});
506504
mBvarInt64Adder g_bvar_rpc_kv_advance_tso_fence_get_counter("rpc_kv_advance_tso_fence_get_counter",{"instance_id"});
507505
mBvarInt64Adder g_bvar_rpc_kv_advance_tso_fence_put_counter("rpc_kv_advance_tso_fence_put_counter",{"instance_id"});
508506
// clean_txn_label
@@ -715,7 +713,6 @@ mBvarInt64Adder g_bvar_rpc_kv_abort_txn_with_coordinator_get_bytes("rpc_kv_abort
715713
mBvarInt64Adder g_bvar_rpc_kv_get_prepare_txn_by_coordinator_get_bytes("rpc_kv_get_prepare_txn_by_coordinator_get_bytes",{"instance_id"});
716714
// check_txn_conflict
717715
mBvarInt64Adder g_bvar_rpc_kv_check_txn_conflict_get_bytes("rpc_kv_check_txn_conflict_get_bytes",{"instance_id"});
718-
mBvarInt64Adder g_bvar_rpc_kv_get_tso_recovery_transactions_get_bytes("rpc_kv_get_tso_recovery_transactions_get_bytes",{"instance_id"});
719716
mBvarInt64Adder g_bvar_rpc_kv_advance_tso_fence_get_bytes("rpc_kv_advance_tso_fence_get_bytes",{"instance_id"});
720717
mBvarInt64Adder g_bvar_rpc_kv_advance_tso_fence_put_bytes("rpc_kv_advance_tso_fence_put_bytes",{"instance_id"});
721718
// clean_txn_label

‎cloud/src/common/bvars.h‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -553,7 +553,6 @@ extern BvarLatencyRecorderWithTag g_bvar_ms_get_txn;
553553
extern BvarLatencyRecorderWithTag g_bvar_ms_get_current_max_txn_id;
554554
extern BvarLatencyRecorderWithTag g_bvar_ms_create_meta_sync_point;
555555
extern BvarLatencyRecorderWithTag g_bvar_ms_check_txn_conflict;
556-
extern BvarLatencyRecorderWithTag g_bvar_ms_get_tso_recovery_transactions;
557556
extern BvarLatencyRecorderWithTag g_bvar_ms_advance_tso_fence;
558557
extern BvarLatencyRecorderWithTag g_bvar_ms_abort_txn_with_coordinator;
559558
extern BvarLatencyRecorderWithTag g_bvar_ms_get_prepare_txn_by_coordinator;
@@ -909,7 +908,6 @@ extern mBvarInt64Adder g_bvar_rpc_kv_abort_sub_txn_put_counter;
909908
extern mBvarInt64Adder g_bvar_rpc_kv_abort_txn_with_coordinator_get_counter;
910909
extern mBvarInt64Adder g_bvar_rpc_kv_get_prepare_txn_by_coordinator_get_counter;
911910
extern mBvarInt64Adder g_bvar_rpc_kv_check_txn_conflict_get_counter;
912-
extern mBvarInt64Adder g_bvar_rpc_kv_get_tso_recovery_transactions_get_counter;
913911
extern mBvarInt64Adder g_bvar_rpc_kv_advance_tso_fence_get_counter;
914912
extern mBvarInt64Adder g_bvar_rpc_kv_advance_tso_fence_put_counter;
915913
extern mBvarInt64Adder g_bvar_rpc_kv_clean_txn_label_get_counter;
@@ -1055,7 +1053,6 @@ extern mBvarInt64Adder g_bvar_rpc_kv_abort_sub_txn_put_bytes;
10551053
extern mBvarInt64Adder g_bvar_rpc_kv_abort_txn_with_coordinator_get_bytes;
10561054
extern mBvarInt64Adder g_bvar_rpc_kv_get_prepare_txn_by_coordinator_get_bytes;
10571055
extern mBvarInt64Adder g_bvar_rpc_kv_check_txn_conflict_get_bytes;
1058-
extern mBvarInt64Adder g_bvar_rpc_kv_get_tso_recovery_transactions_get_bytes;
10591056
extern mBvarInt64Adder g_bvar_rpc_kv_advance_tso_fence_get_bytes;
10601057
extern mBvarInt64Adder g_bvar_rpc_kv_advance_tso_fence_put_bytes;
10611058
extern mBvarInt64Adder g_bvar_rpc_kv_clean_txn_label_get_bytes;

‎cloud/src/meta-service/meta_service.h‎

Lines changed: 0 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -145,11 +145,6 @@ class MetaServiceImpl : public cloud::MetaService {
145145
CheckTxnConflictResponse* response,
146146
::google::protobuf::Closure* done) override;
147147

148-
void get_tso_recovery_transactions(::google::protobuf::RpcController* controller,
149-
const GetTsoRecoveryTransactionsRequest* request,
150-
GetTsoRecoveryTransactionsResponse* response,
151-
::google::protobuf::Closure* done) override;
152-
153148
void advance_tso_fence(::google::protobuf::RpcController* controller,
154149
const AdvanceTsoFenceRequest* request, AdvanceTsoFenceResponse* response,
155150
::google::protobuf::Closure* done) override;
@@ -628,14 +623,6 @@ class MetaServiceProxy final : public MetaService {
628623
call_impl(&cloud::MetaService::check_txn_conflict, controller, request, response, done);
629624
}
630625

631-
void get_tso_recovery_transactions(::google::protobuf::RpcController* controller,
632-
const GetTsoRecoveryTransactionsRequest* request,
633-
GetTsoRecoveryTransactionsResponse* response,
634-
::google::protobuf::Closure* done) override {
635-
call_impl(&cloud::MetaService::get_tso_recovery_transactions, controller, request, response,
636-
done);
637-
}
638-
639626
void advance_tso_fence(::google::protobuf::RpcController* controller,
640627
const AdvanceTsoFenceRequest* request, AdvanceTsoFenceResponse* response,
641628
::google::protobuf::Closure* done) override {

‎cloud/src/meta-service/meta_service_txn.cpp‎

Lines changed: 0 additions & 102 deletions
Original file line numberDiff line numberDiff line change
@@ -4835,108 +4835,6 @@ void MetaServiceImpl::advance_tso_fence(::google::protobuf::RpcController* contr
48354835
response->set_tso_fence(effective_fence_tso);
48364836
}
48374837

4838-
void MetaServiceImpl::get_tso_recovery_transactions(
4839-
::google::protobuf::RpcController* controller,
4840-
const GetTsoRecoveryTransactionsRequest* request,
4841-
GetTsoRecoveryTransactionsResponse* response, ::google::protobuf::Closure* done) {
4842-
RPC_PREPROCESS(get_tso_recovery_transactions, get);
4843-
if (request->end_txn_id() <= 0 || request->batch_size() <= 0 || request->batch_size() > 1000 ||
4844-
request->tso_fence() <= 0) {
4845-
code = MetaServiceCode::INVALID_ARGUMENT;
4846-
msg = "invalid TSO recovery transaction bound or batch size";
4847-
return;
4848-
}
4849-
instance_id = get_instance_id(resource_mgr_, request->cloud_unique_id());
4850-
if (instance_id.empty()) {
4851-
code = MetaServiceCode::INVALID_ARGUMENT;
4852-
msg = "cannot find instance_id for TSO recovery";
4853-
return;
4854-
}
4855-
RPC_RATE_LIMIT(get_tso_recovery_transactions)
4856-
// Keys sort by database first. Scan the instance and apply the fixed exclusive ID bound
4857-
// to each key; a batch containing only newer transactions does not finish the scan.
4858-
std::string begin_key = txn_running_key({instance_id, 0, 0});
4859-
std::string end_key = txn_running_key({instance_id, INT64_MAX, INT64_MAX});
4860-
end_key.push_back('\x00');
4861-
if (!request->start_key().empty()) {
4862-
if (request->start_key() < begin_key || request->start_key() >= end_key) {
4863-
code = MetaServiceCode::INVALID_ARGUMENT;
4864-
msg = "invalid TSO recovery start key";
4865-
return;
4866-
}
4867-
begin_key = request->start_key();
4868-
}
4869-
TxnErrorCode err = txn_kv_->create_txn(&txn);
4870-
if (err != TxnErrorCode::TXN_OK) {
4871-
code = cast_as<ErrCategory::CREATE>(err);
4872-
msg = "failed to create TSO recovery read transaction";
4873-
return;
4874-
}
4875-
std::unique_ptr<RangeGetIterator> it;
4876-
err = txn->get(begin_key, end_key, &it, true, request->batch_size());
4877-
if (err != TxnErrorCode::TXN_OK) {
4878-
code = cast_as<ErrCategory::READ>(err);
4879-
msg = "failed to get running transactions during TSO recovery";
4880-
return;
4881-
}
4882-
while (it->has_next()) {
4883-
auto [key, value] = it->next();
4884-
if (!it->has_next()) {
4885-
begin_key = key;
4886-
}
4887-
std::string_view encoded_key = key;
4888-
encoded_key.remove_prefix(1);
4889-
std::vector<std::tuple<std::variant<int64_t, std::string>, int, int>> fields;
4890-
if (decode_key(&encoded_key, &fields) != 0 || fields.size() != 5 ||
4891-
!std::holds_alternative<int64_t>(std::get<0>(fields[3])) ||
4892-
!std::holds_alternative<int64_t>(std::get<0>(fields[4]))) {
4893-
code = MetaServiceCode::UNDEFINED_ERR;
4894-
msg = "failed to decode running transaction key during TSO recovery";
4895-
return;
4896-
}
4897-
const auto db_id = std::get<int64_t>(std::get<0>(fields[3]));
4898-
const auto txn_id = std::get<int64_t>(std::get<0>(fields[4]));
4899-
if (txn_id >= request->end_txn_id()) {
4900-
continue;
4901-
}
4902-
// Running keys are removed atomically with real VISIBLE/ABORTED. Expired COMMITTED
4903-
// lazy transactions still block. Read the details in the same KV snapshot.
4904-
std::string info_val;
4905-
err = txn->get(txn_info_key({instance_id, db_id, txn_id}), &info_val, true);
4906-
if (err != TxnErrorCode::TXN_OK) {
4907-
code = cast_as<ErrCategory::READ>(err);
4908-
msg = fmt::format("failed to read TSO recovery transaction, db_id={}, txn_id={}", db_id,
4909-
txn_id);
4910-
return;
4911-
}
4912-
TxnInfoPB info;
4913-
if (!info.ParseFromString(info_val)) {
4914-
code = MetaServiceCode::PROTOBUF_PARSE_ERR;
4915-
msg = "failed to parse TSO recovery transaction";
4916-
return;
4917-
}
4918-
// Subtransactions update txn_info.table_ids; the running record can be older.
4919-
if (info.db_id() != db_id || info.txn_id() != txn_id || info.table_ids().empty()) {
4920-
code = MetaServiceCode::UNDEFINED_ERR;
4921-
msg = "invalid TSO recovery transaction identity or tables";
4922-
return;
4923-
}
4924-
if (info.status() != TxnStatusPB::TXN_STATUS_COMMITTED || !info.has_commit_tso() ||
4925-
info.commit_tso() <= 0 || info.commit_tso() > request->tso_fence()) {
4926-
continue;
4927-
}
4928-
// Recovery needs identities and visibility boundaries, not commit attachments.
4929-
auto* recovered = response->add_txn_infos();
4930-
recovered->set_db_id(db_id);
4931-
recovered->set_txn_id(txn_id);
4932-
recovered->mutable_table_ids()->CopyFrom(info.table_ids());
4933-
recovered->set_status(info.status());
4934-
recovered->set_commit_tso(info.commit_tso());
4935-
}
4936-
begin_key.push_back('\x00');
4937-
response->set_next_start_key(it->more() ? begin_key : "");
4938-
}
4939-
49404838
void MetaServiceImpl::check_txn_conflict(::google::protobuf::RpcController* controller,
49414839
const CheckTxnConflictRequest* request,
49424840
CheckTxnConflictResponse* response,

0 commit comments

Comments
 (0)