Skip to content

Commit 783c7d3

Browse files
platonekomorningman
authored andcommitted
[fix](replica) Fix inconsistent replica id between BE and FE in corner case of tablet rebalance (#16889)
1 parent 8534abc commit 783c7d3

4 files changed

Lines changed: 27 additions & 16 deletions

File tree

‎be/src/olap/tablet_manager.cpp‎

Lines changed: 5 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -436,8 +436,7 @@ Status TabletManager::drop_tablet(TTabletId tablet_id, TReplicaId replica_id,
436436
auto& shard = _get_tablets_shard(tablet_id);
437437
std::lock_guard wrlock(shard.lock);
438438
if (shard.tablets_under_clone.count(tablet_id) > 0) {
439-
LOG(INFO) << "tablet " << tablet_id << " is under clone, skip drop task";
440-
return Status::Aborted("aborted");
439+
return Status::Aborted("tablet {} is under clone, skip drop task", tablet_id);
441440
}
442441
SCOPED_CONSUME_MEM_TRACKER(_mem_tracker);
443442
return _drop_tablet_unlocked(tablet_id, replica_id, false, is_drop_table_or_partition);
@@ -459,12 +458,9 @@ Status TabletManager::_drop_tablet_unlocked(TTabletId tablet_id, TReplicaId repl
459458
// We should compare replica id to avoid dropping new cloned tablet.
460459
// Iff request replica id is 0, FE may be an older release, then we drop this tablet as before.
461460
if (to_drop_tablet->replica_id() != replica_id && replica_id != 0) {
462-
LOG(WARNING) << "fail to drop tablet because replica id not match. "
463-
<< "tablet_id=" << tablet_id << ", replica_id=" << to_drop_tablet->replica_id()
464-
<< ", request replica_id=" << replica_id;
465-
return Status::Aborted("aborted");
461+
return Status::Aborted("replica_id not match({} vs {})", to_drop_tablet->replica_id(),
462+
replica_id);
466463
}
467-
468464
_remove_tablet_from_partition(to_drop_tablet);
469465
tablet_map_t& tablet_map = _get_tablet_map(tablet_id);
470466
tablet_map.erase(tablet_id);
@@ -1043,10 +1039,10 @@ Status TabletManager::start_trash_sweep() {
10431039
return Status::OK();
10441040
} // start_trash_sweep
10451041

1046-
void TabletManager::register_clone_tablet(int64_t tablet_id) {
1042+
bool TabletManager::register_clone_tablet(int64_t tablet_id) {
10471043
tablets_shard& shard = _get_tablets_shard(tablet_id);
10481044
std::lock_guard<std::shared_mutex> wrlock(shard.lock);
1049-
shard.tablets_under_clone.insert(tablet_id);
1045+
return shard.tablets_under_clone.insert(tablet_id).second;
10501046
}
10511047

10521048
void TabletManager::unregister_clone_tablet(int64_t tablet_id) {

‎be/src/olap/tablet_manager.h‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -133,7 +133,8 @@ class TabletManager {
133133

134134
void obtain_specific_quantity_tablets(std::vector<TabletInfo>& tablets_info, int64_t num);
135135

136-
void register_clone_tablet(int64_t tablet_id);
136+
// return `true` if register success
137+
bool register_clone_tablet(int64_t tablet_id);
137138
void unregister_clone_tablet(int64_t tablet_id);
138139

139140
void get_tablets_distribution_on_different_disks(

‎be/src/olap/task/engine_clone_task.cpp‎

Lines changed: 13 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -63,7 +63,9 @@ EngineCloneTask::EngineCloneTask(const TCloneReq& clone_req, const TMasterInfo&
6363
Status EngineCloneTask::execute() {
6464
// register the tablet to avoid it is deleted by gc thread during clone process
6565
SCOPED_ATTACH_TASK(_mem_tracker);
66-
StorageEngine::instance()->tablet_manager()->register_clone_tablet(_clone_req.tablet_id);
66+
if (StorageEngine::instance()->tablet_manager()->register_clone_tablet(_clone_req.tablet_id)) {
67+
return Status::InternalError("tablet {} is under clone", _clone_req.tablet_id);
68+
}
6769
Status st = _do_clone();
6870
StorageEngine::instance()->tablet_manager()->unregister_clone_tablet(_clone_req.tablet_id);
6971
return st;
@@ -81,6 +83,13 @@ Status EngineCloneTask::_do_clone() {
8183
std::vector<Version> missed_versions;
8284
// try to repair a tablet with missing version
8385
if (tablet != nullptr) {
86+
if (tablet->replica_id() != _clone_req.replica_id) {
87+
// `tablet` may be a dropped replica in FE, e.g: BE1 migrates replica of tablet_1 to BE2,
88+
// but before BE1 drop this replica, another new replica of tablet_1 is migrated to BE1.
89+
// If we allow to clone success on dropped replica, replica id may never be consistent between FE and BE.
90+
return Status::InternalError("replica_id not match({} vs {})", tablet->replica_id(),
91+
_clone_req.replica_id);
92+
}
8493
std::shared_lock migration_rlock(tablet->get_migration_lock(), std::try_to_lock);
8594
if (!migration_rlock.owns_lock()) {
8695
return Status::OLAPInternalError(OLAP_ERR_RWLOCK_ERROR);
@@ -96,23 +105,23 @@ Status EngineCloneTask::_do_clone() {
96105
// completed. Or remote be will just return header not the rowset files. clone will failed.
97106
if (missed_versions.empty()) {
98107
LOG(INFO) << "missed version size = 0, skip clone and return success. tablet_id="
99-
<< _clone_req.tablet_id;
108+
<< _clone_req.tablet_id << " replica_id=" << _clone_req.replica_id;
100109
_set_tablet_info(is_new_tablet);
101110
return Status::OK();
102111
}
103112

104113
LOG(INFO) << "clone to existed tablet. missed_versions_size=" << missed_versions.size()
105114
<< ", allow_incremental_clone=" << allow_incremental_clone
106115
<< ", signature=" << _signature << ", tablet_id=" << _clone_req.tablet_id
107-
<< ", committed_version=" << _clone_req.committed_version;
116+
<< ", committed_version=" << _clone_req.committed_version
117+
<< ", replica_id=" << _clone_req.replica_id;
108118

109119
// try to download missing version from src backend.
110120
// if tablet on src backend does not contains missing version, it will download all versions,
111121
// and set allow_incremental_clone to false
112122
RETURN_IF_ERROR(_make_and_download_snapshots(*(tablet->data_dir()), local_data_path,
113123
&src_host, &src_file_path, missed_versions,
114124
&allow_incremental_clone));
115-
116125
RETURN_IF_ERROR(_finish_clone(tablet.get(), local_data_path, _clone_req.committed_version,
117126
allow_incremental_clone));
118127
} else {

‎fe/fe-core/src/main/java/org/apache/doris/master/ReportHandler.java‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -921,6 +921,12 @@ private static void addReplica(long tabletId, TabletMeta tabletMeta, TTabletInfo
921921
throw new MetaNotFoundException("tablet[" + tabletId + "] does not exist");
922922
}
923923

924+
// check replica id
925+
long replicaId = backendTabletInfo.getReplicaId();
926+
if (replicaId <= 0) {
927+
throw new MetaNotFoundException("replica id is invalid, tablet id: " + tabletId);
928+
}
929+
924930
long visibleVersion = partition.getVisibleVersion();
925931

926932
// check replica version
@@ -964,8 +970,7 @@ private static void addReplica(long tabletId, TabletMeta tabletMeta, TTabletInfo
964970
} else if (version < partition.getCommittedVersion()) {
965971
lastFailedVersion = partition.getCommittedVersion();
966972
}
967-
968-
long replicaId = Env.getCurrentEnv().getNextId();
973+
// use replicaId reported by BE to maintain replica meta consistent between FE and BE
969974
Replica replica = new Replica(replicaId, backendId, version, schemaHash,
970975
dataSize, remoteDataSize, rowCount, ReplicaState.NORMAL,
971976
lastFailedVersion, version);

0 commit comments

Comments
 (0)