Skip to content

Commit 235a08f

Browse files
committed
Delete bad replicas when no BE can be used to create new replica. for apache#4104
1 parent ad17afe commit 235a08f

7 files changed

Lines changed: 47 additions & 36 deletions

File tree

‎fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1229,7 +1229,7 @@ public void checkStableAndNormal(String clusterName) throws DdlException {
12291229
}
12301230

12311231
public boolean isStable(SystemInfoService infoService, TabletScheduler tabletScheduler, String clusterName) {
1232-
int availableBackendsNum = infoService.getClusterBackendIds(clusterName, true).size();
1232+
List<Long> aliveBeIdsInCluster = infoService.getClusterBackendIds(clusterName, true);
12331233
for (Partition partition : idToPartition.values()) {
12341234
long visibleVersion = partition.getVisibleVersion();
12351235
long visibleVersionHash = partition.getVisibleVersionHash();
@@ -1242,7 +1242,7 @@ public boolean isStable(SystemInfoService infoService, TabletScheduler tabletSch
12421242

12431243
Pair<TabletStatus, TabletSchedCtx.Priority> statusPair = tablet.getHealthStatusWithPriority(
12441244
infoService, clusterName, visibleVersion, visibleVersionHash, replicationNum,
1245-
availableBackendsNum);
1245+
aliveBeIdsInCluster);
12461246
if (statusPair.first != TabletStatus.HEALTHY) {
12471247
LOG.info("table {} is not stable because tablet {} status is {}. replicas: {}",
12481248
id, tablet.getId(), statusPair.first, tablet.getReplicas());

‎fe/fe-core/src/main/java/org/apache/doris/catalog/Tablet.java‎

Lines changed: 29 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,11 @@
1717

1818
package org.apache.doris.catalog;
1919

20+
import com.google.common.collect.HashMultimap;
21+
import com.google.common.collect.Lists;
22+
import com.google.common.collect.Multimap;
23+
import com.google.common.collect.Sets;
24+
import com.google.gson.annotations.SerializedName;
2025
import org.apache.doris.catalog.Replica.ReplicaState;
2126
import org.apache.doris.clone.TabletSchedCtx;
2227
import org.apache.doris.clone.TabletSchedCtx.Priority;
@@ -25,13 +30,6 @@
2530
import org.apache.doris.common.io.Writable;
2631
import org.apache.doris.system.Backend;
2732
import org.apache.doris.system.SystemInfoService;
28-
29-
import com.google.common.collect.HashMultimap;
30-
import com.google.common.collect.Lists;
31-
import com.google.common.collect.Multimap;
32-
import com.google.common.collect.Sets;
33-
import com.google.gson.annotations.SerializedName;
34-
3533
import org.apache.logging.log4j.LogManager;
3634
import org.apache.logging.log4j.Logger;
3735

@@ -42,6 +40,7 @@
4240
import java.util.Iterator;
4341
import java.util.List;
4442
import java.util.Set;
43+
import java.util.stream.Collectors;
4544

4645
/**
4746
* This class represents the olap tablet related metadata.
@@ -407,7 +406,7 @@ public long getDataSize(boolean singleReplica) {
407406
public Pair<TabletStatus, TabletSchedCtx.Priority> getHealthStatusWithPriority(
408407
SystemInfoService systemInfoService, String clusterName,
409408
long visibleVersion, long visibleVersionHash, int replicationNum,
410-
int availableBackendsNum) {
409+
List<Long> aliveBeIdsInCluster) {
411410

412411
int alive = 0;
413412
int aliveAndVersionComplete = 0;
@@ -453,15 +452,16 @@ public Pair<TabletStatus, TabletSchedCtx.Priority> getHealthStatusWithPriority(
453452
}
454453

455454
// 1. alive replicas are not enough
456-
if (alive < replicationNum && replicas.size() >= availableBackendsNum
457-
&& availableBackendsNum >= replicationNum && replicationNum > 1) {
455+
int aliveBackendsNum = aliveBeIdsInCluster.size();
456+
if (alive < replicationNum && replicas.size() >= aliveBackendsNum
457+
&& aliveBackendsNum >= replicationNum && replicationNum > 1) {
458458
// there is no enough backend for us to create a new replica, so we have to delete an existing replica,
459459
// so there can be available backend for us to create a new replica.
460460
// And if there is only one replica, we will not handle it(maybe need human interference)
461461
// condition explain:
462462
// 1. alive < replicationNum: replica is missing or bad
463-
// 2. replicas.size() >= availableBackendsNum: the existing replicas occupies all available backends
464-
// 3. availableBackendsNum >= replicationNum: make sure after deleting, there will be at least one backend for new replica.
463+
// 2. replicas.size() >= aliveBackendsNum: the existing replicas occupies all available backends
464+
// 3. aliveBackendsNum >= replicationNum: make sure after deleting, there will be at least one backend for new replica.
465465
// 4. replicationNum > 1: if replication num is set to 1, do not delete any replica, for safety reason
466466
return Pair.create(TabletStatus.FORCE_REDUNDANT, TabletSchedCtx.Priority.VERY_HIGH);
467467
} else if (alive < (replicationNum / 2) + 1) {
@@ -484,10 +484,23 @@ public Pair<TabletStatus, TabletSchedCtx.Priority> getHealthStatusWithPriority(
484484
}
485485

486486
// 3. replica is under relocating
487-
if (stable < (replicationNum / 2) + 1) {
488-
return Pair.create(TabletStatus.REPLICA_RELOCATING, TabletSchedCtx.Priority.NORMAL);
489-
} else if (stable < replicationNum) {
490-
return Pair.create(TabletStatus.REPLICA_RELOCATING, TabletSchedCtx.Priority.LOW);
487+
if (stable < replicationNum) {
488+
List<Long> replicaBeIds = replicas.stream()
489+
.map(Replica::getBackendId).collect(Collectors.toList());
490+
List<Long> availableBeIds = aliveBeIdsInCluster.stream()
491+
.filter(systemInfoService::checkBackendAvailable)
492+
.collect(Collectors.toList());
493+
if (replicaBeIds.containsAll(availableBeIds)
494+
&& availableBeIds.size() >= replicationNum
495+
&& replicationNum > 1) { // No BE can be choose to create a new replica
496+
return Pair.create(TabletStatus.FORCE_REDUNDANT,
497+
stable < (replicationNum / 2) + 1 ? TabletSchedCtx.Priority.NORMAL : TabletSchedCtx.Priority.LOW);
498+
}
499+
if (stable < (replicationNum / 2) + 1) {
500+
return Pair.create(TabletStatus.REPLICA_RELOCATING, TabletSchedCtx.Priority.NORMAL);
501+
} else if (stable < replicationNum) {
502+
return Pair.create(TabletStatus.REPLICA_RELOCATING, TabletSchedCtx.Priority.LOW);
503+
}
491504
}
492505

493506
// 4. healthy replicas in cluster are not enough

‎fe/fe-core/src/main/java/org/apache/doris/clone/TabletChecker.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -206,7 +206,7 @@ private void checkTablets() {
206206

207207
db.readLock();
208208
try {
209-
int availableBackendsNum = infoService.getClusterBackendIds(db.getClusterName(), true).size();
209+
List<Long> aliveBeIdsInCluster = infoService.getClusterBackendIds(db.getClusterName(), true);
210210
for (Table table : db.getTables()) {
211211
if (!table.needSchedule()) {
212212
continue;
@@ -239,7 +239,7 @@ private void checkTablets() {
239239
partition.getVisibleVersion(),
240240
partition.getVisibleVersionHash(),
241241
olapTbl.getPartitionInfo().getReplicationNum(partition.getId()),
242-
availableBackendsNum);
242+
aliveBeIdsInCluster);
243243

244244
if (statusWithPrio.first == TabletStatus.HEALTHY) {
245245
// Only set last status check time when status is healthy.

‎fe/fe-core/src/main/java/org/apache/doris/clone/TabletSchedCtx.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -799,11 +799,11 @@ public void finishCloneTask(CloneTask cloneTask, TFinishTaskRequest request)
799799
throw new SchedException(Status.UNRECOVERABLE, "tablet does not exist");
800800
}
801801

802-
int availableBackendsNum = infoService.getClusterBackendIds(db.getClusterName(), true).size();
802+
List<Long> aliveBeIdsInCluster = infoService.getClusterBackendIds(db.getClusterName(), true);
803803
short replicationNum = olapTable.getPartitionInfo().getReplicationNum(partitionId);
804804
Pair<TabletStatus, TabletSchedCtx.Priority> pair = tablet.getHealthStatusWithPriority(
805805
infoService, db.getClusterName(), visibleVersion, visibleVersionHash, replicationNum,
806-
availableBackendsNum);
806+
aliveBeIdsInCluster);
807807
if (pair.first == TabletStatus.HEALTHY) {
808808
throw new SchedException(Status.FINISHED, "tablet is healthy");
809809
}

‎fe/fe-core/src/main/java/org/apache/doris/clone/TabletScheduler.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -495,13 +495,13 @@ private void scheduleTablet(TabletSchedCtx tabletCtx, AgentBatchTask batchTask)
495495
statusPair = Pair.create(st, Priority.HIGH);
496496
tabletCtx.setColocateGroupBackendIds(backendsSet);
497497
} else {
498-
int availableBackendsNum = infoService.getClusterBackendIds(db.getClusterName(), true).size();
498+
List<Long> aliveBeIdsInCluster = infoService.getClusterBackendIds(db.getClusterName(), true);
499499
statusPair = tablet.getHealthStatusWithPriority(
500500
infoService, tabletCtx.getCluster(),
501501
partition.getVisibleVersion(),
502502
partition.getVisibleVersionHash(),
503503
tbl.getPartitionInfo().getReplicationNum(partition.getId()),
504-
availableBackendsNum);
504+
aliveBeIdsInCluster);
505505
}
506506

507507
if (tabletCtx.getType() == TabletSchedCtx.Type.BALANCE && tableState != OlapTableState.NORMAL) {

‎fe/fe-core/src/main/java/org/apache/doris/common/proc/StatisticProcDir.java‎

Lines changed: 8 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,10 @@
1717

1818
package org.apache.doris.common.proc;
1919

20+
import com.google.common.base.Preconditions;
21+
import com.google.common.collect.HashMultimap;
22+
import com.google.common.collect.ImmutableList;
23+
import com.google.common.collect.Multimap;
2024
import org.apache.doris.catalog.Catalog;
2125
import org.apache.doris.catalog.Database;
2226
import org.apache.doris.catalog.MaterializedIndex;
@@ -34,19 +38,13 @@
3438
import org.apache.doris.system.SystemInfoService;
3539
import org.apache.doris.task.AgentTaskQueue;
3640
import org.apache.doris.thrift.TTaskType;
37-
38-
import com.google.common.base.Preconditions;
39-
import com.google.common.collect.HashMultimap;
40-
import com.google.common.collect.ImmutableList;
41-
import com.google.common.collect.Multimap;
41+
import org.apache.logging.log4j.LogManager;
42+
import org.apache.logging.log4j.Logger;
4243

4344
import java.util.ArrayList;
4445
import java.util.Collections;
4546
import java.util.List;
4647

47-
import org.apache.logging.log4j.LogManager;
48-
import org.apache.logging.log4j.Logger;
49-
5048
public class StatisticProcDir implements ProcDirInterface {
5149
public static final ImmutableList<String> TITLE_NAMES = new ImmutableList.Builder<String>()
5250
.add("DbId").add("DbName").add("TableNum").add("PartitionNum")
@@ -108,7 +106,7 @@ public ProcResult fetchResult() throws AnalysisException {
108106
}
109107

110108
++totalDbNum;
111-
int availableBackendsNum = infoService.getClusterBackendIds(db.getClusterName(), true).size();
109+
List<Long> aliveBeIdsInCluster = infoService.getClusterBackendIds(db.getClusterName(), true);
112110
db.readLock();
113111
try {
114112
int dbTableNum = 0;
@@ -137,7 +135,7 @@ public ProcResult fetchResult() throws AnalysisException {
137135
Pair<TabletStatus, Priority> res = tablet.getHealthStatusWithPriority(
138136
infoService, db.getClusterName(),
139137
partition.getVisibleVersion(), partition.getVisibleVersionHash(),
140-
replicationNum, availableBackendsNum);
138+
replicationNum, aliveBeIdsInCluster);
141139

142140
// here we treat REDUNDANT as HEALTHY, for user friendly.
143141
if (res.first != TabletStatus.HEALTHY && res.first != TabletStatus.REDUNDANT

‎fe/fe-core/src/main/java/org/apache/doris/master/ReportHandler.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1027,10 +1027,10 @@ private static void addReplica(long tabletId, TTabletInfo backendTabletInfo, lon
10271027
return;
10281028
}
10291029

1030-
int availableBackendsNum = infoService.getClusterBackendIds(db.getClusterName(), true).size();
1030+
List<Long> aliveBeIdsInCluster = infoService.getClusterBackendIds(db.getClusterName(), true);
10311031
Pair<TabletStatus, TabletSchedCtx.Priority> status = tablet.getHealthStatusWithPriority(infoService,
10321032
db.getClusterName(), visibleVersion, visibleVersionHash,
1033-
replicationNum, availableBackendsNum);
1033+
replicationNum, aliveBeIdsInCluster);
10341034

10351035
if (status.first == TabletStatus.VERSION_INCOMPLETE || status.first == TabletStatus.REPLICA_MISSING) {
10361036
long lastFailedVersion = -1L;

0 commit comments

Comments
 (0)