Skip to content

Commit 69de5df

Browse files
authored
Fix bug that cluster balance may cause load job failed (#1581)
The bug is described in issue #1580 . And this patch will fix 2 cases of cluster balance After finish adding the new replica, the new replica's version may not catch up with the visible version, so the new replica may be treated as a stale and redundant replica, which will be deleting at next tablet checking round. I add a mark named needFurtherRepair to the newly added replica, only mark it when that replica's version does not catch up with visible version. This replica will receive a further repair at next tablet checking round, instead of being deleted. When deleting the redundant replicas, there may be some load jobs on it. Delete these replicas may cause the load job fail. Before deleting a redundant replica, I first mark the next txn id on that replica, and set replica's state to CLONE. The CLONE state will ensure that no more load jobs will be on that replica, and we will wait all load jobs before the marked txn id to be finished. After that, the replica can be deleted safely.
1 parent 326d765 commit 69de5df

10 files changed

Lines changed: 178 additions & 61 deletions

File tree

‎fe/src/main/java/org/apache/doris/alter/RollupHandler.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -361,6 +361,7 @@ private void processAddRollup(AddRollupClause alterClause, Database db, OlapTabl
361361
long rollupReplicaId = catalog.getNextId();
362362
long backendId = baseReplica.getBackendId();
363363
if (baseReplica.getState() == ReplicaState.CLONE
364+
|| baseReplica.getState() == ReplicaState.DECOMMISSION
364365
|| baseReplica.getLastFailedVersion() > 0) {
365366
// just skip it.
366367
continue;

‎fe/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -973,6 +973,7 @@ private void createJob(long dbId, OlapTable olapTable, Map<Long, LinkedList<Colu
973973
int replicaNum = 0;
974974
for (Replica replica : tablet.getReplicas()) {
975975
if (replica.getState() == ReplicaState.CLONE
976+
|| replica.getState() == ReplicaState.DECOMMISSION
976977
|| replica.getLastFailedVersion() > 0) {
977978
// just skip it (replica cloned from old schema will be deleted)
978979
continue;
@@ -1052,7 +1053,9 @@ private void createJob(long dbId, OlapTable olapTable, Map<Long, LinkedList<Colu
10521053
// set replica state
10531054
for (Tablet tablet : alterIndex.getTablets()) {
10541055
for (Replica replica : tablet.getReplicas()) {
1055-
if (replica.getState() == ReplicaState.CLONE || replica.getLastFailedVersion() > 0) {
1056+
if (replica.getState() == ReplicaState.CLONE
1057+
|| replica.getState() == ReplicaState.DECOMMISSION
1058+
|| replica.getLastFailedVersion() > 0) {
10561059
// this should not happen, cause we only allow schema change when table is stable.
10571060
LOG.error("replica {} of tablet {} on backend {} is not NORMAL: {}",
10581061
replica.getId(), tablet.getId(), replica.getBackendId(), replica);

‎fe/src/main/java/org/apache/doris/alter/SchemaChangeJob.java‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -514,7 +514,9 @@ public synchronized void cancel(OlapTable olapTable, String msg) {
514514
for (Tablet tablet : index.getTablets()) {
515515
long tabletId = tablet.getId();
516516
for (Replica replica : tablet.getReplicas()) {
517-
if (replica.getState() == ReplicaState.CLONE || replica.getState() == ReplicaState.NORMAL) {
517+
if (replica.getState() == ReplicaState.CLONE
518+
|| replica.getState() == ReplicaState.DECOMMISSION
519+
|| replica.getState() == ReplicaState.NORMAL) {
518520
continue;
519521
}
520522
Preconditions.checkState(replica.getState() == ReplicaState.SCHEMA_CHANGE);
@@ -918,7 +920,8 @@ public void replayInitJob(Database db) {
918920
// set state to SCHEMA_CHANGE
919921
for (Tablet tablet : index.getTablets()) {
920922
for (Replica replica : tablet.getReplicas()) {
921-
if (replica.getState() == ReplicaState.CLONE) {
923+
if (replica.getState() == ReplicaState.CLONE
924+
|| replica.getState() == ReplicaState.DECOMMISSION) {
922925
// add log here, because there should no more CLONE replica when processing alter jobs.
923926
LOG.warn(String.format(
924927
"replica %d of tablet %d in backend %d state is invalid: %s",
@@ -1063,7 +1066,9 @@ public void replayCancel(Database db) {
10631066
}
10641067
for (Tablet tablet : index.getTablets()) {
10651068
for (Replica replica : tablet.getReplicas()) {
1066-
if (replica.getState() == ReplicaState.CLONE || replica.getState() == ReplicaState.NORMAL) {
1069+
if (replica.getState() == ReplicaState.CLONE
1070+
|| replica.getState() == ReplicaState.DECOMMISSION
1071+
|| replica.getState() == ReplicaState.NORMAL) {
10671072
continue;
10681073
}
10691074
replica.setState(ReplicaState.NORMAL);

‎fe/src/main/java/org/apache/doris/catalog/Replica.java‎

Lines changed: 46 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,12 @@ public enum ReplicaState {
4141
NORMAL,
4242
ROLLUP,
4343
SCHEMA_CHANGE,
44-
CLONE
44+
CLONE,
45+
DECOMMISSION; // replica is ready to be deleted
46+
47+
public boolean isLoadable() {
48+
return this == ReplicaState.NORMAL || this == ReplicaState.SCHEMA_CHANGE;
49+
}
4550
}
4651

4752
public enum ReplicaStatus {
@@ -75,6 +80,26 @@ public enum ReplicaStatus {
7580

7681
private boolean bad = false;
7782

83+
/*
84+
* If set to true, with means this replica need to be repaired. explicitly.
85+
* This can happen when this replica is created by a balance clone task, and
86+
* when task finished, the version of this replica is behind the partition's visible version.
87+
* So this replica need a further repair.
88+
* If we do not do this, this replica will be treated as version stale, and will be removed,
89+
* so that the balance task is failed, which is unexpected.
90+
*
91+
* furtherRepairSetTime set alone with needFurtherRepair.
92+
* This is an insurance, in case that further repair task always fail. If 20 min passed
93+
* since we set needFurtherRepair to true, the 'needFurtherRepair' will be set to false.
94+
*/
95+
private boolean needFurtherRepair = false;
96+
private long furtherRepairSetTime = -1;
97+
private static final long FURTHER_REPAIR_TIMEOUT_MS = 20 * 60 * 1000L; // 20min
98+
99+
// if this watermarkTxnId is set, which means before deleting a replica,
100+
// we should ensure that all txns on this replicas are finished.
101+
private long watermarkTxnId = -1;
102+
78103
public Replica() {
79104
}
80105

@@ -192,6 +217,18 @@ public boolean setBad(boolean bad) {
192217
return true;
193218
}
194219

220+
public boolean needFurtherRepair() {
221+
if (needFurtherRepair && System.currentTimeMillis() - this.furtherRepairSetTime < FURTHER_REPAIR_TIMEOUT_MS) {
222+
return true;
223+
}
224+
return false;
225+
}
226+
227+
public void setNeedFurtherRepair(boolean needFurtherRepair) {
228+
this.needFurtherRepair = needFurtherRepair;
229+
this.furtherRepairSetTime = System.currentTimeMillis();
230+
}
231+
195232
// only update data size and row num
196233
public synchronized void updateStat(long dataSize, long rowNum) {
197234
this.dataSize = dataSize;
@@ -482,4 +519,12 @@ public int compare(T replica1, T replica2) {
482519
}
483520
}
484521
}
522+
523+
public void setWatermarkTxnId(long watermarkTxnId) {
524+
this.watermarkTxnId = watermarkTxnId;
525+
}
526+
527+
public long getWatermarkTxnId() {
528+
return watermarkTxnId;
529+
}
485530
}

‎fe/src/main/java/org/apache/doris/catalog/Tablet.java‎

Lines changed: 21 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -49,15 +49,16 @@ public class Tablet extends MetaObject implements Writable {
4949

5050
public enum TabletStatus {
5151
HEALTHY,
52-
REPLICA_MISSING, // not enough alive replica num
53-
VERSION_INCOMPLETE, // alive replica num is enough, but version is missing
54-
REPLICA_RELOCATING, // replica is healthy, but is under relocating (eg. BE is decommission)
55-
REDUNDANT, // too much replicas
56-
REPLICA_MISSING_IN_CLUSTER, // not enough healthy replicas in correct cluster
52+
REPLICA_MISSING, // not enough alive replica num.
53+
VERSION_INCOMPLETE, // alive replica num is enough, but version is missing.
54+
REPLICA_RELOCATING, // replica is healthy, but is under relocating (eg. BE is decommission).
55+
REDUNDANT, // too much replicas.
56+
REPLICA_MISSING_IN_CLUSTER, // not enough healthy replicas in correct cluster.
5757
FORCE_REDUNDANT, // some replica is missing or bad, but there is no other backends for repair,
5858
// at least one replica has to be deleted first to make room for new replica.
5959
COLOCATE_MISMATCH, // replicas do not all locate in right colocate backends set.
60-
COLOCATE_REDUNDANT, // replicas match the colocate backends set, but redundant
60+
COLOCATE_REDUNDANT, // replicas match the colocate backends set, but redundant.
61+
NEED_FURTHER_REPAIR, // one of replicas need a definite repair.
6162
}
6263

6364
private long id;
@@ -71,7 +72,7 @@ public enum TabletStatus {
7172
// last time that the tablet checker checks this tablet.
7273
// no need to persist
7374
private long lastStatusCheckTime = -1;
74-
75+
7576
public Tablet() {
7677
this(0L, new ArrayList<Replica>());
7778
}
@@ -383,10 +384,12 @@ public Pair<TabletStatus, TabletSchedCtx.Priority> getHealthStatusWithPriority(
383384
int stable = 0;
384385
int availableInCluster = 0;
385386

387+
Replica needFurtherRepairReplica = null;
386388
Set<String> hosts = Sets.newHashSet();
387389
for (Replica replica : replicas) {
388390
Backend backend = systemInfoService.getBackend(replica.getBackendId());
389391
if (backend == null || !backend.isAlive() || replica.getState() == ReplicaState.CLONE
392+
|| replica.getState() == ReplicaState.DECOMMISSION
390393
|| replica.isBad() || !hosts.add(backend.getHost())) {
391394
// this replica is not alive,
392395
// or if this replica is on same host with another replica, we also treat it as 'dead',
@@ -414,6 +417,10 @@ public Pair<TabletStatus, TabletSchedCtx.Priority> getHealthStatusWithPriority(
414417
continue;
415418
}
416419
availableInCluster++;
420+
421+
if (replica.needFurtherRepair() && needFurtherRepairReplica == null) {
422+
needFurtherRepairReplica = replica;
423+
}
417424
}
418425

419426
// 1. alive replicas are not enough
@@ -440,6 +447,10 @@ public Pair<TabletStatus, TabletSchedCtx.Priority> getHealthStatusWithPriority(
440447
} else if (aliveAndVersionComplete < replicationNum) {
441448
return Pair.create(TabletStatus.VERSION_INCOMPLETE, TabletSchedCtx.Priority.NORMAL);
442449
} else if (aliveAndVersionComplete > replicationNum) {
450+
if (needFurtherRepairReplica != null) {
451+
return Pair.create(TabletStatus.NEED_FURTHER_REPAIR, TabletSchedCtx.Priority.HIGH);
452+
}
453+
// we set REDUNDANT as VERY_HIGH, because delete redundant replicas can free the space quickly.
443454
return Pair.create(TabletStatus.REDUNDANT, TabletSchedCtx.Priority.VERY_HIGH);
444455
}
445456

@@ -454,6 +465,9 @@ public Pair<TabletStatus, TabletSchedCtx.Priority> getHealthStatusWithPriority(
454465
if (availableInCluster < replicationNum) {
455466
return Pair.create(TabletStatus.REPLICA_MISSING_IN_CLUSTER, TabletSchedCtx.Priority.LOW);
456467
} else if (replicas.size() > replicationNum) {
468+
if (needFurtherRepairReplica != null) {
469+
return Pair.create(TabletStatus.NEED_FURTHER_REPAIR, TabletSchedCtx.Priority.HIGH);
470+
}
457471
// we set REDUNDANT as VERY_HIGH, because delete redundant replicas can free the space quickly.
458472
return Pair.create(TabletStatus.REDUNDANT, TabletSchedCtx.Priority.VERY_HIGH);
459473
}

‎fe/src/main/java/org/apache/doris/clone/TabletSchedCtx.java‎

Lines changed: 25 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -341,6 +341,10 @@ public void setTablet(Tablet tablet) {
341341
this.tablet = tablet;
342342
}
343343

344+
public Tablet getTablet() {
345+
return tablet;
346+
}
347+
344348
// database lock should be held.
345349
public List<Replica> getReplicas() {
346350
return tablet.getReplicas();
@@ -520,6 +524,7 @@ public void chooseSrcReplicaForVersionIncomplete(Map<Long, PathSlot> backendsWor
520524
* 1. replica's last failed version > 0
521525
* 2. better to choose a replica which has a lower last failed version
522526
* 3. best to choose a replica if its last success version > last failed version
527+
* 4. if these is replica which need further repair, choose that replica.
523528
*
524529
* database lock should be held.
525530
*/
@@ -536,11 +541,17 @@ public void chooseDestReplicaForVersionIncomplete(Map<Long, PathSlot> backendsWo
536541
continue;
537542
}
538543

539-
if (replica.getLastFailedVersion() <= 0 && ((replica.getVersion() == visibleVersion
540-
&& replica.getVersionHash() == visibleVersionHash) || replica.getVersion() > visibleVersion)) {
544+
if (replica.getLastFailedVersion() <= 0
545+
&& ((replica.getVersion() == visibleVersion && replica.getVersionHash() == visibleVersionHash)
546+
|| replica.getVersion() > visibleVersion)) {
541547
// skip healthy replica
542548
continue;
543549
}
550+
551+
if (replica.needFurtherRepair()) {
552+
chosenReplica = replica;
553+
break;
554+
}
544555

545556
if (chosenReplica == null) {
546557
chosenReplica = replica;
@@ -866,8 +877,17 @@ public void finishCloneTask(CloneTask cloneTask, TFinishTaskRequest request)
866877
replica.updateVersionInfo(reportedTablet.getVersion(), reportedTablet.getVersion_hash(),
867878
reportedTablet.getData_size(), reportedTablet.getRow_count());
868879

869-
state = State.FINISHED;
870-
880+
if (this.type == Type.BALANCE) {
881+
long partitionVisibleVersion = partition.getVisibleVersion();
882+
if (replica.getVersion() < partitionVisibleVersion) {
883+
// see comment 'needFurtherRepair' of Replica for explanation.
884+
// no need to persist this info. If FE restart, just do it again.
885+
replica.setNeedFurtherRepair(true);
886+
}
887+
} else {
888+
replica.setNeedFurtherRepair(false);
889+
}
890+
871891
ReplicaPersistInfo info = ReplicaPersistInfo.createForClone(dbId, tblId, partitionId, indexId,
872892
tabletId, destBackendId, replica.getId(),
873893
reportedTablet.getVersion(),
@@ -889,6 +909,7 @@ public void finishCloneTask(CloneTask cloneTask, TFinishTaskRequest request)
889909
Catalog.getInstance().getEditLog().logUpdateReplica(info);
890910
}
891911

912+
state = State.FINISHED;
892913
LOG.info("clone finished: {}", this);
893914
} catch (SchedException e) {
894915
// if failed to too many times, remove this task

0 commit comments

Comments
 (0)