Skip to content

Commit 265dc11

Browse files
committed
[refactor](fe) Flatten IVM broken state update
### What problem does this PR solve? Issue Number: N/A Related PR: apache#28 Problem Summary: Flatten the nested condition in MTMV.markIvmBinlogBroken while preserving the existing behavior for partition-localizable and global broken states. ### Release note None ### Check List (For Author) - Test: Not run; this commit only restructures an equivalent conditional branch and will be covered by the consolidated test run. - Behavior changed: No - Does this need documentation: No
1 parent d972fe6 commit 265dc11

4 files changed

Lines changed: 30 additions & 27 deletions

File tree

‎fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java‎

Lines changed: 17 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -513,31 +513,38 @@ public void alterIvmInfo(IvmInfo ivmInfo) {
513513
}
514514

515515
public boolean markIvmBinlogBroken() {
516-
return markIvmBinlogBroken(false);
516+
return markIvmBinlogBroken(true);
517517
}
518518

519519
public boolean markIvmBinlogBrokenForPartitionChange() {
520-
return markIvmBinlogBroken(true);
520+
return markIvmBinlogBroken(false);
521521
}
522522

523-
private boolean markIvmBinlogBroken(boolean canLocalizeByPartition) {
523+
private boolean markIvmBinlogBroken(boolean requiresCompleteRefresh) {
524524
writeMvLock();
525525
try {
526526
if (ivmInfo == null) {
527527
ivmInfo = new IvmInfo();
528528
}
529+
// Invalidate refresh tasks and COMPLETE publication started before this metadata change.
529530
ivmInfo.increaseBinlogBrokenGeneration();
530531
schemaChangeVersion++;
531-
boolean changed = !ivmInfo.isBinlogBroken();
532-
if (canLocalizeByPartition) {
533-
if (changed) {
534-
ivmInfo.getBinlogBrokenPartitions().clear();
532+
// A global marker from an earlier DDL must not be narrowed by a later partition-only change.
533+
if (ivmInfo.isBinlogBroken()) {
534+
if (requiresCompleteRefresh) {
535+
return ivmInfo.getBinlogBrokenPartitions().addAll(getPartitionNames());
535536
}
536-
} else {
537-
changed |= ivmInfo.getBinlogBrokenPartitions().addAll(getPartitionNames());
537+
return false;
538538
}
539+
539540
ivmInfo.setBinlogBroken(true);
540-
return changed;
541+
if (requiresCompleteRefresh) {
542+
ivmInfo.getBinlogBrokenPartitions().addAll(getPartitionNames());
543+
return true;
544+
}
545+
// A new partition-only break is localized again from refresh snapshots.
546+
ivmInfo.getBinlogBrokenPartitions().clear();
547+
return true;
541548
} finally {
542549
writeMvUnlock();
543550
}

‎fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java‎

Lines changed: 2 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -505,10 +505,7 @@ private void executeCompleteAttempt(MTMVRefreshContext context)
505505
*/
506506
private boolean repairIvmBrokenPartitions(MTMVRefreshContext context, RefreshRequest request)
507507
throws JobException, AnalysisException {
508-
if (!mtmv.isIvm()) {
509-
return false;
510-
}
511-
if (request.refreshMode == RefreshMode.COMPLETE) {
508+
if (!mtmv.isIvm() || request.refreshMode == RefreshMode.COMPLETE) {
512509
return false;
513510
}
514511
if (!mtmv.getIvmInfo().isBinlogBroken()
@@ -680,9 +677,8 @@ private void executePartitionBasedRefresh(MTMVRefreshContext context, RefreshMod
680677
IvmPlanSignature fullRefreshPlanSignature = null;
681678
for (int i = 0; i < execNum; i++) {
682679
int start = i * refreshPartitionNum;
683-
int end = start + refreshPartitionNum;
684680
Set<String> execPartitionNames = Sets.newHashSet(needRefreshPartitions
685-
.subList(start, Math.min(end, needRefreshPartitions.size())));
681+
.subList(start, Math.min(start + refreshPartitionNum, needRefreshPartitions.size())));
686682
Map<BaseTableInfo, Set<Long>> batchResetPartitionIds = useIvmFallbackStreams
687683
? collectPctResetPartitionIds(context, execPartitionNames) : Maps.newHashMap();
688684
Optional<IvmRewriteContext> rewriteContext = Optional.empty();

‎fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelationManager.java‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -88,15 +88,15 @@ public Set<BaseTableInfo> getMtmvsByBaseTableOneLevelAndFromView(BaseTableInfo t
8888
* These operations can change visible rows without producing consumable row binlog deltas.
8989
*/
9090
public void markIvmBinlogBroken(BaseTableInfo baseTableInfo, String reason) {
91-
markIvmBinlogBroken(baseTableInfo, reason, false);
91+
markIvmBinlogBroken(baseTableInfo, reason, true);
9292
}
9393

9494
public void markIvmBinlogBrokenForPartitionChange(BaseTableInfo baseTableInfo, String reason) {
95-
markIvmBinlogBroken(baseTableInfo, reason, true);
95+
markIvmBinlogBroken(baseTableInfo, reason, false);
9696
}
9797

9898
private void markIvmBinlogBroken(BaseTableInfo baseTableInfo, String reason,
99-
boolean canLocalizeByPartition) {
99+
boolean requiresCompleteRefresh) {
100100
Set<BaseTableInfo> mtmvsByBaseTable = getMtmvsByBaseTableOneLevelAndFromView(baseTableInfo);
101101
if (CollectionUtils.isEmpty(mtmvsByBaseTable)) {
102102
return;
@@ -113,8 +113,8 @@ private void markIvmBinlogBroken(BaseTableInfo baseTableInfo, String reason,
113113
if (!mtmv.isIvm()) {
114114
continue;
115115
}
116-
if (canLocalizeByPartition
117-
? mtmv.markIvmBinlogBrokenForPartitionChange() : mtmv.markIvmBinlogBroken()) {
116+
if (requiresCompleteRefresh
117+
? mtmv.markIvmBinlogBroken() : mtmv.markIvmBinlogBrokenForPartitionChange()) {
118118
LOG.info("Marked IVM binlog broken, baseTable={}, mtmv={}, reason={}",
119119
baseTableInfo, mtmvInfo, reason);
120120
}

‎fe/fe-core/src/main/java/org/apache/doris/mtmv/ivm/IvmIncrRefreshManager.java‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -150,10 +150,10 @@ public static long markIvmBaselineBroken(MTMV mtmv) {
150150
try {
151151
// Stage a detached metadata snapshot before catalog update and journaling.
152152
IvmInfo updatedIvmInfo = new IvmInfo(mtmv.getIvmInfo());
153-
boolean changed = !updatedIvmInfo.isBinlogBroken();
154-
updatedIvmInfo.setBinlogBroken(true);
155-
changed |= updatedIvmInfo.getBinlogBrokenPartitions().addAll(mtmv.getPartitionNames());
156-
if (changed) {
153+
if (!updatedIvmInfo.isBinlogBroken()
154+
|| !updatedIvmInfo.getBinlogBrokenPartitions().containsAll(mtmv.getPartitionNames())) {
155+
updatedIvmInfo.setBinlogBroken(true);
156+
updatedIvmInfo.getBinlogBrokenPartitions().addAll(mtmv.getPartitionNames());
157157
persistIvmInfo(mtmv, updatedIvmInfo);
158158
}
159159
return mtmv.getIvmBinlogBrokenGeneration();
@@ -336,8 +336,8 @@ public static void finishIvmFullRefresh(MTMV mtmv, long expectedGeneration,
336336
}
337337

338338
private static void persistIvmInfo(MTMV mtmv, IvmInfo ivmInfo) {
339-
TableNameInfo tableName = new TableNameInfo(mtmv.getQualifiedDbName(), mtmv.getName());
340-
Env.getCurrentEnv().alterMTMVIvmInfo(tableName, ivmInfo);
339+
Env.getCurrentEnv().alterMTMVIvmInfo(
340+
new TableNameInfo(mtmv.getQualifiedDbName(), mtmv.getName()), ivmInfo);
341341
}
342342

343343
}

0 commit comments

Comments
 (0)