Skip to content

Commit bbaaecf

Browse files
seawindeyujun777
authored andcommitted
[fix](fe) Repair partition-level IVM baselines before refresh (apache#28)
Base-table DROP or TRUNCATE PARTITION can remove data without row-level binlog records. If the materialized view continues an incremental refresh from the old baseline, its result can be incorrect. This change records affected MV partitions in the existing IVM metadata and repairs them before any incremental attempt. Partitioned MVs repair only affected partitions; non-partitioned MVs fall back to COMPLETE refresh. ADD PARTITION remains eligible for normal binlog incremental refresh. Root cause: The refresh path previously treated a base partition metadata removal as a global broken state or allowed incremental refresh to continue without rebuilding the affected baseline. Change summary: | Area | Change | |------|--------| | IVM metadata | Track partition-level broken state with the existing `IvmInfo.binlogBrokenPartitions` field. | | Refresh flow | Resolve broken MV partitions during partition synchronization, repair them before IVM, and preserve broken state when repair fails. | | Refresh scope | Use partition refresh for affected partitioned MVs and COMPLETE for non-partitioned MVs or ambiguous mappings. | | Tests | Add FE unit coverage for successful and failed repairs, multi-base-partition mappings, and regression coverage for ADD, DROP, TRUNCATE, fallback, and recovery. | ```mermaid graph TD A[Refresh MV] --> B[syncPartitionsIfNeeded] B --> C[Resolve broken MV partitions] C --> D[Repair affected baseline] D --> E{Repair complete?} E -->|yes| F[IVM incremental refresh] E -->|no| G[Keep broken state and fail or use COMPLETE]
1 parent 01649db commit bbaaecf

37 files changed

Lines changed: 1317 additions & 406 deletions

‎fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java‎

Lines changed: 14 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -366,9 +366,6 @@ private boolean processAlterOlapTableInternal(List<AlterOp> alterOps, OlapTable
366366
Env.getCurrentEnv().getMtmvService()
367367
.alterTable(oldBaseTableInfo, newBaseTableInfo, currentAlterOps.hasReplaceTableOp());
368368
}
369-
markIvmBinlogBrokenIfNeeded(oldBaseTableInfo, newBaseTableInfo,
370-
currentAlterOps.hasReplaceTableOp(), alterOps);
371-
372369
olapTable.writeLock();
373370
try {
374371
NereidsSqlCacheManager sqlCacheManager = Env.getCurrentEnv().getSqlCacheManager();
@@ -464,38 +461,6 @@ private boolean needChangeMTMVState(List<AlterOp> alterOps) {
464461
return false;
465462
}
466463

467-
private void markIvmBinlogBrokenIfNeeded(BaseTableInfo oldBaseTableInfo,
468-
Optional<BaseTableInfo> newBaseTableInfo, boolean isReplaceTable, List<AlterOp> alterOps) {
469-
if (!needMarkIvmBinlogBroken(alterOps)) {
470-
return;
471-
}
472-
if (isReplaceTable) {
473-
// REPLACE TABLE A WITH TABLE B affects dependencies of both logical names:
474-
// swap=true: A points to old B, and B points to old A.
475-
// swap=false: A points to old B, and B no longer exists.
476-
// Mark both dependency sets broken. Stream recovery is handled after full refresh.
477-
Env.getCurrentEnv().getMtmvService().getRelationManager()
478-
.markIvmBinlogBroken(oldBaseTableInfo, "Base table replace changed IVM baseline");
479-
if (newBaseTableInfo.isPresent()) {
480-
Env.getCurrentEnv().getMtmvService().getRelationManager()
481-
.markIvmBinlogBroken(newBaseTableInfo.get(), "Base table replace changed IVM baseline");
482-
}
483-
} else {
484-
Env.getCurrentEnv().getMtmvService().getRelationManager().markIvmBinlogBroken(
485-
oldBaseTableInfo, "Base table alter changed IVM baseline");
486-
}
487-
}
488-
489-
private boolean needMarkIvmBinlogBroken(List<AlterOp> alterOps) {
490-
for (AlterOp alterOp : alterOps) {
491-
if (alterOp instanceof ReplaceTableOp
492-
|| alterOp instanceof DropColumnOp) {
493-
return true;
494-
}
495-
}
496-
return false;
497-
}
498-
499464
private void processModifyTableComment(Database db, OlapTable tbl, AlterOp alterOp)
500465
throws DdlException {
501466
tbl.writeLockOrDdlException();
@@ -719,6 +684,12 @@ public void processReplaceTable(Database db, OlapTable origTable, String newTblN
719684
if (swapTable) {
720685
origTable.checkAndSetName(newTblName, true);
721686
}
687+
// REPLACE TABLE affects dependencies of both logical names. Persist both
688+
// barriers before changing the catalog or journaling the replacement.
689+
Env.getCurrentEnv().getMtmvService().getRelationManager().markIvmBaselineRebuild(
690+
new BaseTableInfo(origTable), "Base table replace changed IVM baseline");
691+
Env.getCurrentEnv().getMtmvService().getRelationManager().markIvmBaselineRebuild(
692+
new BaseTableInfo(olapNewTbl), "Base table replace changed IVM baseline");
722693
replaceTableInternal(db, origTable, olapNewTbl, swapTable, false, isForce);
723694
// write edit log
724695
ReplaceTableOperationLog log = new ReplaceTableOperationLog(db.getId(),
@@ -1312,7 +1283,6 @@ public AlterHandler getSystemHandler() {
13121283
public void processAlterMTMV(AlterMTMV alterMTMV, boolean isReplay) {
13131284
TableNameInfo tbl = alterMTMV.getMvName();
13141285
MTMV mtmv = null;
1315-
boolean alterSuccess = true;
13161286
try {
13171287
Database db = Env.getCurrentInternalCatalog().getDbOrDdlException(tbl.getDb());
13181288
mtmv = (MTMV) db.getTableOrMetaException(tbl.getTbl(), TableType.MATERIALIZED_VIEW);
@@ -1333,19 +1303,20 @@ public void processAlterMTMV(AlterMTMV alterMTMV, boolean isReplay) {
13331303
updateIvmStreamsForExcludedTables(db, mtmv, oldExcludedTriggerTables,
13341304
newExcludedTriggerTables, isReplay);
13351305
}
1336-
mtmv.alterMvProperties(alterMTMV.getMvProperties());
1337-
break;
1306+
mtmv.alterMvProperties(alterMTMV, isReplay);
1307+
return;
13381308
case ADD_TASK:
1339-
alterSuccess = mtmv.addTaskResult(alterMTMV.getTask(), alterMTMV.getRelation(),
1340-
alterMTMV.getPartitionSnapshots(),
1341-
isReplay);
1309+
if (!mtmv.addTaskResult(alterMTMV, isReplay)) {
1310+
return;
1311+
}
13421312
// If it is not a replay thread, it means that the current service is already a new version
13431313
// and does not require compatibility
13441314
if (isReplay) {
13451315
mtmv.compatible(Env.getCurrentEnv().getCatalogMgr());
13461316
}
1347-
break;
1317+
return;
13481318
case ALTER_IVM_INFO:
1319+
// Live IVM changes are journaled inside MTMV; this branch applies the journal snapshot.
13491320
mtmv.alterIvmInfo(alterMTMV.getIvmInfo());
13501321
break;
13511322
default:
@@ -1354,8 +1325,7 @@ public void processAlterMTMV(AlterMTMV alterMTMV, boolean isReplay) {
13541325
if (alterMTMV.isNeedRebuildJob()) {
13551326
Env.getCurrentEnv().getMtmvService().alterJob(mtmv, isReplay);
13561327
}
1357-
// 4. log it and replay it in the follower
1358-
if (!isReplay && alterSuccess) {
1328+
if (!isReplay) {
13591329
Env.getCurrentEnv().getEditLog().logAlterMTMV(alterMTMV);
13601330
}
13611331
} catch (UserException e) {
@@ -1372,7 +1342,6 @@ private void updateIvmStreamsForExcludedTables(Database db, MTMV mtmv,
13721342
return;
13731343
}
13741344
Set<BaseTableInfo> baseTables = relation.getBaseTables();
1375-
boolean hasNewlyIncludedBaseTable = false;
13761345
for (BaseTableInfo baseTableInfo : baseTables) {
13771346
TableNameInfo baseTableName = new TableNameInfo(baseTableInfo.getCtlName(),
13781347
baseTableInfo.getDbName(), baseTableInfo.getTableName());
@@ -1381,16 +1350,12 @@ private void updateIvmStreamsForExcludedTables(Database db, MTMV mtmv,
13811350
boolean isExcluded = MTMVPartitionUtil.isTableExcluded(
13821351
newExcludedTriggerTables, baseTableName);
13831352
if (wasExcluded && !isExcluded) {
1384-
hasNewlyIncludedBaseTable = true;
13851353
if (!isReplay) {
13861354
TableIf baseTable = MTMVUtil.getTable(baseTableInfo);
13871355
CreateMTMVCommand.createTableStream(ConnectContext.get(), db, mtmv, baseTable);
13881356
}
13891357
}
13901358
}
1391-
if (hasNewlyIncludedBaseTable) {
1392-
mtmv.markIvmBinlogBroken();
1393-
}
13941359
db.writeLockOrDdlException();
13951360
try {
13961361
for (BaseTableInfo baseTableInfo : baseTables) {

‎fe/fe-core/src/main/java/org/apache/doris/catalog/CatalogRecycleBin.java‎

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,7 @@
5050
import java.io.IOException;
5151
import java.util.ArrayList;
5252
import java.util.Collection;
53+
import java.util.Collections;
5354
import java.util.HashMap;
5455
import java.util.Iterator;
5556
import java.util.List;
@@ -1066,9 +1067,10 @@ public void recoverPartition(long dbId, OlapTable table, String partitionName,
10661067
recoverPartition.setName(newPartitionName);
10671068
}
10681069

1069-
// Recovering a partition restores visible rows from recycle metadata without row binlog entries.
1070-
Env.getCurrentEnv().getMtmvService().getRelationManager().markIvmBinlogBroken(
1071-
new BaseTableInfo(table, dbId), "Base table partition was recovered without row binlog");
1070+
Env.getCurrentEnv().getMtmvService().getRelationManager().markIvmBaselineRebuildForPartitionChange(
1071+
new BaseTableInfo(table, dbId),
1072+
Collections.singletonMap(partitionName, recoverPartition.getId()),
1073+
"Base table partition was recovered without row binlog");
10721074
// recover partition
10731075
table.addPartition(recoverPartition);
10741076

@@ -1129,9 +1131,6 @@ public void replayRecoverPartition(OlapTable table, long partitionId,
11291131
throw new DdlException("Partition name[" + newPartitionName + "] is already used");
11301132
}
11311133
}
1132-
Env.getCurrentEnv().getMtmvService().getRelationManager().markIvmBinlogBroken(
1133-
new BaseTableInfo(table, recyclePartitionInfo.getDbId()),
1134-
"Base table partition was recovered without row binlog");
11351134
table.addPartition(recyclePartitionInfo.getPartition());
11361135
if (!Strings.isNullOrEmpty(newPartitionName)) {
11371136
table.renamePartition(recyclePartitionInfo.getPartition().getName(), newPartitionName);

‎fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java‎

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -162,7 +162,6 @@
162162
import org.apache.doris.mtmv.MTMVService;
163163
import org.apache.doris.mtmv.MTMVStatus;
164164
import org.apache.doris.mtmv.MTMVUtil;
165-
import org.apache.doris.mtmv.ivm.IvmInfo;
166165
import org.apache.doris.mtmv.ivm.IvmUtil;
167166
import org.apache.doris.mysql.authenticate.AuthenticateType;
168167
import org.apache.doris.mysql.authenticate.AuthenticatorManager;
@@ -7015,9 +7014,18 @@ public void replaceTempPartition(Database db, OlapTable olapTable, ReplacePartit
70157014
throw new DdlException("Temp partition[" + partName + "] does not exist");
70167015
}
70177016
}
7018-
// Replacing partitions swaps visible data through metadata, so row-binlog continuity is broken.
7019-
getMtmvService().getRelationManager().markIvmBinlogBroken(
7020-
new BaseTableInfo(olapTable), "Base table partitions were replaced without row binlog");
7017+
if (isStrictRange) {
7018+
Map<String, Long> replacedPartitions = Maps.newHashMapWithExpectedSize(partitionNames.size());
7019+
for (String partitionName : partitionNames) {
7020+
replacedPartitions.put(partitionName, olapTable.getPartition(partitionName).getId());
7021+
}
7022+
getMtmvService().getRelationManager().markIvmBaselineRebuildForPartitionChange(
7023+
new BaseTableInfo(olapTable), replacedPartitions,
7024+
"Base table partitions were replaced without row binlog");
7025+
} else {
7026+
getMtmvService().getRelationManager().markIvmBaselineRebuild(
7027+
new BaseTableInfo(olapTable), "Base table partitions were replaced without row binlog");
7028+
}
70217029
List<Long> replacedPartitionIds = olapTable.replaceTempPartitions(db.getId(), partitionNames,
70227030
tempPartitionNames, isStrictRange,
70237031
useTempPartitionName, isForceDropOld);
@@ -7069,8 +7077,6 @@ public void replayReplaceTempPartition(ReplacePartitionOperationLog replaceTempP
70697077
.getTableOrMetaException(tableId, Lists.newArrayList(TableType.OLAP, TableType.MATERIALIZED_VIEW));
70707078
olapTable.writeLock();
70717079
try {
7072-
getMtmvService().getRelationManager().markIvmBinlogBroken(
7073-
new BaseTableInfo(olapTable), "Base table partitions were replaced without row binlog");
70747080
olapTable.replaceTempPartitions(dbId, replaceTempPartitionLog.getPartitions(),
70757081
replaceTempPartitionLog.getTempPartitions(), replaceTempPartitionLog.isStrictRange(),
70767082
replaceTempPartitionLog.useTempPartitionName(), replaceTempPartitionLog.isForce());
@@ -7612,12 +7618,6 @@ public void addMTMVTaskResult(TableNameInfo mvName, MTMVTask task, MTMVRelation
76127618
this.alter.processAlterMTMV(alter, false);
76137619
}
76147620

7615-
public void alterMTMVIvmInfo(TableNameInfo mvName, IvmInfo ivmInfo) {
7616-
AlterMTMV alter = new AlterMTMV(mvName, MTMVAlterOpType.ALTER_IVM_INFO);
7617-
alter.setIvmInfo(ivmInfo);
7618-
this.alter.processAlterMTMV(alter, false);
7619-
}
7620-
76217621
// Ensure the env is ready, otherwise throw an exception.
76227622
public void checkReadyOrThrow() throws Exception {
76237623
if (isReady()) {

0 commit comments

Comments
 (0)