Skip to content

Commit 5cdbd2c

Browse files
seawindeyujun777
authored andcommitted
[fix](fe) Rebuild IVM baseline after excluded trigger changes (apache#18)
Changing `excluded_trigger_tables` can change which base table updates are considered when deciding whether an MTMV is fresh. On the IVM branch, adding a changed base table to `excluded_trigger_tables` could leave the old refresh snapshot baseline in place and allow the refresh path to skip the changed table or take the IVM no-op fast path, leaving stale MV data. Root cause: In the MTMV refresh path, the refresh snapshot baseline was not fully invalidated/rebuilt after `excluded_trigger_tables` changed, and fallback-capable IVM refresh could still use the IVM fast path before a complete baseline was rebuilt. Change Summary: | File | Change Description | |------|--------------------| | `MTMVTask.java` | Treat incomplete refresh snapshots as requiring a full partition refresh and avoid silently using stale incremental baselines. | | `MTMVTaskTest.java` | Cover incomplete refresh snapshot partition selection behavior. | | `AlterMTMVTest.java` | Cover excluded-trigger ALTER validation and baseline invalidation scope. | | `test_ivm_excluded_trigger_table.groovy` | Add IVM excluded-trigger coverage for incremental refresh suppression and fallback after ALTER. | | `test_ivm_plan_signature_fallback.groovy` | Verify IVM plan signature changes trigger fallback-capable refresh to rebuild through COMPLETE mode. | | `test_excluded_trigger_table_mtmv.groovy` | Update manual refresh modes away from illegal `AUTO` and add ordinary partitioned MTMV excluded-trigger coverage. | Design Rationale: Fallback-capable IVM refresh now rebuilds a complete baseline when the refresh snapshot is incomplete. Explicit `INCREMENTAL` refresh without fallback still fails for incomplete baselines instead of silently falling back, preserving the requested refresh semantics. ```mermaid graph TD A[ALTER excluded_trigger_tables] --> B[Invalidate refresh snapshot baseline] B --> C[Next REFRESH] C --> D{Snapshot complete?} D -- No + fallback capable --> E[Run COMPLETE refresh] D -- No + explicit INCREMENTAL --> F[Reject incomplete baseline] D -- Yes --> G[Run incremental/partition refresh] ```
1 parent b7224dd commit 5cdbd2c

10 files changed

Lines changed: 645 additions & 73 deletions

File tree

‎fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -469,12 +469,13 @@ public MTMVRefreshSnapshot getRefreshSnapshot() {
469469
return refreshSnapshot;
470470
}
471471

472-
public boolean hasCompleteRefreshSnapshot() {
473-
Set<String> partitionNames = getPartitionNames();
472+
public boolean hasRefreshSnapshot() {
474473
readMvLock();
475474
try {
476-
// A refresh baseline is complete only when every current MV partition has a snapshot.
477-
return refreshSnapshot.getPartitionSnapshots().keySet().containsAll(partitionNames);
475+
// IVM only needs to know whether a baseline has ever been built.
476+
// A newly added MV partition legitimately has no PCT snapshot yet,
477+
// but that must not block row-level incremental refresh.
478+
return refreshSnapshot != null && !MapUtils.isEmpty(refreshSnapshot.getPartitionSnapshots());
478479
} finally {
479480
readMvUnlock();
480481
}

‎fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java‎

Lines changed: 28 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -120,8 +120,8 @@ public class MTMVTask extends AbstractTask {
120120
new Column("CompletedPartitions", ScalarType.createStringType()),
121121
new Column("Progress", ScalarType.createStringType()),
122122
new Column("LastQueryId", ScalarType.createStringType()),
123-
new Column("IvmFallbackReason", ScalarType.createStringType()),
124-
new Column("ComputeGroup", ScalarType.createStringType()));
123+
new Column("ComputeGroup", ScalarType.createStringType()),
124+
new Column("IvmFallbackReason", ScalarType.createStringType()));
125125

126126
public static final ImmutableMap<String, Integer> COLUMN_TO_INDEX;
127127

@@ -291,8 +291,14 @@ public void run() throws JobException {
291291
// refresh fallback: incompatible MV definitions must fail directly.
292292
ensureQueryUsableIfNeeded(ctx, tableIfs);
293293
RefreshRequest request = resolveRefreshRequest();
294+
List<RefreshAttemptType> attempts = buildAttempts(request);
295+
try {
296+
syncPartitionsIfNeeded(ctx, tableIfs);
297+
} catch (PartitionPlanningException e) {
298+
throw new JobException(e.getMessage(), e);
299+
}
294300
boolean disablePartitionRefresh = false;
295-
for (RefreshAttemptType attemptType : buildAttempts(request)) {
301+
for (RefreshAttemptType attemptType : attempts) {
296302
switch (attemptType) {
297303
case IVM:
298304
AttemptResultType ivmResult = executeIvmAttempt(request);
@@ -307,7 +313,7 @@ public void run() throws JobException {
307313
if (disablePartitionRefresh) {
308314
break;
309315
}
310-
if (executePartitionBasedRefresh(ctx, tableIfs, request)) {
316+
if (executePartitionBasedRefresh(tableIfs, request)) {
311317
return;
312318
}
313319
break;
@@ -400,6 +406,10 @@ private RefreshRequest resolveRefreshRequest() throws JobException {
400406
}
401407

402408
private List<RefreshAttemptType> buildAttempts(RefreshRequest request) {
409+
if (taskContext.getTriggerMode() != MTMVTaskTriggerMode.MANUAL
410+
&& mtmv.isIvm() && !mtmv.hasRefreshSnapshot()) {
411+
return Lists.newArrayList(RefreshAttemptType.COMPLETE);
412+
}
403413
List<RefreshAttemptType> attempts = Lists.newArrayList();
404414
switch (request.refreshMode) {
405415
case AUTO:
@@ -433,8 +443,8 @@ private List<RefreshAttemptType> buildAttempts(RefreshRequest request) {
433443
return attempts;
434444
}
435445

436-
private PartitionRefreshPlan planPartitionRefresh(ConnectContext ctx, List<TableIf> tableIfs,
437-
RefreshRequest request) throws JobException, AnalysisException, DdlException {
446+
private PartitionRefreshPlan planPartitionRefresh(List<TableIf> tableIfs, RefreshRequest request)
447+
throws AnalysisException {
438448
if (mtmv.isIvm() && mtmv.getIvmInfo().isRunningIvmRefresh()) {
439449
// A failed IVM run may have written partial delta data. Only a
440450
// COMPLETE refresh can be used as recovery; PARTITIONS is skipped.
@@ -452,11 +462,6 @@ private PartitionRefreshPlan planPartitionRefresh(ConnectContext ctx, List<Table
452462
"The partition method of this asynchronous materialized view "
453463
+ "does not support refreshing by partition");
454464
}
455-
try {
456-
syncPartitionsIfNeeded(ctx, tableIfs);
457-
} catch (PartitionPlanningException e) {
458-
return PartitionRefreshPlan.fallback(e.getMessage());
459-
}
460465
MTMVRefreshContext context = buildRefreshContext(tableIfs);
461466
boolean fresh;
462467
try {
@@ -549,19 +554,23 @@ private AttemptResultType executeIvmAttempt(RefreshRequest request) throws JobEx
549554
mtmv.getName(), getTaskId());
550555
return AttemptResultType.FALLBACK_TO_COMPLETE;
551556
}
557+
if (ivmResult.getFailureReason() == IvmFailureReason.PLAN_SIGNATURE_MISMATCH) {
558+
LOG.warn("IVM refresh fell back for mv={}, reason={}, detail={}, taskId={}. "
559+
+ "Continuing with COMPLETE refresh.",
560+
mtmv.getName(), ivmResult.getFailureReason(),
561+
ivmResult.getDetailMessage(), getTaskId());
562+
return AttemptResultType.FALLBACK_TO_COMPLETE;
563+
}
552564
LOG.warn("IVM refresh fell back for mv={}, reason={}, detail={}, taskId={}. "
553565
+ "Continuing with partition-based refresh.",
554566
mtmv.getName(), ivmResult.getFailureReason(),
555567
ivmResult.getDetailMessage(), getTaskId());
556-
if (ivmResult.getFailureReason() == IvmFailureReason.PLAN_SIGNATURE_MISMATCH) {
557-
return AttemptResultType.FALLBACK_TO_COMPLETE;
558-
}
559568
return AttemptResultType.FALLBACK_ALLOWED;
560569
}
561570

562-
private boolean executePartitionBasedRefresh(ConnectContext ctx, List<TableIf> tableIfs,
563-
RefreshRequest request) throws JobException, AnalysisException, DdlException {
564-
PartitionRefreshPlan partitionPlan = planPartitionRefresh(ctx, tableIfs, request);
571+
private boolean executePartitionBasedRefresh(List<TableIf> tableIfs, RefreshRequest request)
572+
throws JobException, AnalysisException {
573+
PartitionRefreshPlan partitionPlan = planPartitionRefresh(tableIfs, request);
565574
if (!partitionPlan.canRefreshByPartitions) {
566575
if (request.allowFallback) {
567576
LOG.warn("MTMV partition refresh fell back for mv={}, reason={}, taskId={}",
@@ -913,10 +922,10 @@ public TRow getTvfInfo(String jobName) {
913922
new TCell().setStringVal(getProgress()));
914923
trow.addToColumnValue(
915924
new TCell().setStringVal(lastQueryId));
916-
trow.addToColumnValue(new TCell().setStringVal(
917-
ivmFallbackReason == null ? FeConstants.null_string : ivmFallbackReason));
918925
trow.addToColumnValue(new TCell().setStringVal(
919926
computeGroup == null || computeGroup.isEmpty() ? FeConstants.null_string : computeGroup));
927+
trow.addToColumnValue(new TCell().setStringVal(
928+
ivmFallbackReason == null ? FeConstants.null_string : ivmFallbackReason));
920929
return trow;
921930
}
922931

@@ -1002,12 +1011,6 @@ public List<String> calculateNeedRefreshPartitions(MTMVRefreshContext context)
10021011
if (mtmv.getRefreshInfo().getRefreshMethod() == RefreshMethod.COMPLETE) {
10031012
return Lists.newArrayList(mtmv.getPartitionNames());
10041013
}
1005-
// An incomplete baseline cannot be checked by isMTMVSync, because the current exclude rules may
1006-
// skip the changed base tables and incorrectly mark the MV as fresh. Rebuild it with a full refresh.
1007-
if (!mtmv.hasCompleteRefreshSnapshot()) {
1008-
return Lists.newArrayList(mtmv.getPartitionNames());
1009-
}
1010-
// check if data is fresh
10111014
// We need to use a newly generated relationship and cannot retrieve it using mtmv.getRelation()
10121015
// to avoid rebuilding the baseTable and causing a change in the tableId
10131016
boolean fresh = MTMVPartitionUtil.isMTMVSync(context, relation.getBaseTablesOneLevelAndFromView(),

‎fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/RefreshMTMVInfo.java‎

Lines changed: 13 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -118,28 +118,32 @@ private void validateRefreshModeCompat(MTMV mtmv) {
118118
// expanded to a full refresh by fallback.
119119
throw new AnalysisException("partitionSpec does not support FALLBACK");
120120
}
121+
if (!CollectionUtils.isEmpty(partitions) && mtmv.isIvm()) {
122+
// IVM MVs should not bypass incremental semantics through a legacy
123+
// partitionSpec. Users can explicitly choose the PARTITIONS keyword
124+
// when they want the partition-refresh strategy.
125+
throw new AnalysisException(
126+
"partitionSpec is not allowed on a materialized view with INCREMENTAL capability, "
127+
+ "use PARTITIONS keyword instead.");
128+
}
121129
RefreshMethod mvRefreshMethod = mtmv.getRefreshInfo() == null
122130
? null : mtmv.getRefreshInfo().getRefreshMethod();
123131
if (mvRefreshMethod == null) {
124132
throw new AnalysisException("Materialized view has unknown refresh method.");
125133
}
134+
if (!CollectionUtils.isEmpty(partitions)) {
135+
return;
136+
}
126137
if (!isRefreshModeCompatible(mtmv, mvRefreshMethod)) {
127138
throw new AnalysisException("Cannot use " + refreshMode
128139
+ " refresh on a materialized view with " + mvRefreshMethod + " refresh policy.");
129140
}
130-
if (mtmv.isIvm() && !CollectionUtils.isEmpty(partitions)) {
131-
// IVM MVs should not bypass incremental semantics through a legacy
132-
// partitionSpec. Users can explicitly choose the PARTITIONS keyword
133-
// when they want the partition-refresh strategy.
134-
throw new AnalysisException(
135-
"partitionSpec is not allowed on a materialized view with INCREMENTAL capability, "
136-
+ "use PARTITIONS keyword instead.");
137-
}
138141
}
139142

140143
private boolean isRefreshModeCompatible(MTMV mtmv, RefreshMethod mvRefreshMethod) {
141144
if (refreshMode == RefreshMode.AUTO) {
142-
return false;
145+
// Keep compatibility with legacy manual REFRESH ... AUTO syntax.
146+
return true;
143147
}
144148
switch (mvRefreshMethod) {
145149
case COMPLETE:

‎fe/fe-core/src/test/java/org/apache/doris/mtmv/AlterMTMVTest.java‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -371,4 +371,28 @@ public void testCreateIncrementalMtmvExcludeTriggerTableSkipsStream() throws Exc
371371
.getTableOrMetaException(includedStreamName);
372372
Assertions.assertNotNull(includedStream, "Non-excluded table should have a stream auto-created");
373373
}
374+
375+
@Test
376+
public void testAlterIvmExcludedTriggerTablesCanOnlyExpandScope() throws Exception {
377+
createDatabaseAndUse("alter_ivm_excluded_trigger_test");
378+
createTable("CREATE TABLE alter_ivm_excluded_trigger_test.ivm_base (k1 int, v1 int)\n"
379+
+ "DUPLICATE KEY(k1)\n"
380+
+ "DISTRIBUTED BY HASH(k1) BUCKETS 1\n"
381+
+ "PROPERTIES ('replication_num' = '1', 'binlog.enable' = 'true', 'binlog.format' = 'ROW')");
382+
createMvByNereids("CREATE MATERIALIZED VIEW ivm_excluded_mv\n"
383+
+ " BUILD DEFERRED REFRESH INCREMENTAL ON MANUAL\n"
384+
+ " DISTRIBUTED BY RANDOM BUCKETS 2\n"
385+
+ " PROPERTIES ('replication_num' = '1',"
386+
+ " 'excluded_trigger_tables' = 'alter_ivm_excluded_trigger_test.ivm_base')\n"
387+
+ " AS SELECT k1, v1 FROM ivm_base");
388+
389+
alterMv("ALTER MATERIALIZED VIEW ivm_excluded_mv\n"
390+
+ " SET ('excluded_trigger_tables' = 'ivm_base')");
391+
392+
Exception ex = Assertions.assertThrows(Exception.class,
393+
() -> alterMv("ALTER MATERIALIZED VIEW ivm_excluded_mv\n"
394+
+ " SET ('excluded_trigger_tables' = 'alter_ivm_excluded_trigger_test.ivm_base')"));
395+
Assertions.assertTrue(ex.getMessage().contains("Existing excluded trigger tables can only be expanded"),
396+
"unexpected message: " + ex.getMessage());
397+
}
374398
}

0 commit comments

Comments
 (0)