Skip to content

Commit d7e7c01

Browse files
seawindeyujun777
authored andcommitted
[fix](fe) Reject invalid IVM refresh before partition sync (apache#32)
### What problem does this PR solve? Issue Number: N/A Related PR: apache#28 Problem Summary: Strict IVM refresh could synchronize MV partitions before discovering that its binlog baseline was invalid. The refresh then failed while leaving the MV partially changed. The root cause was that baseline validation happened after partition synchronization in MTMVTask. The fix validates pending baseline state before any partition DDL, rejecting strict incremental and explicit partition refreshes atomically while preserving AUTO and FALLBACK recovery. FE unit coverage verifies the validation matrix, and a regression test verifies that a failed strict refresh preserves the original MV rows before AUTO recovery rebuilds the baseline. ### Release note Strict IVM refresh now fails before changing MV partitions when its baseline requires rebuilding. ### Check List (For Author) - Test: Regression test and Unit Test - MTMVTaskTest: 43 tests passed - test_ivm_strict_failure_partition_atomicity: passed - test_ivm_partition_baseline_rebuild: passed - Behavior changed: No, this restores atomic failure behavior. - Does this need documentation: No
1 parent 11a0eba commit d7e7c01

4 files changed

Lines changed: 144 additions & 20 deletions

File tree

‎fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java‎

Lines changed: 24 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -310,6 +310,7 @@ public void run() throws JobException {
310310
// refresh fallback: incompatible MV definitions must fail directly.
311311
ensureQueryUsableIfNeeded(ctx, tableIfs);
312312
RefreshRequest request = resolveRefreshRequest();
313+
validateIvmBaselineBeforePartitionSync(request);
313314
List<RefreshAttemptType> attempts = buildAttempts(request, queryAnalysis.containsOneRowRelation());
314315
try {
315316
syncPartitionsIfNeeded(ctx, tableIfs);
@@ -541,19 +542,8 @@ private boolean handlePendingIvmBaselineRebuild(MTMVRefreshContext context, Refr
541542
return false;
542543
}
543544
ivmFallbackReason = IvmFailureReason.BINLOG_BROKEN.name();
544-
if ((request.refreshMode == RefreshMode.INCREMENTAL && !request.allowFallback)
545-
|| request.explicitPartitions) {
546-
refreshMode = MTMVTaskRefreshMode.NOT_REFRESH;
547-
throw new JobException("IVM baseline rebuild is pending for mv=" + mtmv.getName()
548-
+ "; run an AUTO or COMPLETE refresh first");
549-
}
550545
IvmInfo ivmInfo = mtmv.getIvmInfo();
551546
if (ivmInfo.requiresCompleteBaselineRebuild()) {
552-
if (request.refreshMode == RefreshMode.PARTITIONS && !request.allowFallback) {
553-
refreshMode = MTMVTaskRefreshMode.NOT_REFRESH;
554-
throw new JobException("COMPLETE IVM baseline rebuild is pending for mv=" + mtmv.getName()
555-
+ "; run a PARTITIONS FALLBACK, AUTO, or COMPLETE refresh");
556-
}
557547
executeCompleteAttempt(context, ctx);
558548
return true;
559549
}
@@ -568,6 +558,29 @@ private boolean handlePendingIvmBaselineRebuild(MTMVRefreshContext context, Refr
568558
return true;
569559
}
570560

561+
private void validateIvmBaselineBeforePartitionSync(RefreshRequest request) throws JobException {
562+
if (!mtmv.isIvm() || request.refreshMode == RefreshMode.COMPLETE) {
563+
return;
564+
}
565+
IvmInfo ivmInfo = mtmv.getIvmInfo();
566+
if (!ivmInfo.isBaselineRebuildRequired()) {
567+
return;
568+
}
569+
ivmFallbackReason = IvmFailureReason.BINLOG_BROKEN.name();
570+
if ((request.refreshMode == RefreshMode.INCREMENTAL && !request.allowFallback)
571+
|| request.explicitPartitions) {
572+
refreshMode = MTMVTaskRefreshMode.NOT_REFRESH;
573+
throw new JobException("IVM baseline rebuild is pending for mv=" + mtmv.getName()
574+
+ "; run an AUTO or COMPLETE refresh first");
575+
}
576+
if (request.refreshMode == RefreshMode.PARTITIONS && !request.allowFallback
577+
&& ivmInfo.requiresCompleteBaselineRebuild()) {
578+
refreshMode = MTMVTaskRefreshMode.NOT_REFRESH;
579+
throw new JobException("COMPLETE IVM baseline rebuild is pending for mv=" + mtmv.getName()
580+
+ "; run a PARTITIONS FALLBACK, AUTO, or COMPLETE refresh");
581+
}
582+
}
583+
571584
private AttemptResultType executeIvmAttempt(MTMVRefreshContext refreshContext,
572585
RefreshRequest request, ConnectContext ctx, List<TableIf> tableIfs) throws JobException {
573586
if (!mtmv.isIvm()) {

‎fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVTaskTest.java‎

Lines changed: 4 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -689,27 +689,22 @@ public void testExecuteIvmAttemptFallsBackToCompleteForBrokenBaseline() throws E
689689
}
690690

691691
@Test
692-
public void testStrictIncrementalRejectsPendingBaselineBeforeIvm() throws Exception {
692+
public void testStrictIncrementalRejectsPendingBaselineBeforePartitionSync() throws Exception {
693693
Mockito.when(mtmv.isIvm()).thenReturn(true);
694694
IvmInfo ivmInfo = new IvmInfo();
695695
ivmInfo.requireCompleteBaselineRebuild();
696696
Mockito.when(mtmv.getIvmInfo()).thenReturn(ivmInfo);
697697
Mockito.when(mtmv.getPartitionNames()).thenReturn(Collections.singleton(poneName));
698698
MTMVTask task = new MTMVTask(mtmv, relation, MTMVTaskContext.of(
699699
MTMVTaskTriggerMode.MANUAL, null, RefreshMode.INCREMENTAL, false, null));
700-
MTMVRefreshContext refreshContext = Mockito.mock(MTMVRefreshContext.class);
701-
Mockito.when(refreshContext.getMtmv()).thenReturn(mtmv);
702700
Object request = Deencapsulation.invoke(task, "resolveRefreshRequest");
703701

704702
JobException exception = Assert.assertThrows(JobException.class,
705-
() -> Deencapsulation.invoke(task, "handlePendingIvmBaselineRebuild", refreshContext, request,
706-
new ConnectContext()));
703+
() -> Deencapsulation.invoke(task, "validateIvmBaselineBeforePartitionSync", request));
707704

708705
Assert.assertTrue(exception.getMessage().contains("run an AUTO or COMPLETE refresh first"));
709706
Assert.assertEquals(IvmFailureReason.BINLOG_BROKEN.name(),
710707
Deencapsulation.getField(task, "ivmFallbackReason"));
711-
mtmvPartitionUtilStatic.verify(() -> MTMVPartitionUtil.getMTMVNeedRefreshPartitions(
712-
Mockito.same(refreshContext), Mockito.nullable(Set.class)), Mockito.never());
713708
}
714709

715710
@Test
@@ -723,8 +718,7 @@ public void testBarePartitionsRejectsCompletePendingBaseline() throws Exception
723718
Object request = Deencapsulation.invoke(task, "resolveRefreshRequest");
724719

725720
JobException exception = Assert.assertThrows(JobException.class,
726-
() -> Deencapsulation.invoke(task, "handlePendingIvmBaselineRebuild",
727-
Mockito.mock(MTMVRefreshContext.class), request, new ConnectContext()));
721+
() -> Deencapsulation.invoke(task, "validateIvmBaselineBeforePartitionSync", request));
728722

729723
Assert.assertTrue(exception.getMessage().contains("run a PARTITIONS FALLBACK, AUTO, or COMPLETE refresh"));
730724
}
@@ -740,6 +734,7 @@ public void testPartitionsFallbackRebuildsPendingBaselineWithComplete() throws E
740734
MTMVTaskTriggerMode.MANUAL, null, RefreshMode.PARTITIONS, true, null));
741735
Object request = Deencapsulation.invoke(task, "resolveRefreshRequest");
742736

737+
Deencapsulation.invoke(task, "validateIvmBaselineBeforePartitionSync", request);
743738
Assert.assertTrue(Deencapsulation.invoke(task, "handlePendingIvmBaselineRebuild",
744739
Mockito.mock(MTMVRefreshContext.class), request, new ConnectContext()));
745740
Assert.assertEquals(MTMVTask.MTMVTaskRefreshMode.NOT_REFRESH,
Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
-- This file is automatically generated. You should know what you did if you want to edit this
2+
-- !before_ddl --
3+
2026-01-10 1 10
4+
2026-02-10 2 20
5+
2026-03-10 3 30
6+
7+
-- !strict_task --
8+
FAILED BINLOG_BROKEN true
9+
10+
-- !after_strict_failure --
11+
2026-01-10 1 10
12+
2026-02-10 2 20
13+
2026-03-10 3 30
14+
15+
-- !after_auto_recovery --
16+
2026-03-10 3 30
17+
Lines changed: 99 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,99 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
import org.awaitility.Awaitility
19+
import static java.util.concurrent.TimeUnit.SECONDS
20+
21+
suite("test_ivm_strict_failure_partition_atomicity", "nonConcurrent") {
22+
sql """DROP MATERIALIZED VIEW IF EXISTS ivm_strict_atomicity_mv"""
23+
sql """DROP TABLE IF EXISTS ivm_strict_atomicity_t"""
24+
sql """
25+
CREATE TABLE ivm_strict_atomicity_t (
26+
dt DATE NOT NULL,
27+
id BIGINT NOT NULL,
28+
v INT
29+
)
30+
UNIQUE KEY(dt, id)
31+
PARTITION BY RANGE(dt) (
32+
PARTITION p1 VALUES [('2026-01-01'), ('2026-02-01')),
33+
PARTITION p2 VALUES [('2026-02-01'), ('2026-03-01')),
34+
PARTITION p3 VALUES [('2026-03-01'), ('2026-04-01'))
35+
)
36+
DISTRIBUTED BY HASH(id) BUCKETS 1
37+
PROPERTIES (
38+
"replication_num" = "1",
39+
"enable_unique_key_merge_on_write" = "true",
40+
"binlog.enable" = "true",
41+
"binlog.format" = "ROW",
42+
"binlog.need_historical_value" = "true"
43+
)
44+
"""
45+
sql """INSERT INTO ivm_strict_atomicity_t VALUES
46+
('2026-01-10', 1, 10),
47+
('2026-02-10', 2, 20),
48+
('2026-03-10', 3, 30)"""
49+
sql """
50+
CREATE MATERIALIZED VIEW ivm_strict_atomicity_mv
51+
BUILD DEFERRED REFRESH INCREMENTAL ON MANUAL
52+
PARTITION BY(dt)
53+
DISTRIBUTED BY HASH(id) BUCKETS 1
54+
PROPERTIES ("replication_num" = "1")
55+
AS SELECT dt, id, v FROM ivm_strict_atomicity_t
56+
"""
57+
sql """REFRESH MATERIALIZED VIEW ivm_strict_atomicity_mv COMPLETE"""
58+
waitingMTMVTaskFinishedByMvName("ivm_strict_atomicity_mv")
59+
order_qt_before_ddl """
60+
SELECT dt, id, v FROM ivm_strict_atomicity_mv ORDER BY dt, id
61+
"""
62+
63+
sql """TRUNCATE TABLE ivm_strict_atomicity_t PARTITION(p1)"""
64+
sql """ALTER TABLE ivm_strict_atomicity_t DROP PARTITION p2"""
65+
def previousTaskId = sql("""
66+
SELECT TaskId FROM tasks('type'='mv')
67+
WHERE MvDatabaseName = '${context.dbName}' AND MvName = 'ivm_strict_atomicity_mv'
68+
ORDER BY CreateTime DESC, TaskId DESC LIMIT 1
69+
""")[0][0].toString()
70+
71+
sql """REFRESH MATERIALIZED VIEW ivm_strict_atomicity_mv INCREMENTAL"""
72+
Awaitility.await().atMost(300, SECONDS).pollInterval(2, SECONDS).until({
73+
def task = sql_return_maparray("""
74+
SELECT TaskId, Status FROM tasks('type'='mv')
75+
WHERE MvDatabaseName = '${context.dbName}' AND MvName = 'ivm_strict_atomicity_mv'
76+
ORDER BY CreateTime DESC, TaskId DESC LIMIT 1
77+
""")
78+
return !task.isEmpty()
79+
&& task[0].TaskId.toString() != previousTaskId
80+
&& task[0].Status.toString() != 'PENDING'
81+
&& task[0].Status.toString() != 'RUNNING'
82+
})
83+
order_qt_strict_task """
84+
SELECT Status, IvmFallbackReason,
85+
ErrorMsg LIKE '%baseline rebuild is pending%'
86+
FROM tasks('type'='mv')
87+
WHERE MvDatabaseName = '${context.dbName}' AND MvName = 'ivm_strict_atomicity_mv'
88+
ORDER BY CreateTime DESC, TaskId DESC LIMIT 1
89+
"""
90+
order_qt_after_strict_failure """
91+
SELECT dt, id, v FROM ivm_strict_atomicity_mv ORDER BY dt, id
92+
"""
93+
94+
sql """REFRESH MATERIALIZED VIEW ivm_strict_atomicity_mv AUTO"""
95+
waitingMTMVTaskFinishedByMvName("ivm_strict_atomicity_mv")
96+
order_qt_after_auto_recovery """
97+
SELECT dt, id, v FROM ivm_strict_atomicity_mv ORDER BY dt, id
98+
"""
99+
}

0 commit comments

Comments
 (0)