Skip to content

Commit f90adc1

Browse files
committed
[fix](fe) Respect sync state for self-managed MTMV refresh
### What problem does this PR solve? Issue Number: close apache#27313 Related PR: apache#28 Problem Summary: SELF_MANAGE materialized views fell back to COMPLETE before checking whether their base data was already synchronized. This caused AUTO refreshes with no data changes to run unnecessary COMPLETE refreshes. Check synchronization first so unchanged views return NOT_REFRESH, while changed SELF_MANAGE views still fall back to COMPLETE. Update affected MTMV and IVM regression cases for the current refresh policy, debug-point name, and generated stream naming. ### Release note None ### Check List (For Author) - Test: Unit Test and Regression test - ./run-fe-ut.sh --run org.apache.doris.mtmv.MTMVTaskTest - Focused mtmv_p0 and mtmv_p0/ivm regression suites Behavior changed: Yes (unchanged SELF_MANAGE AUTO refreshes now return NOT_REFRESH) Does this need documentation: No
1 parent 16ae468 commit f90adc1

9 files changed

Lines changed: 54 additions & 33 deletions

File tree

‎fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -478,13 +478,6 @@ private PartitionRefreshPlan planPartitionRefresh(MTMVRefreshContext context,
478478
if (request.explicitPartitions) {
479479
return PartitionRefreshPlan.success(context, request.partitions);
480480
}
481-
if (mtmv.getMvPartitionInfo().getPartitionType() == MTMVPartitionType.SELF_MANAGE) {
482-
// Keep this inside the PARTITIONS attempt so PARTITIONS FALLBACK and
483-
// AUTO can still continue to COMPLETE for non-partitioned MVs.
484-
return PartitionRefreshPlan.fallback(
485-
"The partition method of this asynchronous materialized view "
486-
+ "does not support refreshing by partition");
487-
}
488481
boolean fresh;
489482
try {
490483
fresh = MTMVPartitionUtil.isMTMVSync(context, relation.getBaseTablesOneLevelAndFromView(),
@@ -495,6 +488,13 @@ private PartitionRefreshPlan planPartitionRefresh(MTMVRefreshContext context,
495488
if (fresh) {
496489
return PartitionRefreshPlan.success(context, Lists.newArrayList());
497490
}
491+
if (mtmv.getMvPartitionInfo().getPartitionType() == MTMVPartitionType.SELF_MANAGE) {
492+
// Keep this inside the PARTITIONS attempt so PARTITIONS FALLBACK and
493+
// AUTO can still continue to COMPLETE for non-partitioned MVs.
494+
return PartitionRefreshPlan.fallback(
495+
"The partition method of this asynchronous materialized view "
496+
+ "does not support refreshing by partition");
497+
}
498498
try {
499499
return PartitionRefreshPlan.success(context,
500500
MTMVPartitionUtil.getMTMVNeedRefreshPartitions(context,

‎fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVTaskTest.java‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -177,6 +177,20 @@ public void testCalculateNeedRefreshPartitionsSystem() throws AnalysisException,
177177
Assert.assertTrue(CollectionUtils.isEmpty(result));
178178
}
179179

180+
@Test
181+
public void testPlanPartitionRefreshSelfManageWhenSync() throws Exception {
182+
Mockito.when(mtmvPartitionInfo.getPartitionType()).thenReturn(MTMVPartitionType.SELF_MANAGE);
183+
MTMVTask task = new MTMVTask(mtmv, relation,
184+
MTMVTaskContext.of(MTMVTaskTriggerMode.MANUAL, null, RefreshMode.AUTO));
185+
Object request = Deencapsulation.invoke(task, "resolveRefreshRequest");
186+
187+
Object plan = Deencapsulation.invoke(task, "planPartitionRefresh",
188+
Mockito.mock(MTMVRefreshContext.class), request);
189+
190+
Assert.assertTrue(Deencapsulation.getField(plan, "canRefreshByPartitions"));
191+
Assert.assertTrue(CollectionUtils.isEmpty(Deencapsulation.getField(plan, "partitions")));
192+
}
193+
180194
@Test
181195
public void testCalculateNeedRefreshPartitionsSystemComplete() throws AnalysisException, JobException {
182196
MTMVTaskContext context = new MTMVTaskContext(MTMVTaskTriggerMode.SYSTEM);

‎regression-test/data/mtmv_p0/ivm/test_ivm_chained_mtmv_2.out‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,8 +33,9 @@
3333
3 35
3434
4 40
3535
5 50
36+
3637
-- !child_incremental_after_root_complete --
37-
FAILED IVM incremental refresh failed for mv=child_ivm_commit_tso, reason=BINLOG_BROKEN, detail=Stream binlog is marked as broken
38+
FAILED IVM baseline rebuild is pending for mv=child_ivm_commit_tso; run an AUTO or COMPLETE refresh first
3839

3940
-- !child_auto_refresh_mode --
4041
COMPLETE
@@ -43,3 +44,4 @@ COMPLETE
4344
3 35
4445
4 40
4546
5 50
47+

‎regression-test/data/mtmv_p0/test_commit_mtmv.out‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -5,15 +5,15 @@
55
3 2017-03-15 3
66

77
-- !task1 --
8-
{"triggerMode":"COMMIT","partitions":[],"isComplete":false,"refreshMode":"AUTO"}
8+
{"triggerMode":"COMMIT","isComplete":false,"md":true}
99

1010
-- !mv2 --
1111
1 2017-01-15 1
1212
2 2017-02-15 2
1313
3 2017-03-15 3
1414

1515
-- !task2 --
16-
{"triggerMode":"COMMIT","partitions":[],"isComplete":false,"refreshMode":"AUTO"}
16+
{"triggerMode":"COMMIT","isComplete":false,"md":true}
1717

1818
-- !mv1_pause --
1919
1 2017-01-15 1
@@ -65,19 +65,19 @@
6565
1 1 60
6666

6767
-- !task_sag --
68-
{"triggerMode":"COMMIT","partitions":[],"isComplete":false,"refreshMode":"AUTO"}
68+
{"triggerMode":"COMMIT","isComplete":false,"md":true}
6969

7070
-- !mv_sag1 --
7171
1 1 60
7272

7373
-- !task_sag1 --
74-
{"triggerMode":"COMMIT","partitions":[],"isComplete":false,"refreshMode":"AUTO"}
74+
{"triggerMode":"COMMIT","isComplete":false,"md":true}
7575

7676
-- !mv_sag2 --
7777
1 1 60
7878
1 2 70
7979
2 1 70
8080

8181
-- !task_sag2 --
82-
{"triggerMode":"COMMIT","partitions":[],"isComplete":false,"refreshMode":"AUTO"}
82+
{"triggerMode":"COMMIT","isComplete":false,"md":true}
8383

‎regression-test/data/mtmv_p0/test_create_mtmv_with_view.out‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,10 +22,9 @@ COMPLETE
2222
2 2
2323

2424
-- !trigger_table_not_need_refresh --
25-
COMPLETE
25+
NOT_REFRESH
2626

2727
-- !after_trigger_table --
2828
1 1
2929
2 2
30-
3 3
3130

‎regression-test/suites/mtmv_p0/ivm/test_ivm_fallback_stream_multi_batch.groovy‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ suite("test_ivm_fallback_stream_multi_batch", "nonConcurrent") {
2020
return
2121
}
2222

23-
def forcedFallbackDebugPoint = "IvmRefreshManager.doRefresh.force_fallback_reason"
23+
def forcedFallbackDebugPoint = "IvmIncrRefreshManager.doRefresh.force_fallback_reason"
2424

2525
GetDebugPoint().disableDebugPointForAllFEs(forcedFallbackDebugPoint)
2626

‎regression-test/suites/mtmv_p0/ivm/test_ivm_fallback_stream_multi_batch_dup.groovy‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ suite("test_ivm_fallback_stream_multi_batch_dup", "nonConcurrent") {
2020
return
2121
}
2222

23-
def forcedFallbackDebugPoint = "IvmRefreshManager.doRefresh.force_fallback_reason"
23+
def forcedFallbackDebugPoint = "IvmIncrRefreshManager.doRefresh.force_fallback_reason"
2424
GetDebugPoint().disableDebugPointForAllFEs(forcedFallbackDebugPoint)
2525

2626
sql """drop materialized view if exists ivm_fbs_dup_mv;"""

‎regression-test/suites/mtmv_p0/ivm/test_ivm_replace_stream_cleanup.groovy‎

Lines changed: 20 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,22 @@ suite("test_ivm_replace_stream_cleanup") {
6363
AS SELECT k1, v1 FROM ivm_replace_new_base
6464
"""
6565

66+
def streamRows = sql """
67+
SELECT STREAM_NAME, BASE_TABLE_NAME
68+
FROM information_schema.table_streams
69+
WHERE DB_NAME = '${context.dbName}'
70+
AND BASE_TABLE_NAME IN ('ivm_replace_old_base', 'ivm_replace_new_base')
71+
AND starts_with(STREAM_NAME, '__doris_ivm_stream_')
72+
"""
73+
assertEquals(2, streamRows.size())
74+
def streamsByBaseTable = streamRows.collectEntries { row ->
75+
[(row[1].toString()): row[0].toString()]
76+
}
77+
def oldStreamName = streamsByBaseTable['ivm_replace_old_base']
78+
def newStreamName = streamsByBaseTable['ivm_replace_new_base']
79+
assertNotNull(oldStreamName)
80+
assertNotNull(newStreamName)
81+
6682
sql "INSERT INTO ivm_replace_old_base VALUES (1, 10), (2, 20)"
6783
sql "REFRESH MATERIALIZED VIEW ivm_replace_old_mv INCREMENTAL"
6884
waitingMTMVTaskFinishedByMvName("ivm_replace_old_mv")
@@ -75,12 +91,8 @@ suite("test_ivm_replace_stream_cleanup") {
7591

7692
qt_streams_before_replace """
7793
SELECT
78-
COUNT(DISTINCT CASE WHEN RIGHT(STREAM_NAME,
79-
LENGTH('ivm_replace_old_base'))
80-
= 'ivm_replace_old_base' THEN STREAM_NAME END) AS old_streams,
81-
COUNT(DISTINCT CASE WHEN RIGHT(STREAM_NAME,
82-
LENGTH('ivm_replace_new_base'))
83-
= 'ivm_replace_new_base' THEN STREAM_NAME END) AS new_streams
94+
COUNT(DISTINCT CASE WHEN STREAM_NAME = '${oldStreamName}' THEN STREAM_NAME END) AS old_streams,
95+
COUNT(DISTINCT CASE WHEN STREAM_NAME = '${newStreamName}' THEN STREAM_NAME END) AS new_streams
8496
FROM information_schema.table_stream_consumption
8597
WHERE DB_NAME = '${context.dbName}'
8698
"""
@@ -93,12 +105,8 @@ suite("test_ivm_replace_stream_cleanup") {
93105

94106
qt_streams_after_replace """
95107
SELECT
96-
COUNT(DISTINCT CASE WHEN RIGHT(STREAM_NAME,
97-
LENGTH('ivm_replace_old_base'))
98-
= 'ivm_replace_old_base' THEN STREAM_NAME END) AS old_streams,
99-
COUNT(DISTINCT CASE WHEN RIGHT(STREAM_NAME,
100-
LENGTH('ivm_replace_new_base'))
101-
= 'ivm_replace_new_base' THEN STREAM_NAME END) AS new_streams
108+
COUNT(DISTINCT CASE WHEN STREAM_NAME = '${oldStreamName}' THEN STREAM_NAME END) AS old_streams,
109+
COUNT(DISTINCT CASE WHEN STREAM_NAME = '${newStreamName}' THEN STREAM_NAME END) AS new_streams
102110
FROM information_schema.table_stream_consumption
103111
WHERE DB_NAME = '${context.dbName}'
104112
"""

‎regression-test/suites/mtmv_p0/test_mtmv_property.groovy‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -172,7 +172,7 @@ suite("test_mtmv_property","mtmv") {
172172
assertTrue(desc_res[1][3] == "false")
173173

174174
// enable_unique_key_merge_on_write
175-
try {
175+
test {
176176
sql """
177177
CREATE MATERIALIZED VIEW mv_unique_key_merge_on_write
178178
BUILD DEFERRED REFRESH AUTO ON MANUAL
@@ -184,9 +184,7 @@ suite("test_mtmv_property","mtmv") {
184184
AS
185185
SELECT * from ${tableName};
186186
"""
187-
} catch (Exception e) {
188-
log.info(e.getMessage())
189-
assertTrue(e.getMessage().contains("Unknown properties"))
187+
exception "enable_unique_key_merge_on_write property only support unique key table"
190188
}
191189

192190
// group_commit_interval_ms

0 commit comments

Comments
 (0)