Skip to content

Commit a4dae0b

Browse files
authored
[fix](fe) Clean up IVM streams after materialized view replacement (#25)
Replacing an incremental materialized view with `swap=false` removes the old MV but leaves its hidden table streams registered in the database and `TableStreamManager`. The orphaned metadata also remains after journal replay. **Root cause:** `Alter.replaceTableInternal()` bypasses the normal MTMV drop path, so the IVM stream cleanup in `InternalCatalog.unprotectDropTable()` is not invoked. **Change Summary:** | File | Change Description | |------|-------------------| | `Alter.java` | Clean up the replaced IVM streams for non-swap replacement on both Leader handling and journal replay | | `InternalCatalog.java` | Extract reusable IVM stream cleanup and hold each stream write lock during deletion | | `AlterMTMVTest.java` | Cover non-swap cleanup, replay cleanup, and swap preservation | | Regression test | Verify the old stream is removed and the replacement stream remains | ```mermaid graph LR A[REPLACE MTMV] --> B{swap} B -->|true| C[Keep both IVM streams] B -->|false| D[Drop old IVM streams] E[Journal replay] --> D D --> F[Clean database and TableStreamManager] ```
1 parent 645df5d commit a4dae0b

5 files changed

Lines changed: 247 additions & 16 deletions

File tree

‎fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -816,6 +816,10 @@ private void replaceTableInternal(Database db, OlapTable origTable, OlapTable ne
816816
if (!isReplay) {
817817
Env.getCurrentEnv().getConstraintManager().checkNoReferencingForeignKeys(origTableInfo);
818818
}
819+
if (origTable instanceof MTMV) {
820+
Env.getCurrentInternalCatalog().unprotectDropIvmStreams(
821+
db, (MTMV) origTable, isForce, isReplay);
822+
}
819823
Env.getCurrentEnv().getConstraintManager().dropAndRenameConstraints(origTableInfo, newTableInfo);
820824
}
821825

‎fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java‎

Lines changed: 23 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -1042,22 +1042,7 @@ public boolean unprotectDropTable(Database db, Table table, boolean isForceDrop,
10421042
long recycleTime) throws DdlException {
10431043
if (table instanceof MTMV) {
10441044
Env.getCurrentEnv().getMtmvService().dropJob((MTMV) table, isReplay);
1045-
// Drop associated IVM streams
1046-
MTMV mtmv = (MTMV) table;
1047-
MTMVRelation relation = mtmv.getRelation();
1048-
if (relation != null && mtmv.isIvm()) {
1049-
Set<BaseTableInfo> baseTables = relation.getBaseTables();
1050-
if (baseTables != null) {
1051-
for (BaseTableInfo baseTableInfo : baseTables) {
1052-
String streamName = IvmUtil.streamName(mtmv.getId(), baseTableInfo.getTableName());
1053-
TableIf streamTable = db.getTableNullable(streamName);
1054-
if (streamTable != null) {
1055-
unprotectDropTable(db, (Table) streamTable, isForceDrop, isReplay, 0L);
1056-
LOG.info("dropped stream {} associated with MTMV {}", streamName, mtmv.getName());
1057-
}
1058-
}
1059-
}
1060-
}
1045+
unprotectDropIvmStreams(db, (MTMV) table, isForceDrop, isReplay);
10611046
}
10621047
if (table instanceof View) {
10631048
Env.getCurrentEnv().getMtmvService().dropView(new BaseTableInfo(table));
@@ -1080,6 +1065,28 @@ public boolean unprotectDropTable(Database db, Table table, boolean isForceDrop,
10801065
return true;
10811066
}
10821067

1068+
public void unprotectDropIvmStreams(Database db, MTMV mtmv, boolean isForceDrop, boolean isReplay)
1069+
throws DdlException {
1070+
MTMVRelation relation = mtmv.getRelation();
1071+
if (!mtmv.isIvm() || relation == null || relation.getBaseTables() == null) {
1072+
return;
1073+
}
1074+
for (BaseTableInfo baseTableInfo : relation.getBaseTables()) {
1075+
String streamName = IvmUtil.streamName(mtmv.getId(), baseTableInfo.getTableName());
1076+
TableIf streamTable = db.getTableNullable(streamName);
1077+
if (streamTable != null) {
1078+
Table stream = (Table) streamTable;
1079+
stream.writeLock();
1080+
try {
1081+
unprotectDropTable(db, stream, isForceDrop, isReplay, 0L);
1082+
LOG.info("dropped stream {} associated with MTMV {}", streamName, mtmv.getName());
1083+
} finally {
1084+
stream.writeUnlock();
1085+
}
1086+
}
1087+
}
1088+
}
1089+
10831090
private void dropTable(Database db, long tableId, boolean isForceDrop, boolean isReplay,
10841091
Long recycleTime) throws MetaNotFoundException, DdlException {
10851092
Table table = db.getTableOrMetaException(tableId);

‎fe/fe-core/src/test/java/org/apache/doris/mtmv/AlterMTMVTest.java‎

Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717

1818
package org.apache.doris.mtmv;
1919

20+
import org.apache.doris.catalog.Database;
2021
import org.apache.doris.catalog.Env;
2122
import org.apache.doris.catalog.MTMV;
2223
import org.apache.doris.catalog.Table;
@@ -27,6 +28,7 @@
2728
import org.apache.doris.mtmv.ivm.IvmInfo;
2829
import org.apache.doris.mtmv.ivm.IvmUtil;
2930
import org.apache.doris.nereids.exceptions.AnalysisException;
31+
import org.apache.doris.persist.ReplaceTableOperationLog;
3032
import org.apache.doris.utframe.TestWithFeService;
3133

3234
import org.junit.jupiter.api.Assertions;
@@ -334,6 +336,96 @@ public void testCreateIncrementalMtmvAutoCreatesStream() throws Exception {
334336
"Should be an OlapTableStream");
335337
}
336338

339+
@Test
340+
public void testReplaceIvmWithoutSwapRemovesOldStreams() throws Exception {
341+
createDatabaseAndUse("ivm_replace_stream_test");
342+
createIvmBaseAndMv("ivm_replace_stream_test", "replace_old_base", "replace_old_mv");
343+
createIvmBaseAndMv("ivm_replace_stream_test", "replace_new_base", "replace_new_mv");
344+
345+
Database db = Env.getCurrentInternalCatalog().getDbOrDdlException("ivm_replace_stream_test");
346+
MTMV oldMtmv = (MTMV) db.getTableOrMetaException("replace_old_mv");
347+
MTMV newMtmv = (MTMV) db.getTableOrMetaException("replace_new_mv");
348+
String oldStreamName = IvmUtil.streamName(oldMtmv.getId(), "replace_old_base");
349+
String newStreamName = IvmUtil.streamName(newMtmv.getId(), "replace_new_base");
350+
long oldStreamId = db.getTableOrMetaException(oldStreamName).getId();
351+
long newStreamId = db.getTableOrMetaException(newStreamName).getId();
352+
353+
alterMv("ALTER MATERIALIZED VIEW replace_old_mv REPLACE WITH MATERIALIZED VIEW replace_new_mv "
354+
+ "PROPERTIES ('swap' = 'false')");
355+
356+
Assertions.assertNull(db.getTableNullable(oldStreamName));
357+
Assertions.assertFalse(Env.getCurrentEnv().getTableStreamManager().getTableStreamIds(db)
358+
.contains(oldStreamId));
359+
Assertions.assertNotNull(db.getTableNullable(newStreamName));
360+
Assertions.assertTrue(Env.getCurrentEnv().getTableStreamManager().getTableStreamIds(db)
361+
.contains(newStreamId));
362+
Assertions.assertEquals(newMtmv.getId(), db.getTableOrMetaException("replace_old_mv").getId());
363+
Assertions.assertNull(db.getTableNullable("replace_new_mv"));
364+
}
365+
366+
@Test
367+
public void testReplayReplaceIvmWithoutSwapRemovesOldStreams() throws Exception {
368+
createDatabaseAndUse("ivm_replay_replace_stream_test");
369+
createIvmBaseAndMv("ivm_replay_replace_stream_test", "replay_old_base", "replay_old_mv");
370+
createIvmBaseAndMv("ivm_replay_replace_stream_test", "replay_new_base", "replay_new_mv");
371+
372+
Database db = Env.getCurrentInternalCatalog().getDbOrDdlException("ivm_replay_replace_stream_test");
373+
MTMV oldMtmv = (MTMV) db.getTableOrMetaException("replay_old_mv");
374+
MTMV newMtmv = (MTMV) db.getTableOrMetaException("replay_new_mv");
375+
String oldStreamName = IvmUtil.streamName(oldMtmv.getId(), "replay_old_base");
376+
String newStreamName = IvmUtil.streamName(newMtmv.getId(), "replay_new_base");
377+
long oldStreamId = db.getTableOrMetaException(oldStreamName).getId();
378+
long newStreamId = db.getTableOrMetaException(newStreamName).getId();
379+
ReplaceTableOperationLog log = new ReplaceTableOperationLog(db.getId(), oldMtmv.getId(),
380+
oldMtmv.getName(), newMtmv.getId(), newMtmv.getName(), false, true);
381+
382+
Env.getCurrentEnv().getAlterInstance().replayReplaceTable(log);
383+
384+
Assertions.assertNull(db.getTableNullable(oldStreamName));
385+
Assertions.assertFalse(Env.getCurrentEnv().getTableStreamManager().getTableStreamIds(db)
386+
.contains(oldStreamId));
387+
Assertions.assertNotNull(db.getTableNullable(newStreamName));
388+
Assertions.assertTrue(Env.getCurrentEnv().getTableStreamManager().getTableStreamIds(db)
389+
.contains(newStreamId));
390+
}
391+
392+
@Test
393+
public void testReplaceIvmWithSwapKeepsBothStreams() throws Exception {
394+
createDatabaseAndUse("ivm_swap_replace_stream_test");
395+
createIvmBaseAndMv("ivm_swap_replace_stream_test", "swap_old_base", "swap_old_mv");
396+
createIvmBaseAndMv("ivm_swap_replace_stream_test", "swap_new_base", "swap_new_mv");
397+
398+
Database db = Env.getCurrentInternalCatalog().getDbOrDdlException("ivm_swap_replace_stream_test");
399+
MTMV oldMtmv = (MTMV) db.getTableOrMetaException("swap_old_mv");
400+
MTMV newMtmv = (MTMV) db.getTableOrMetaException("swap_new_mv");
401+
String oldStreamName = IvmUtil.streamName(oldMtmv.getId(), "swap_old_base");
402+
String newStreamName = IvmUtil.streamName(newMtmv.getId(), "swap_new_base");
403+
long oldStreamId = db.getTableOrMetaException(oldStreamName).getId();
404+
long newStreamId = db.getTableOrMetaException(newStreamName).getId();
405+
406+
alterMv("ALTER MATERIALIZED VIEW swap_old_mv REPLACE WITH MATERIALIZED VIEW swap_new_mv "
407+
+ "PROPERTIES ('swap' = 'true')");
408+
409+
Assertions.assertNotNull(db.getTableNullable(oldStreamName));
410+
Assertions.assertNotNull(db.getTableNullable(newStreamName));
411+
Assertions.assertTrue(Env.getCurrentEnv().getTableStreamManager().getTableStreamIds(db)
412+
.contains(oldStreamId));
413+
Assertions.assertTrue(Env.getCurrentEnv().getTableStreamManager().getTableStreamIds(db)
414+
.contains(newStreamId));
415+
}
416+
417+
private void createIvmBaseAndMv(String dbName, String baseTableName, String mvName) throws Exception {
418+
createTable("CREATE TABLE " + dbName + "." + baseTableName + " (k1 int, v1 int)\n"
419+
+ "DUPLICATE KEY(k1)\n"
420+
+ "DISTRIBUTED BY HASH(k1) BUCKETS 1\n"
421+
+ "PROPERTIES ('replication_num' = '1', 'binlog.enable' = 'true', 'binlog.format' = 'ROW')");
422+
createMvByNereids("CREATE MATERIALIZED VIEW " + dbName + "." + mvName + "\n"
423+
+ " BUILD DEFERRED REFRESH INCREMENTAL ON MANUAL\n"
424+
+ " DISTRIBUTED BY RANDOM BUCKETS 2\n"
425+
+ " PROPERTIES ('replication_num' = '1')\n"
426+
+ " AS SELECT k1, v1 FROM " + dbName + "." + baseTableName);
427+
}
428+
337429
@Test
338430
public void testCreateIncrementalMtmvExcludeTriggerTableSkipsStream() throws Exception {
339431
createDatabaseAndUse("stream_excl_test");
Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
-- This file is automatically generated. You should know what you did if you want to edit this
2+
-- !old_mv_before_replace --
3+
1 10
4+
2 20
5+
6+
-- !new_mv_before_replace --
7+
10 100
8+
9+
-- !streams_before_replace --
10+
1 1
11+
12+
-- !streams_after_replace --
13+
0 1
14+
15+
-- !mv_after_replace --
16+
10 100
17+
20 200
18+
Lines changed: 110 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,110 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
suite("test_ivm_replace_stream_cleanup") {
19+
sql "DROP MATERIALIZED VIEW IF EXISTS ivm_replace_old_mv"
20+
sql "DROP MATERIALIZED VIEW IF EXISTS ivm_replace_new_mv"
21+
sql "DROP TABLE IF EXISTS ivm_replace_old_base"
22+
sql "DROP TABLE IF EXISTS ivm_replace_new_base"
23+
24+
sql """
25+
CREATE TABLE ivm_replace_old_base (
26+
k1 INT,
27+
v1 INT
28+
)
29+
DUPLICATE KEY(k1)
30+
DISTRIBUTED BY HASH(k1) BUCKETS 1
31+
PROPERTIES (
32+
'replication_num' = '1',
33+
'binlog.enable' = 'true',
34+
'binlog.format' = 'ROW'
35+
)
36+
"""
37+
sql """
38+
CREATE TABLE ivm_replace_new_base (
39+
k1 INT,
40+
v1 INT
41+
)
42+
DUPLICATE KEY(k1)
43+
DISTRIBUTED BY HASH(k1) BUCKETS 1
44+
PROPERTIES (
45+
'replication_num' = '1',
46+
'binlog.enable' = 'true',
47+
'binlog.format' = 'ROW'
48+
)
49+
"""
50+
51+
sql """
52+
CREATE MATERIALIZED VIEW ivm_replace_old_mv
53+
BUILD DEFERRED REFRESH INCREMENTAL ON MANUAL
54+
DISTRIBUTED BY RANDOM BUCKETS 1
55+
PROPERTIES ('replication_num' = '1')
56+
AS SELECT k1, v1 FROM ivm_replace_old_base
57+
"""
58+
sql """
59+
CREATE MATERIALIZED VIEW ivm_replace_new_mv
60+
BUILD DEFERRED REFRESH INCREMENTAL ON MANUAL
61+
DISTRIBUTED BY RANDOM BUCKETS 1
62+
PROPERTIES ('replication_num' = '1')
63+
AS SELECT k1, v1 FROM ivm_replace_new_base
64+
"""
65+
66+
sql "INSERT INTO ivm_replace_old_base VALUES (1, 10), (2, 20)"
67+
sql "REFRESH MATERIALIZED VIEW ivm_replace_old_mv INCREMENTAL"
68+
waitingMTMVTaskFinishedByMvName("ivm_replace_old_mv")
69+
order_qt_old_mv_before_replace "SELECT k1, v1 FROM ivm_replace_old_mv ORDER BY k1"
70+
71+
sql "INSERT INTO ivm_replace_new_base VALUES (10, 100)"
72+
sql "REFRESH MATERIALIZED VIEW ivm_replace_new_mv INCREMENTAL"
73+
waitingMTMVTaskFinishedByMvName("ivm_replace_new_mv")
74+
order_qt_new_mv_before_replace "SELECT k1, v1 FROM ivm_replace_new_mv ORDER BY k1"
75+
76+
qt_streams_before_replace """
77+
SELECT
78+
COUNT(DISTINCT CASE WHEN RIGHT(STREAM_NAME,
79+
LENGTH('ivm_replace_old_base'))
80+
= 'ivm_replace_old_base' THEN STREAM_NAME END) AS old_streams,
81+
COUNT(DISTINCT CASE WHEN RIGHT(STREAM_NAME,
82+
LENGTH('ivm_replace_new_base'))
83+
= 'ivm_replace_new_base' THEN STREAM_NAME END) AS new_streams
84+
FROM information_schema.table_stream_consumption
85+
WHERE DB_NAME = '${context.dbName}'
86+
"""
87+
88+
sql """
89+
ALTER MATERIALIZED VIEW ivm_replace_old_mv
90+
REPLACE WITH MATERIALIZED VIEW ivm_replace_new_mv
91+
PROPERTIES ('swap' = 'false')
92+
"""
93+
94+
qt_streams_after_replace """
95+
SELECT
96+
COUNT(DISTINCT CASE WHEN RIGHT(STREAM_NAME,
97+
LENGTH('ivm_replace_old_base'))
98+
= 'ivm_replace_old_base' THEN STREAM_NAME END) AS old_streams,
99+
COUNT(DISTINCT CASE WHEN RIGHT(STREAM_NAME,
100+
LENGTH('ivm_replace_new_base'))
101+
= 'ivm_replace_new_base' THEN STREAM_NAME END) AS new_streams
102+
FROM information_schema.table_stream_consumption
103+
WHERE DB_NAME = '${context.dbName}'
104+
"""
105+
106+
sql "INSERT INTO ivm_replace_new_base VALUES (20, 200)"
107+
sql "REFRESH MATERIALIZED VIEW ivm_replace_old_mv INCREMENTAL"
108+
waitingMTMVTaskFinishedByMvName("ivm_replace_old_mv")
109+
order_qt_mv_after_replace "SELECT k1, v1 FROM ivm_replace_old_mv ORDER BY k1"
110+
}

0 commit comments

Comments
 (0)