Skip to content

Commit 7129a3e

Browse files
authored
[fix](ivm) Propagate and compensate failures of the IVM excluded-trigger-tables ALTER (#67665)
Follow-up of the #62606 review round. ALTER MATERIALIZED VIEW ... SET ('excluded_trigger_tables' = ...) on an IVM MTMV transitions the base-table streams alongside the property. That transition used to run inside `processAlterMTMV`, whose catch swallowed any `UserException`: a mid-transition failure (e.g. a stream create throwing after an earlier create was already journaled) reported success to the client while the property stayed unchanged and stray streams remained. Key changes: - Live ALTER PROPERTY statements now run through the new `Alter.processAlterMTMVProperty`, which propagates failures to the client; the journal replay path keeps the tolerant `processAlterMTMV` behavior. - The stream transition is reordered: create the streams of newly un-excluded bases first (compensated by dropping exactly the streams created in this call on failure), then apply the property, then best-effort drop the streams of newly excluded bases - a failed drop only leaks a stream of an already excluded table and never fails the ALTER after the property took effect. - Two test debug points (count-based, independent of the base-table iteration order) inject stream create/drop failures. Tests: `AlterMTMVTest` two new cases covering create-failure compensation and drop-failure best effort with multi-table excluded-set changes; the full class (24 tests) passes. Trace issue: #65418
1 parent c1e522e commit 7129a3e

4 files changed

Lines changed: 264 additions & 24 deletions

File tree

‎fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java‎

Lines changed: 150 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,7 @@
4343
import org.apache.doris.common.MetaNotFoundException;
4444
import org.apache.doris.common.UserException;
4545
import org.apache.doris.common.cache.NereidsSqlCacheManager;
46+
import org.apache.doris.common.util.DebugPointUtil;
4647
import org.apache.doris.common.util.DynamicPartitionUtil;
4748
import org.apache.doris.common.util.MetaLockUtils;
4849
import org.apache.doris.common.util.PropertyAnalyzer;
@@ -128,6 +129,15 @@
128129
public class Alter {
129130
private static final Logger LOG = LogManager.getLogger(Alter.class);
130131

132+
// Test hooks for the IVM excluded_trigger_tables stream transition (see
133+
// alterIvmExcludedTriggerTables): when enabled, the "value" param is the number of
134+
// successful stream creates/drops to allow first (e.g. "1" allows one and fails the
135+
// next), independent of the base-table iteration order.
136+
public static final String DEBUG_POINT_CREATE_EXCLUDED_STREAM_FAIL =
137+
"Alter.alterIvmExcludedTriggerTables.create_stream_fail";
138+
public static final String DEBUG_POINT_DROP_EXCLUDED_STREAM_FAIL =
139+
"Alter.alterIvmExcludedTriggerTables.drop_stream_fail";
140+
131141
private AlterHandler schemaChangeHandler;
132142
private AlterHandler materializedViewHandler;
133143
private SystemHandler systemHandler;
@@ -1315,16 +1325,10 @@ public void processAlterMTMV(AlterMTMV alterMTMV, boolean isReplay) {
13151325
mtmv.alterStatus(alterMTMV.getStatus());
13161326
break;
13171327
case ALTER_PROPERTY:
1318-
if (mtmv.isIvm() && alterMTMV.getMvProperties().containsKey(
1319-
PropertyAnalyzer.PROPERTIES_EXCLUDED_TRIGGER_TABLES)) {
1320-
Set<TableNameInfo> oldExcludedTriggerTables = mtmv.getExcludedTriggerTables();
1321-
Set<TableNameInfo> newExcludedTriggerTables = MTMVPropertyUtil.parseTableNameInfos(
1322-
alterMTMV.getMvProperties().get(
1323-
PropertyAnalyzer.PROPERTIES_EXCLUDED_TRIGGER_TABLES));
1324-
updateIvmStreamsForExcludedTables(db, mtmv, oldExcludedTriggerTables,
1325-
newExcludedTriggerTables, isReplay);
1326-
}
1327-
mtmv.alterMvProperties(alterMTMV, isReplay);
1328+
// Live ALTER PROPERTY statements call processAlterMTMVProperty directly
1329+
// (see Env.alterMTMVProperty) so that failures surface to the client;
1330+
// this path only replays the journaled op, where errors stay tolerated.
1331+
processAlterMTMVProperty(alterMTMV, isReplay);
13281332
return;
13291333
case ADD_TASK:
13301334
if (!mtmv.addTaskResult(alterMTMV, isReplay)) {
@@ -1355,28 +1359,141 @@ public void processAlterMTMV(AlterMTMV alterMTMV, boolean isReplay) {
13551359
}
13561360
}
13571361

1358-
private void updateIvmStreamsForExcludedTables(Database db, MTMV mtmv,
1362+
/**
1363+
* Applies an ALTER PROPERTY op. Live statements call this method directly through
1364+
* {@code Env.alterMTMVProperty} so that a failure (e.g. a partial IVM stream
1365+
* transition) propagates to the client instead of being swallowed; the replay path
1366+
* runs it through {@link #processAlterMTMV} where errors are tolerated.
1367+
*/
1368+
public void processAlterMTMVProperty(AlterMTMV alterMTMV, boolean isReplay) throws UserException {
1369+
MTMV mtmv;
1370+
Database db = Env.getCurrentInternalCatalog().getDbOrDdlException(alterMTMV.getMvName().getDb());
1371+
// Fail before touching anything when the database is being dropped: a later step
1372+
// (e.g. the stream drop under the db write lock) would otherwise fail after the
1373+
// property was already applied.
1374+
if (db.isDropped()) {
1375+
throw new DdlException("unknown db, dbName=" + db.getFullName());
1376+
}
1377+
mtmv = (MTMV) db.getTableOrMetaException(alterMTMV.getMvName().getTbl(), TableType.MATERIALIZED_VIEW);
1378+
if (mtmv.isIvm() && alterMTMV.getMvProperties().containsKey(
1379+
PropertyAnalyzer.PROPERTIES_EXCLUDED_TRIGGER_TABLES)) {
1380+
Set<TableNameInfo> oldExcludedTriggerTables = mtmv.getExcludedTriggerTables();
1381+
Set<TableNameInfo> newExcludedTriggerTables = MTMVPropertyUtil.parseTableNameInfos(
1382+
alterMTMV.getMvProperties().get(PropertyAnalyzer.PROPERTIES_EXCLUDED_TRIGGER_TABLES));
1383+
alterIvmExcludedTriggerTables(db, mtmv, oldExcludedTriggerTables,
1384+
newExcludedTriggerTables, alterMTMV, isReplay);
1385+
return;
1386+
}
1387+
mtmv.alterMvProperties(alterMTMV, isReplay);
1388+
}
1389+
1390+
private void alterIvmExcludedTriggerTables(Database db, MTMV mtmv,
1391+
Set<TableNameInfo> oldExcludedTriggerTables, Set<TableNameInfo> newExcludedTriggerTables,
1392+
AlterMTMV alterMTMV, boolean isReplay) throws UserException {
1393+
// Step 1: create streams for base tables leaving the excluded set (skipped on
1394+
// replay, each create journals itself). A mid-loop failure is compensated by
1395+
// dropping exactly the streams created here, leaving the previous property in
1396+
// effect without stray streams.
1397+
List<String> createdStreamNames = new ArrayList<>();
1398+
if (!isReplay) {
1399+
try {
1400+
createStreamsForUnExcludedTables(db, mtmv, oldExcludedTriggerTables,
1401+
newExcludedTriggerTables, createdStreamNames);
1402+
} catch (UserException e) {
1403+
try {
1404+
dropStreamsByNames(db, createdStreamNames, false);
1405+
} catch (UserException compensateException) {
1406+
// Keep the original failure for the client; a failed compensation
1407+
// leaves stray streams that the idempotent retry of the ALTER
1408+
// cleans up.
1409+
LOG.error("failed to compensate streams created for mv={} after create failure",
1410+
mtmv.getName(), compensateException);
1411+
}
1412+
throw e;
1413+
}
1414+
}
1415+
// Step 2: apply the property, the source of truth for refresh.
1416+
mtmv.alterMvProperties(alterMTMV, isReplay);
1417+
// Step 3: drop streams of base tables that just joined the excluded set. This
1418+
// runs after the property on purpose: a failed drop then only leaks the stream
1419+
// of a table that is already excluded (harmless, and stream leaks are a
1420+
// pre-existing risk), whereas dropping first could remove the stream of a table
1421+
// that is still active under the old property.
1422+
try {
1423+
dropStreamsForExcludedTables(db, mtmv, newExcludedTriggerTables, isReplay);
1424+
} catch (UserException e) {
1425+
LOG.warn("failed to drop IVM streams for excluded trigger tables of mv={}, "
1426+
+ "the streams may leak: {}", mtmv.getName(), e.getMessage());
1427+
}
1428+
}
1429+
1430+
private void createStreamsForUnExcludedTables(Database db, MTMV mtmv,
13591431
Set<TableNameInfo> oldExcludedTriggerTables, Set<TableNameInfo> newExcludedTriggerTables,
1360-
boolean isReplay) throws UserException {
1432+
List<String> createdStreamNames) throws UserException {
13611433
MTMVRelation relation = mtmv.getRelation();
13621434
if (relation == null || relation.getBaseTables() == null) {
13631435
return;
13641436
}
1365-
Set<BaseTableInfo> baseTables = relation.getBaseTables();
1366-
for (BaseTableInfo baseTableInfo : baseTables) {
1437+
int createdCount = 0;
1438+
for (BaseTableInfo baseTableInfo : relation.getBaseTables()) {
13671439
TableNameInfo baseTableName = new TableNameInfo(baseTableInfo.getCtlName(),
13681440
baseTableInfo.getDbName(), baseTableInfo.getTableName());
1369-
boolean wasExcluded = MTMVPartitionUtil.isTableExcluded(
1370-
oldExcludedTriggerTables, baseTableName);
1371-
boolean isExcluded = MTMVPartitionUtil.isTableExcluded(
1372-
newExcludedTriggerTables, baseTableName);
1373-
if (wasExcluded && !isExcluded) {
1374-
if (!isReplay) {
1375-
TableIf baseTable = MTMVUtil.getTable(baseTableInfo);
1376-
CreateMTMVCommand.createTableStream(ConnectContext.get(), db, mtmv, baseTable);
1441+
if (!MTMVPartitionUtil.isTableExcluded(oldExcludedTriggerTables, baseTableName)
1442+
|| MTMVPartitionUtil.isTableExcluded(newExcludedTriggerTables, baseTableName)) {
1443+
continue;
1444+
}
1445+
failCreateStreamIfDebugPointed(createdCount);
1446+
TableIf baseTable = MTMVUtil.getTable(baseTableInfo);
1447+
CreateMTMVCommand.createTableStream(ConnectContext.get(), db, mtmv, baseTable);
1448+
createdStreamNames.add(IvmUtil.streamName(mtmv.getId(), baseTable.getFullQualifiers()));
1449+
createdCount++;
1450+
}
1451+
}
1452+
1453+
/** Fails before the (allowedCreates+1)-th create when the create debug point is enabled. */
1454+
private void failCreateStreamIfDebugPointed(int createdCount) throws DdlException {
1455+
String allowCreates = DebugPointUtil.getDebugParamOrDefault(DEBUG_POINT_CREATE_EXCLUDED_STREAM_FAIL, "");
1456+
if (!allowCreates.isEmpty() && createdCount >= Integer.parseInt(allowCreates)) {
1457+
throw new DdlException("debug point: creating IVM stream for an excluded-trigger base table "
1458+
+ "failed after " + allowCreates + " successful create(s)");
1459+
}
1460+
}
1461+
1462+
/** Drops the named stream tables under the db write lock; missing/non-stream tables are skipped. */
1463+
private void dropStreamsByNames(Database db, List<String> streamNames, boolean isReplay)
1464+
throws UserException {
1465+
if (streamNames.isEmpty()) {
1466+
return;
1467+
}
1468+
db.writeLockOrDdlException();
1469+
try {
1470+
for (String streamName : streamNames) {
1471+
TableIf streamTable = db.getTableNullable(streamName);
1472+
if (!(streamTable instanceof BaseTableStream)) {
1473+
continue;
1474+
}
1475+
Table table = (Table) streamTable;
1476+
table.writeLock();
1477+
try {
1478+
Env.getCurrentEnv().unprotectDropTable(db, table, true, isReplay, 0L);
1479+
} finally {
1480+
table.writeUnlock();
13771481
}
1482+
LOG.info("dropped IVM stream {}", streamName);
13781483
}
1484+
} finally {
1485+
db.writeUnlock();
1486+
}
1487+
}
1488+
1489+
private void dropStreamsForExcludedTables(Database db, MTMV mtmv,
1490+
Set<TableNameInfo> newExcludedTriggerTables, boolean isReplay) throws UserException {
1491+
MTMVRelation relation = mtmv.getRelation();
1492+
if (relation == null || relation.getBaseTables() == null) {
1493+
return;
13791494
}
1495+
Set<BaseTableInfo> baseTables = relation.getBaseTables();
1496+
int droppedCount = 0;
13801497
db.writeLockOrDdlException();
13811498
try {
13821499
for (BaseTableInfo baseTableInfo : baseTables) {
@@ -1385,6 +1502,7 @@ private void updateIvmStreamsForExcludedTables(Database db, MTMV mtmv,
13851502
if (!MTMVPartitionUtil.isTableExcluded(newExcludedTriggerTables, baseTableName)) {
13861503
continue;
13871504
}
1505+
failDropStreamIfDebugPointed(droppedCount);
13881506
List<String> baseTableFullQualifiers = baseTableInfo.toList();
13891507
String streamName = IvmUtil.streamName(mtmv.getId(), baseTableFullQualifiers);
13901508
TableIf streamTable = db.getTableNullable(streamName);
@@ -1409,9 +1527,19 @@ private void updateIvmStreamsForExcludedTables(Database db, MTMV mtmv,
14091527
}
14101528
LOG.info("dropped IVM stream {} because its base table is excluded from MTMV",
14111529
streamName);
1530+
droppedCount++;
14121531
}
14131532
} finally {
14141533
db.writeUnlock();
14151534
}
14161535
}
1536+
1537+
/** Fails before the (allowedDrops+1)-th drop when the drop debug point is enabled. */
1538+
private void failDropStreamIfDebugPointed(int droppedCount) throws DdlException {
1539+
String allowDrops = DebugPointUtil.getDebugParamOrDefault(DEBUG_POINT_DROP_EXCLUDED_STREAM_FAIL, "");
1540+
if (!allowDrops.isEmpty() && droppedCount >= Integer.parseInt(allowDrops)) {
1541+
throw new DdlException("debug point: dropping an IVM stream of an excluded-trigger base table "
1542+
+ "failed after " + allowDrops + " successful drop(s)");
1543+
}
1544+
}
14171545
}

‎fe/fe-core/src/main/java/org/apache/doris/catalog/Database.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -176,6 +176,11 @@ public void unmarkDropped() {
176176
isDropped = false;
177177
}
178178

179+
/** True while the database is being dropped (before it is removed from the catalog). */
180+
public boolean isDropped() {
181+
return isDropped;
182+
}
183+
179184
public void readLock() {
180185
this.rwLock.readLock().lock();
181186
}

‎fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7658,10 +7658,12 @@ public void alterMTMVRefreshInfo(AlterMTMVRefreshInfo info) {
76587658
this.alter.processAlterMTMV(alter, false);
76597659
}
76607660

7661-
public void alterMTMVProperty(AlterMTMVPropertyInfo info) {
7661+
public void alterMTMVProperty(AlterMTMVPropertyInfo info) throws UserException {
76627662
AlterMTMV alter = new AlterMTMV(info.getMvName(), MTMVAlterOpType.ALTER_PROPERTY);
76637663
alter.setMvProperties(info.getProperties());
7664-
this.alter.processAlterMTMV(alter, false);
7664+
// Runs outside the tolerant processAlterMTMV catch so that failures (e.g. a
7665+
// partial IVM excluded-trigger-tables stream transition) reach the client.
7666+
this.alter.processAlterMTMVProperty(alter, false);
76657667
}
76667668

76677669
public void alterMTMVStatus(TableNameInfo mvName, MTMVStatus status) {

0 commit comments

Comments
 (0)