Skip to content

Commit a560c74

Browse files
yujun777seawinde
andauthored
[feature](ivm) Support incremental view maintenance (IVM) for materialized tables (MTMV) (#62606)
### What problem does this PR solve? Implement Incremental View Maintenance (IVM) for materialized tables (MTMV) in Apache Doris. When a base table of an MTMV changes, only the affected rows are derived from the base-table deltas (row binlog / Table Stream) and transactionally applied to the materialized table, instead of recomputing the whole result with a full refresh. This enables maintaining near-real-time aggregation and join results at a fraction of the cost of full refresh. Supported operators: projection, filter, aggregate (COUNT/SUM/AVG/MIN/MAX, bitmap aggregates, expression arguments, bare GROUP BY, grouping sets), joins (INNER/CROSS/OUTER JOIN, nested join and self join), UNION ALL, subquery aliases, and OneRowRelation. Supported refresh: `REFRESH MATERIALIZED VIEW ... INCREMENTAL/PARTITIONS`, refresh explain, dry-run, and automatic full-refresh fallback when incremental maintenance is not valid (e.g. binlog broken). Lifecycle: stream creation/lifecycle integrated with MTMV, chained IVM MTMV support. This PR is the MTMV incremental-maintenance layer of the end-to-end incremental computation stack tracked by #65418. Issue Number: close #66719 Related issue: #65418 (tracking issue), #57921 (superseded umbrella issue) --------- Co-authored-by: seawinde <[email protected]>
1 parent 099d5bd commit a560c74

347 files changed

Lines changed: 48763 additions & 920 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎fe/fe-catalog/src/main/java/org/apache/doris/catalog/Column.java‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,11 +51,27 @@ public class Column implements GsonPostProcessable {
5151
public static final String HIDDEN_COLUMN_PREFIX = "__DORIS_";
5252
// all shadow indexes should have this prefix in name
5353
public static final String SHADOW_NAME_PREFIX = "__doris_shadow_";
54+
public static final String IVM_HIDDEN_COLUMN_PREFIX = "__DORIS_IVM_";
5455
// NOTE: you should name hidden column start with '__DORIS_' !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
5556
public static final String DELETE_SIGN = "__DORIS_DELETE_SIGN__";
5657
public static final String WHERE_SIGN = "__DORIS_WHERE_SIGN__";
5758
public static final String SEQUENCE_COL = "__DORIS_SEQUENCE_COL__";
5859
public static final String GLOBAL_ROWID_COL = "__DORIS_GLOBAL_ROWID_COL__";
60+
public static final String IVM_ROW_ID_COL = "__DORIS_IVM_ROW_ID_COL__";
61+
public static final String IVM_AGG_COUNT_COL = "__DORIS_IVM_AGG_COUNT_COL__";
62+
public static final String IVM_DML_FACTOR_COL = "__DORIS_IVM_DML_FACTOR_COL__";
63+
public static final String IVM_BASE_OP_COL = "__DORIS_IVM_BASE_OP_COL__";
64+
public static final String IVM_DELTA_GROUP_COUNT_COL = "__DORIS_IVM_DELTA_GROUP_COUNT_COL__";
65+
// Prefix for sink-level IVM identity key hidden columns (__DORIS_IVM_KEY_).
66+
public static final String IVM_KEY_COL_PREFIX = IVM_HIDDEN_COLUMN_PREFIX + "KEY_";
67+
// Prefix for union arm-index columns (__DORIS_IVM_UNION_ARM_INDEX_).
68+
public static final String IVM_UNION_ARM_INDEX_COL_PREFIX = IVM_HIDDEN_COLUMN_PREFIX + "UNION_ARM_INDEX_";
69+
// Prefix for union positional key columns (__DORIS_IVM_UNION_KEY_).
70+
public static final String IVM_UNION_KEY_COL_PREFIX = IVM_HIDDEN_COLUMN_PREFIX + "UNION_KEY_";
71+
// Suffix for the renamed base-table row-id column (__DORIS_IVM_{n}_ROW_ID_COL__), using the
72+
// standard IVM hidden-column prefix. Kept as an identity key when a cascading MV with only
73+
// row-id keys is scanned under a join.
74+
public static final String IVM_BASE_ROW_ID_COL_SUFFIX = "_ROW_ID_COL__";
5975
public static final String ROW_STORE_COL = "__DORIS_ROW_STORE_COL__";
6076
public static final String VERSION_COL = "__DORIS_VERSION_COL__";
6177
public static final String SKIP_BITMAP_COL = "__DORIS_SKIP_BITMAP_COL__";
@@ -285,6 +301,7 @@ public Column(String name, Type type, boolean isKey, AggregateType aggregateType
285301
false, null, null, Sets.newHashSet(), null);
286302
}
287303

304+
288305
public Column(String name, Type type, boolean isKey, AggregateType aggregateType, boolean isAllowNull,
289306
String defaultValue, String comment, boolean visible, int colUniqueId) {
290307
this(name, type, isKey, aggregateType, isAllowNull, -1, defaultValue, comment, visible, null, colUniqueId, null,

‎fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java‎

Lines changed: 96 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@
3535
import org.apache.doris.catalog.Tablet;
3636
import org.apache.doris.catalog.View;
3737
import org.apache.doris.catalog.info.TableNameInfo;
38+
import org.apache.doris.catalog.stream.BaseTableStream;
3839
import org.apache.doris.cloud.alter.CloudSchemaChangeHandler;
3940
import org.apache.doris.common.AnalysisException;
4041
import org.apache.doris.common.Config;
@@ -49,9 +50,15 @@
4950
import org.apache.doris.datasource.ExternalTable;
5051
import org.apache.doris.info.TableNameInfoUtils;
5152
import org.apache.doris.mtmv.BaseTableInfo;
53+
import org.apache.doris.mtmv.MTMVPartitionUtil;
54+
import org.apache.doris.mtmv.MTMVPropertyUtil;
55+
import org.apache.doris.mtmv.MTMVRelation;
56+
import org.apache.doris.mtmv.MTMVUtil;
57+
import org.apache.doris.mtmv.ivm.IvmUtil;
5258
import org.apache.doris.nereids.trees.plans.commands.AlterSystemCommand;
5359
import org.apache.doris.nereids.trees.plans.commands.AlterTableCommand;
5460
import org.apache.doris.nereids.trees.plans.commands.AlterViewCommand;
61+
import org.apache.doris.nereids.trees.plans.commands.CreateMTMVCommand;
5562
import org.apache.doris.nereids.trees.plans.commands.CreateMaterializedViewCommand;
5663
import org.apache.doris.nereids.trees.plans.commands.DropMaterializedViewCommand;
5764
import org.apache.doris.nereids.trees.plans.commands.info.AddColumnOp;
@@ -359,7 +366,6 @@ private boolean processAlterOlapTableInternal(List<AlterOp> alterOps, OlapTable
359366
Env.getCurrentEnv().getMtmvService()
360367
.alterTable(oldBaseTableInfo, newBaseTableInfo, currentAlterOps.hasReplaceTableOp());
361368
}
362-
363369
olapTable.writeLock();
364370
try {
365371
NereidsSqlCacheManager sqlCacheManager = Env.getCurrentEnv().getSqlCacheManager();
@@ -678,6 +684,12 @@ public void processReplaceTable(Database db, OlapTable origTable, String newTblN
678684
if (swapTable) {
679685
origTable.checkAndSetName(newTblName, true);
680686
}
687+
// REPLACE TABLE affects dependencies of both logical names. Persist both
688+
// barriers before changing the catalog or journaling the replacement.
689+
Env.getCurrentEnv().getMtmvService().getRelationManager().markIvmBaselineRebuild(
690+
new BaseTableInfo(origTable), "Base table replace changed IVM baseline");
691+
Env.getCurrentEnv().getMtmvService().getRelationManager().markIvmBaselineRebuild(
692+
new BaseTableInfo(olapNewTbl), "Base table replace changed IVM baseline");
681693
replaceTableInternal(db, origTable, olapNewTbl, swapTable, false, isForce);
682694
// write edit log
683695
ReplaceTableOperationLog log = new ReplaceTableOperationLog(db.getId(),
@@ -743,6 +755,10 @@ private void replaceTableInternal(Database db, OlapTable origTable, OlapTable ne
743755
if (!isReplay) {
744756
Env.getCurrentEnv().getConstraintManager().checkNoReferencingForeignKeys(origTableInfo);
745757
}
758+
if (origTable instanceof MTMV) {
759+
Env.getCurrentInternalCatalog().unprotectDropIvmStreams(
760+
db, (MTMV) origTable, isForce, isReplay);
761+
}
746762
Env.getCurrentEnv().getConstraintManager().dropAndRenameConstraints(origTableInfo, newTableInfo);
747763
}
748764

@@ -1267,7 +1283,6 @@ public AlterHandler getSystemHandler() {
12671283
public void processAlterMTMV(AlterMTMV alterMTMV, boolean isReplay) {
12681284
TableNameInfo tbl = alterMTMV.getMvName();
12691285
MTMV mtmv = null;
1270-
boolean alterSuccess = true;
12711286
try {
12721287
Database db = Env.getCurrentInternalCatalog().getDbOrDdlException(tbl.getDb());
12731288
mtmv = (MTMV) db.getTableOrMetaException(tbl.getTbl(), TableType.MATERIALIZED_VIEW);
@@ -1279,31 +1294,103 @@ public void processAlterMTMV(AlterMTMV alterMTMV, boolean isReplay) {
12791294
mtmv.alterStatus(alterMTMV.getStatus());
12801295
break;
12811296
case ALTER_PROPERTY:
1282-
mtmv.alterMvProperties(alterMTMV.getMvProperties());
1283-
break;
1297+
if (mtmv.isIvm() && alterMTMV.getMvProperties().containsKey(
1298+
PropertyAnalyzer.PROPERTIES_EXCLUDED_TRIGGER_TABLES)) {
1299+
Set<TableNameInfo> oldExcludedTriggerTables = mtmv.getExcludedTriggerTables();
1300+
Set<TableNameInfo> newExcludedTriggerTables = MTMVPropertyUtil.parseTableNameInfos(
1301+
alterMTMV.getMvProperties().get(
1302+
PropertyAnalyzer.PROPERTIES_EXCLUDED_TRIGGER_TABLES));
1303+
updateIvmStreamsForExcludedTables(db, mtmv, oldExcludedTriggerTables,
1304+
newExcludedTriggerTables, isReplay);
1305+
}
1306+
mtmv.alterMvProperties(alterMTMV, isReplay);
1307+
return;
12841308
case ADD_TASK:
1285-
alterSuccess = mtmv.addTaskResult(alterMTMV.getTask(), alterMTMV.getRelation(),
1286-
alterMTMV.getPartitionSnapshots(),
1287-
isReplay);
1309+
if (!mtmv.addTaskResult(alterMTMV, isReplay)) {
1310+
return;
1311+
}
12881312
// If it is not a replay thread, it means that the current service is already a new version
12891313
// and does not require compatibility
12901314
if (isReplay) {
12911315
mtmv.compatible(Env.getCurrentEnv().getCatalogMgr());
12921316
}
1317+
return;
1318+
case ALTER_IVM_INFO:
1319+
// Live IVM changes are journaled inside MTMV; this branch applies the journal snapshot.
1320+
mtmv.alterIvmInfo(alterMTMV.getIvmInfo());
12931321
break;
12941322
default:
12951323
throw new RuntimeException("Unknown type value: " + alterMTMV.getOpType());
12961324
}
12971325
if (alterMTMV.isNeedRebuildJob()) {
12981326
Env.getCurrentEnv().getMtmvService().alterJob(mtmv, isReplay);
12991327
}
1300-
// 4. log it and replay it in the follower
1301-
if (!isReplay && alterSuccess) {
1328+
if (!isReplay) {
13021329
Env.getCurrentEnv().getEditLog().logAlterMTMV(alterMTMV);
13031330
}
13041331
} catch (UserException e) {
13051332
// if MTMV has been dropped, ignore this exception
13061333
LOG.warn(e);
13071334
}
13081335
}
1336+
1337+
private void updateIvmStreamsForExcludedTables(Database db, MTMV mtmv,
1338+
Set<TableNameInfo> oldExcludedTriggerTables, Set<TableNameInfo> newExcludedTriggerTables,
1339+
boolean isReplay) throws UserException {
1340+
MTMVRelation relation = mtmv.getRelation();
1341+
if (relation == null || relation.getBaseTables() == null) {
1342+
return;
1343+
}
1344+
Set<BaseTableInfo> baseTables = relation.getBaseTables();
1345+
for (BaseTableInfo baseTableInfo : baseTables) {
1346+
TableNameInfo baseTableName = new TableNameInfo(baseTableInfo.getCtlName(),
1347+
baseTableInfo.getDbName(), baseTableInfo.getTableName());
1348+
boolean wasExcluded = MTMVPartitionUtil.isTableExcluded(
1349+
oldExcludedTriggerTables, baseTableName);
1350+
boolean isExcluded = MTMVPartitionUtil.isTableExcluded(
1351+
newExcludedTriggerTables, baseTableName);
1352+
if (wasExcluded && !isExcluded) {
1353+
if (!isReplay) {
1354+
TableIf baseTable = MTMVUtil.getTable(baseTableInfo);
1355+
CreateMTMVCommand.createTableStream(ConnectContext.get(), db, mtmv, baseTable);
1356+
}
1357+
}
1358+
}
1359+
db.writeLockOrDdlException();
1360+
try {
1361+
for (BaseTableInfo baseTableInfo : baseTables) {
1362+
TableNameInfo baseTableName = new TableNameInfo(baseTableInfo.getCtlName(),
1363+
baseTableInfo.getDbName(), baseTableInfo.getTableName());
1364+
if (!MTMVPartitionUtil.isTableExcluded(newExcludedTriggerTables, baseTableName)) {
1365+
continue;
1366+
}
1367+
List<String> baseTableFullQualifiers = baseTableInfo.toList();
1368+
String streamName = IvmUtil.streamName(mtmv.getId(), baseTableFullQualifiers);
1369+
TableIf streamTable = db.getTableNullable(streamName);
1370+
if (streamTable == null) {
1371+
continue;
1372+
}
1373+
if (!(streamTable instanceof BaseTableStream)) {
1374+
LOG.warn("skip dropping IVM stream candidate {} because it is not a stream", streamName);
1375+
continue;
1376+
}
1377+
if (!IvmUtil.isStreamOwnedBy((BaseTableStream) streamTable, baseTableFullQualifiers)) {
1378+
LOG.warn("skip dropping IVM stream candidate {} because it belongs to another base table",
1379+
streamName);
1380+
continue;
1381+
}
1382+
Table table = (Table) streamTable;
1383+
table.writeLock();
1384+
try {
1385+
Env.getCurrentEnv().unprotectDropTable(db, table, true, isReplay, 0L);
1386+
} finally {
1387+
table.writeUnlock();
1388+
}
1389+
LOG.info("dropped IVM stream {} because its base table is excluded from MTMV",
1390+
streamName);
1391+
}
1392+
} finally {
1393+
db.writeUnlock();
1394+
}
1395+
}
13091396
}

‎fe/fe-core/src/main/java/org/apache/doris/catalog/CatalogRecycleBin.java‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@
2929
import org.apache.doris.common.util.DynamicPartitionUtil;
3030
import org.apache.doris.common.util.MasterDaemon;
3131
import org.apache.doris.common.util.TimeUtils;
32+
import org.apache.doris.mtmv.BaseTableInfo;
3233
import org.apache.doris.persist.RecoverInfo;
3334
import org.apache.doris.persist.gson.GsonUtils;
3435
import org.apache.doris.qe.GlobalVariable;
@@ -50,6 +51,7 @@
5051
import java.io.IOException;
5152
import java.util.ArrayList;
5253
import java.util.Collection;
54+
import java.util.Collections;
5355
import java.util.HashMap;
5456
import java.util.Iterator;
5557
import java.util.List;
@@ -1075,6 +1077,10 @@ public void recoverPartition(long dbId, OlapTable table, String partitionName,
10751077
recoverPartition.setName(newPartitionName);
10761078
}
10771079

1080+
Env.getCurrentEnv().getMtmvService().getRelationManager().markIvmBaselineRebuildForPartitionChange(
1081+
new BaseTableInfo(table, dbId),
1082+
Collections.singletonMap(partitionName, recoverPartition.getId()),
1083+
"Base table partition was recovered without row binlog");
10781084
// recover partition
10791085
table.addPartition(recoverPartition);
10801086

0 commit comments

Comments
 (0)