Skip to content

Commit 71a3086

Browse files
seawindeyujun777
authored andcommitted
[improvement](fe) Add IVM create and alter validation for MTMV (#5)
Issue Number: close #xxx Problem Summary: 1. Remove redundant pre-analysis in IVM analyzeQuery - single analyzeQueryInternal call 2. Unify IVM base table validation error messages (AGG_KEYS vs UNIQUE without MOW) 3. Add excluded_trigger_tables support in IvmNormalizeMtmv (transient row-id for excluded tables) 4. Block ALTER MTMV refresh method to/from INCREMENTAL 5. Validate base table models when ALTER MTMV excluded_trigger_tables 6. Extract MTMVPropertyUtil.parseTableNameInfos utility 7. Add comprehensive unit tests for all validation paths
1 parent e63598e commit 71a3086

12 files changed

Lines changed: 627 additions & 59 deletions

File tree

‎fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,7 @@
5151
import org.apache.doris.info.TableNameInfo;
5252
import org.apache.doris.info.TableNameInfoUtils;
5353
import org.apache.doris.mtmv.BaseTableInfo;
54+
import org.apache.doris.mtmv.MTMVPlanUtil;
5455
import org.apache.doris.nereids.trees.plans.commands.AlterSystemCommand;
5556
import org.apache.doris.nereids.trees.plans.commands.AlterTableCommand;
5657
import org.apache.doris.nereids.trees.plans.commands.AlterViewCommand;
@@ -1250,6 +1251,14 @@ public void processAlterMTMV(AlterMTMV alterMTMV, boolean isReplay) {
12501251
mtmv.alterStatus(alterMTMV.getStatus());
12511252
break;
12521253
case ALTER_PROPERTY:
1254+
if (!isReplay && mtmv.isIvm()
1255+
&& alterMTMV.getMvProperties().containsKey(
1256+
PropertyAnalyzer.PROPERTIES_EXCLUDED_TRIGGER_TABLES)) {
1257+
Map<String, String> mergedMvProps = Maps.newHashMap(mtmv.getMvProperties());
1258+
mergedMvProps.putAll(alterMTMV.getMvProperties());
1259+
MTMVPlanUtil.validateIncrementalBaseTableModels(
1260+
mtmv.getRelation(), mergedMvProps);
1261+
}
12531262
mtmv.alterMvProperties(alterMTMV.getMvProperties());
12541263
break;
12551264
case ADD_TASK:

‎fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java‎

Lines changed: 5 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@
3535
import org.apache.doris.mtmv.MTMVPartitionInfo.MTMVPartitionType;
3636
import org.apache.doris.mtmv.MTMVPartitionUtil;
3737
import org.apache.doris.mtmv.MTMVPlanUtil;
38+
import org.apache.doris.mtmv.MTMVPropertyUtil;
3839
import org.apache.doris.mtmv.MTMVRefreshEnum.MTMVRefreshState;
3940
import org.apache.doris.mtmv.MTMVRefreshEnum.MTMVState;
4041
import org.apache.doris.mtmv.MTMVRefreshInfo;
@@ -51,7 +52,6 @@
5152

5253
import com.google.common.collect.Lists;
5354
import com.google.common.collect.Maps;
54-
import com.google.common.collect.Sets;
5555
import com.google.gson.annotations.SerializedName;
5656
import org.apache.commons.collections4.MapUtils;
5757
import org.apache.commons.lang3.StringUtils;
@@ -346,36 +346,20 @@ public int getRefreshPartitionNum() {
346346
}
347347

348348
public Set<TableNameInfo> getExcludedTriggerTables() {
349-
Set<TableNameInfo> res = Sets.newHashSet();
350349
readMvLock();
351350
try {
352-
if (StringUtils.isEmpty(mvProperties.get(PropertyAnalyzer.PROPERTIES_EXCLUDED_TRIGGER_TABLES))) {
353-
return res;
354-
}
355-
String[] split = mvProperties.get(PropertyAnalyzer.PROPERTIES_EXCLUDED_TRIGGER_TABLES).split(",");
356-
for (String alias : split) {
357-
res.add(new TableNameInfo(alias));
358-
}
359-
return res;
351+
return MTMVPropertyUtil.parseTableNameInfos(
352+
mvProperties.get(PropertyAnalyzer.PROPERTIES_EXCLUDED_TRIGGER_TABLES));
360353
} finally {
361354
readMvUnlock();
362355
}
363356
}
364357

365358
public Set<TableNameInfo> getQueryRewriteConsistencyRelaxedTables() {
366-
Set<TableNameInfo> res = Sets.newHashSet();
367359
readMvLock();
368360
try {
369-
String stillRewrittenTables
370-
= mvProperties.get(PropertyAnalyzer.ASYNC_MV_QUERY_REWRITE_CONSISTENCY_RELAXED_TABLES);
371-
if (StringUtils.isEmpty(stillRewrittenTables)) {
372-
return res;
373-
}
374-
String[] split = stillRewrittenTables.split(",");
375-
for (String alias : split) {
376-
res.add(new TableNameInfo(alias));
377-
}
378-
return res;
361+
return MTMVPropertyUtil.parseTableNameInfos(
362+
mvProperties.get(PropertyAnalyzer.ASYNC_MV_QUERY_REWRITE_CONSISTENCY_RELAXED_TABLES));
379363
} finally {
380364
readMvUnlock();
381365
}

‎fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPlanUtil.java‎

Lines changed: 76 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@
3030
import org.apache.doris.catalog.Env;
3131
import org.apache.doris.catalog.KeysType;
3232
import org.apache.doris.catalog.MTMV;
33+
import org.apache.doris.catalog.OlapTable;
3334
import org.apache.doris.catalog.ScalarType;
3435
import org.apache.doris.catalog.TableIf;
3536
import org.apache.doris.catalog.TableIf.TableType;
@@ -44,6 +45,7 @@
4445
import org.apache.doris.common.util.Util;
4546
import org.apache.doris.datasource.CatalogIf;
4647
import org.apache.doris.datasource.ExternalTable;
48+
import org.apache.doris.info.TableNameInfo;
4749
import org.apache.doris.job.exception.JobException;
4850
import org.apache.doris.job.task.AbstractTask;
4951
import org.apache.doris.mtmv.MTMVPartitionInfo.MTMVPartitionType;
@@ -496,7 +498,26 @@ public static MTMVAnalyzeQueryInfo analyzeQuery(ConnectContext ctx, Map<String,
496498
MTMVPartitionDefinition mvPartitionDefinition, DistributionDescriptor distribution,
497499
List<SimpleColumnDefinition> simpleColumnDefinitions, Map<String, String> properties, List<String> keys,
498500
LogicalPlan logicalQuery, boolean isIvm) throws UserException {
499-
try (StatementContext statementContext = ctx.getStatementContext()) {
501+
if (!isIvm) {
502+
return analyzeQueryInternal(ctx, mvProperties, mvPartitionDefinition, distribution,
503+
simpleColumnDefinitions, properties, keys, logicalQuery, false, Sets.newHashSet());
504+
}
505+
MTMVAnalyzeQueryInfo queryInfo = analyzeQueryInternal(ctx, mvProperties, mvPartitionDefinition, distribution,
506+
simpleColumnDefinitions, properties, keys, logicalQuery, true,
507+
getExcludedTriggerTables(mvProperties));
508+
validateIncrementalBaseTableModels(queryInfo.getRelation(), mvProperties);
509+
return queryInfo;
510+
}
511+
512+
private static MTMVAnalyzeQueryInfo analyzeQueryInternal(ConnectContext ctx, Map<String, String> mvProperties,
513+
MTMVPartitionDefinition mvPartitionDefinition, DistributionDescriptor distribution,
514+
List<SimpleColumnDefinition> simpleColumnDefinitions, Map<String, String> properties, List<String> keys,
515+
LogicalPlan logicalQuery, boolean isIvm, Set<TableNameInfo> excludedTriggerTables)
516+
throws UserException {
517+
StatementContext originalStatementContext = ctx.getStatementContext();
518+
try (StatementContext statementContext = new StatementContext(ctx, null)) {
519+
statementContext.setIvmExcludedTriggerTables(excludedTriggerTables);
520+
ctx.setStatementContext(statementContext);
500521
NereidsPlanner planner = new NereidsPlanner(statementContext);
501522
// this is for expression column name infer when not use alias
502523
LogicalSink<Plan> logicalSink = new UnboundResultSink<>(logicalQuery);
@@ -566,6 +587,60 @@ public static MTMVAnalyzeQueryInfo analyzeQuery(ConnectContext ctx, Map<String,
566587
queryInfo::setIvmNormalizeResult);
567588
}
568589
return queryInfo;
590+
} finally {
591+
ctx.setStatementContext(originalStatementContext);
592+
}
593+
}
594+
595+
private static Set<TableNameInfo> getExcludedTriggerTables(Map<String, String> mvProperties) {
596+
if (mvProperties == null) {
597+
return Sets.newHashSet();
598+
}
599+
return MTMVPropertyUtil.parseTableNameInfos(
600+
mvProperties.get(PropertyAnalyzer.PROPERTIES_EXCLUDED_TRIGGER_TABLES));
601+
}
602+
603+
public static void validateIncrementalBaseTableModels(MTMVRelation relation, Map<String, String> mvProperties) {
604+
Set<TableNameInfo> excludedTriggerTables = getExcludedTriggerTables(mvProperties);
605+
if (relation == null || relation.getBaseTablesOneLevelAndFromView() == null) {
606+
return;
607+
}
608+
for (BaseTableInfo baseTableInfo : relation.getBaseTablesOneLevelAndFromView()) {
609+
final TableIf table;
610+
try {
611+
table = MTMVUtil.getTable(baseTableInfo);
612+
} catch (org.apache.doris.common.AnalysisException e) {
613+
throw new AnalysisException(e.getMessage(), e);
614+
}
615+
TableNameInfo tableNameInfo = new TableNameInfo(baseTableInfo.getCtlName(),
616+
baseTableInfo.getDbName(), baseTableInfo.getTableName());
617+
if (MTMVPartitionUtil.isTableExcluded(excludedTriggerTables, tableNameInfo)) {
618+
continue;
619+
}
620+
if (!(table instanceof OlapTable)) {
621+
throw new AnalysisException(
622+
"INCREMENTAL materialized view only supports OlapTable base tables. "
623+
+ "Table '" + table.getName() + "' is " + table.getType() + ".");
624+
}
625+
OlapTable olapTable = (OlapTable) table;
626+
if (olapTable.getKeysType() != KeysType.UNIQUE_KEYS
627+
&& olapTable.getKeysType() != KeysType.DUP_KEYS) {
628+
throw new AnalysisException(
629+
"INCREMENTAL materialized view requires base tables to be "
630+
+ "UNIQUE_KEYS with Merge-On-Write or DUP_KEYS. Table '"
631+
+ olapTable.getName() + "' is " + olapTable.getKeysType()
632+
+ ". If this table does not participate in incremental refresh, "
633+
+ "add it to 'excluded_trigger_tables'.");
634+
}
635+
if (olapTable.getKeysType() == KeysType.UNIQUE_KEYS
636+
&& !olapTable.getEnableUniqueKeyMergeOnWrite()) {
637+
throw new AnalysisException(
638+
"INCREMENTAL materialized view requires UNIQUE_KEYS base tables "
639+
+ "to enable Merge-On-Write. Table '"
640+
+ olapTable.getName() + "' has MOW disabled."
641+
+ " If this table does not participate in incremental refresh, "
642+
+ "add it to 'excluded_trigger_tables'.");
643+
}
569644
}
570645
}
571646

‎fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPropertyUtil.java‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919

2020
import org.apache.doris.catalog.Env;
2121
import org.apache.doris.common.util.PropertyAnalyzer;
22+
import org.apache.doris.info.TableNameInfo;
2223
import org.apache.doris.mysql.privilege.PrivPredicate;
2324
import org.apache.doris.nereids.exceptions.AnalysisException;
2425
import org.apache.doris.qe.ConnectContext;
@@ -125,6 +126,20 @@ private static void analyzeExcludedTriggerTables(String value) {
125126
// do nothing
126127
}
127128

129+
public static Set<TableNameInfo> parseTableNameInfos(String value) {
130+
Set<TableNameInfo> tableNameInfos = Sets.newHashSet();
131+
if (StringUtils.isEmpty(value)) {
132+
return tableNameInfos;
133+
}
134+
for (String tableName : value.split(",")) {
135+
String trimmed = tableName.trim();
136+
if (!trimmed.isEmpty()) {
137+
tableNameInfos.add(new TableNameInfo(trimmed));
138+
}
139+
}
140+
return tableNameInfos;
141+
}
142+
128143
private static void analyzeDataChangeStillRewrittenTables(String value) {
129144
// do nothing
130145
}

‎fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@
3434
import org.apache.doris.datasource.mvcc.MvccTable;
3535
import org.apache.doris.datasource.mvcc.MvccTableInfo;
3636
import org.apache.doris.foundation.format.FormatOptions;
37+
import org.apache.doris.info.TableNameInfo;
3738
import org.apache.doris.mtmv.BaseTableInfo;
3839
import org.apache.doris.nereids.analyzer.UnboundRelation;
3940
import org.apache.doris.nereids.exceptions.AnalysisException;
@@ -320,6 +321,7 @@ public enum TableFrom {
320321

321322
private final Set<CTEId> mustInlineCTE = new HashSet<>();
322323
private final Set<String> usedAIResourceNames = new LinkedHashSet<>();
324+
private final Set<TableNameInfo> ivmExcludedTriggerTables = new HashSet<>();
323325

324326
private final Map<String, Integer> lowerCaseTableNamesCache = Maps.newHashMap();
325327
private final Map<String, Integer> lowerCaseDatabaseNamesCache = Maps.newHashMap();
@@ -415,6 +417,17 @@ public Map<List<String>, TableIf> getOneLevelTables() {
415417
return oneLevelTables;
416418
}
417419

420+
public Set<TableNameInfo> getIvmExcludedTriggerTables() {
421+
return ivmExcludedTriggerTables;
422+
}
423+
424+
public void setIvmExcludedTriggerTables(Set<TableNameInfo> excludedTriggerTables) {
425+
ivmExcludedTriggerTables.clear();
426+
if (excludedTriggerTables != null) {
427+
ivmExcludedTriggerTables.addAll(excludedTriggerTables);
428+
}
429+
}
430+
418431
public Set<MTMV> getCandidateMTMVs() {
419432
return candidateMTMVs;
420433
}

‎fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/IvmNormalizeMtmv.java‎

Lines changed: 39 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -21,12 +21,16 @@
2121
import org.apache.doris.catalog.KeysType;
2222
import org.apache.doris.catalog.OlapTable;
2323
import org.apache.doris.common.Pair;
24+
import org.apache.doris.info.TableNameInfo;
25+
import org.apache.doris.info.TableNameInfoUtils;
26+
import org.apache.doris.mtmv.MTMVPartitionUtil;
2427
import org.apache.doris.mtmv.ivm.IvmAggMeta;
2528
import org.apache.doris.mtmv.ivm.IvmAggMeta.AggTarget;
2629
import org.apache.doris.mtmv.ivm.IvmAggMeta.AggType;
2730
import org.apache.doris.mtmv.ivm.IvmAggMeta.StateKey;
2831
import org.apache.doris.mtmv.ivm.IvmNormalizeResult;
2932
import org.apache.doris.mtmv.ivm.IvmUtil;
33+
import org.apache.doris.nereids.StatementContext;
3034
import org.apache.doris.nereids.exceptions.AnalysisException;
3135
import org.apache.doris.nereids.jobs.JobContext;
3236
import org.apache.doris.nereids.trees.expressions.Alias;
@@ -84,6 +88,7 @@ public class IvmNormalizeMtmv extends DefaultPlanRewriter<Boolean> implements Cu
8488
ImmutableSet.of(Count.class, Sum.class, Avg.class, Min.class, Max.class);
8589

8690
private final IvmNormalizeResult normalizeResult = new IvmNormalizeResult();
91+
private StatementContext statementContext;
8792

8893
@Override
8994
public Plan rewriteRoot(Plan plan, JobContext jobContext) {
@@ -95,6 +100,7 @@ public Plan rewriteRoot(Plan plan, JobContext jobContext) {
95100
if (jobContext.getCascadesContext().getIvmNormalizeResult().isPresent()) {
96101
return plan;
97102
}
103+
statementContext = jobContext.getCascadesContext().getStatementContext();
98104
jobContext.getCascadesContext().setIvmNormalizeResult(normalizeResult);
99105
Plan result = plan.accept(this, true);
100106
normalizeResult.setNormalizedPlan(result);
@@ -112,7 +118,7 @@ public Plan visit(Plan plan, Boolean isFirstNonSink) {
112118
@Override
113119
public Plan visitLogicalOlapScan(LogicalOlapScan scan, Boolean isFirstNonSink) {
114120
OlapTable table = scan.getTable();
115-
Pair<Expression, Boolean> rowId = buildRowId(table, scan);
121+
Pair<Expression, Boolean> rowId = buildRowId(table, scan, isExcludedTriggerTable(table));
116122
Alias rowIdAlias = new Alias(rowId.first, Column.IVM_ROW_ID_COL);
117123
normalizeResult.addRowId(rowIdAlias.toSlot(), rowId.second);
118124
List<NamedExpression> outputs = ImmutableList.<NamedExpression>builder()
@@ -462,7 +468,13 @@ private NamedExpression rewriteIvmHiddenOutput(NamedExpression output, Map<Strin
462468
* - DUP_KEYS: (UuidNumeric(), false) — random per insert
463469
* - Other key types: throws AnalysisException
464470
*/
465-
private Pair<Expression, Boolean> buildRowId(OlapTable table, LogicalOlapScan scan) {
471+
private Pair<Expression, Boolean> buildRowId(OlapTable table, LogicalOlapScan scan,
472+
boolean isExcludedTriggerTable) {
473+
if (isExcludedTriggerTable) {
474+
// Excluded trigger tables never drive incremental maintenance. Use a transient row-id so CREATE / full
475+
// refresh can still build the internal UNIQUE_KEYS schema, and let runtime IVM precheck fall back.
476+
return Pair.of(new UuidNumeric(), false);
477+
}
466478
KeysType keysType = table.getKeysType();
467479
if (keysType == KeysType.UNIQUE_KEYS && table.getEnableUniqueKeyMergeOnWrite()) {
468480
List<String> keyColNames = table.getBaseSchemaKeyColumns().stream()
@@ -480,9 +492,31 @@ private Pair<Expression, Boolean> buildRowId(OlapTable table, LogicalOlapScan sc
480492
if (keysType == KeysType.DUP_KEYS) {
481493
return Pair.of(new UuidNumeric(), false);
482494
}
483-
throw new AnalysisException("IVM does not support table key type: " + keysType
484-
+ " for table: " + table.getName()
485-
+ ". Only MOW (UNIQUE_KEYS with merge-on-write) and DUP_KEYS are supported.");
495+
if (keysType == KeysType.UNIQUE_KEYS) {
496+
throw new AnalysisException(
497+
"INCREMENTAL materialized view requires UNIQUE_KEYS base tables "
498+
+ "to enable Merge-On-Write. Table '"
499+
+ table.getName() + "' has MOW disabled."
500+
+ " If this table does not participate in incremental refresh, "
501+
+ "add it to 'excluded_trigger_tables'.");
502+
}
503+
throw new AnalysisException(
504+
"INCREMENTAL materialized view requires base tables to be "
505+
+ "UNIQUE_KEYS with Merge-On-Write or DUP_KEYS. Table '"
506+
+ table.getName() + "' is " + keysType
507+
+ ". If this table does not participate in incremental refresh, "
508+
+ "add it to 'excluded_trigger_tables'.");
509+
}
510+
511+
private boolean isExcludedTriggerTable(OlapTable table) {
512+
if (statementContext == null || statementContext.getIvmExcludedTriggerTables().isEmpty()) {
513+
return false;
514+
}
515+
TableNameInfo tableNameInfo = TableNameInfoUtils.fromTableOrNull(table);
516+
if (tableNameInfo == null) {
517+
return false;
518+
}
519+
return MTMVPartitionUtil.isTableExcluded(statementContext.getIvmExcludedTriggerTables(), tableNameInfo);
486520
}
487521

488522
/**

‎fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/AlterMTMVRefreshInfo.java‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,8 +18,11 @@
1818
package org.apache.doris.nereids.trees.plans.commands.info;
1919

2020
import org.apache.doris.catalog.Env;
21+
import org.apache.doris.catalog.MTMV;
22+
import org.apache.doris.catalog.TableIf;
2123
import org.apache.doris.common.UserException;
2224
import org.apache.doris.info.TableNameInfo;
25+
import org.apache.doris.mtmv.MTMVRefreshEnum.RefreshMethod;
2326
import org.apache.doris.mtmv.MTMVRefreshInfo;
2427
import org.apache.doris.nereids.exceptions.AnalysisException;
2528
import org.apache.doris.qe.ConnectContext;
@@ -43,6 +46,35 @@ public AlterMTMVRefreshInfo(TableNameInfo mvName, MTMVRefreshInfo refreshInfo) {
4346
public void analyze(ConnectContext ctx) throws AnalysisException {
4447
super.analyze(ctx);
4548
refreshInfo.validate();
49+
validateRefreshMethodCompat();
50+
}
51+
52+
private void validateRefreshMethodCompat() {
53+
try {
54+
MTMV mtmv = (MTMV) Env.getCurrentInternalCatalog()
55+
.getDbOrDdlException(getMvName().getDb())
56+
.getTableOrMetaException(getMvName().getTbl(), TableIf.TableType.MATERIALIZED_VIEW);
57+
RefreshMethod oldMethod = mtmv.getRefreshInfo().getRefreshMethod();
58+
RefreshMethod newMethod = refreshInfo.getRefreshMethod();
59+
if (newMethod != null) {
60+
if (newMethod == RefreshMethod.INCREMENTAL
61+
&& oldMethod != RefreshMethod.INCREMENTAL) {
62+
throw new AnalysisException(
63+
"Cannot ALTER refresh method to INCREMENTAL. "
64+
+ "Please recreate the materialized view.");
65+
}
66+
if (newMethod != RefreshMethod.INCREMENTAL
67+
&& oldMethod == RefreshMethod.INCREMENTAL) {
68+
throw new AnalysisException(
69+
"Cannot ALTER refresh method from INCREMENTAL to "
70+
+ newMethod + ". Please recreate the materialized view.");
71+
}
72+
}
73+
} catch (AnalysisException e) {
74+
throw e;
75+
} catch (Exception e) {
76+
throw new AnalysisException(e.getMessage(), e);
77+
}
4678
}
4779

4880
@Override

0 commit comments

Comments
 (0)