Skip to content

Commit 5e47ca8

Browse files
committed
[improvement](ddl) Preserve state types in async materialized views
### What problem does this PR solve? Issue Number: close apache#65865 Related PR: apache#66664 Problem Summary: Restricting HLL, QUANTILE_STATE, and AGG_STATE columns to Aggregate Key tables also rejected async materialized views whose analyzed query generates an internal Duplicate Key storage column with an aggregate state type. Treat schemas generated by the async materialized view planner as system-generated during column validation, while retaining the restriction for user-created non-Aggregate Key tables. Add coverage that creates an async materialized view with an AGG_STATE result column. ### Release note None ### Check List (For Author) - Test: Unit Test / Build - `./run-fe-ut.sh --run org.apache.doris.mtmv.MTMVPlanUtilTest` - `./run-fe-ut.sh --run org.apache.doris.nereids.trees.plans.commands.info.ColumnDefinitionTest,org.apache.doris.alter.InternalSchemaAlterTest` - `./build.sh -j 192` - Regression tests were not run because no Doris cluster was deployed in this worktree - Behavior changed: No. This preserves existing async materialized view behavior while keeping the user-facing restriction unchanged - Does this need documentation: No
1 parent cb68df6 commit 5e47ca8

5 files changed

Lines changed: 27 additions & 10 deletions

File tree

‎fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPlanUtil.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -491,7 +491,7 @@ private static void validateColumns(List<ColumnDefinition> columns, Set<String>
491491
if (col.getType().isVarBinaryType()) {
492492
throw new AnalysisException("MTMV do not support varbinary type : " + col.getName());
493493
}
494-
col.validate(true, keysSet, Sets.newHashSet(), finalEnableMergeOnWrite, KeysType.DUP_KEYS);
494+
col.validate(true, keysSet, Sets.newHashSet(), finalEnableMergeOnWrite, KeysType.DUP_KEYS, true);
495495
}
496496
}
497497

‎fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -374,8 +374,9 @@ public void validate(boolean isOlap, Set<String> keysSet, Set<String> clusterKey
374374
}
375375

376376
public void validate(boolean isOlap, Set<String> keysSet, Set<String> clusterKeySet, boolean isEnableMergeOnWrite,
377-
KeysType keysType, boolean isInternalSession) {
378-
validateInternal(isOlap, keysSet, clusterKeySet, isEnableMergeOnWrite, keysType, false, isInternalSession);
377+
KeysType keysType, boolean isSystemGeneratedTable) {
378+
validateInternal(isOlap, keysSet, clusterKeySet, isEnableMergeOnWrite, keysType, false,
379+
isSystemGeneratedTable);
379380
}
380381

381382
/**
@@ -387,7 +388,7 @@ public void validateNestedColumn(boolean isOlap, Set<String> keysSet, Set<String
387388
}
388389

389390
private void validateInternal(boolean isOlap, Set<String> keysSet, Set<String> clusterKeySet,
390-
boolean isEnableMergeOnWrite, KeysType keysType, boolean nestedColumn, boolean isInternalSession) {
391+
boolean isEnableMergeOnWrite, KeysType keysType, boolean nestedColumn, boolean isSystemGeneratedTable) {
391392
try {
392393
// if enableAddHiddenColumn is true, can add hidden column.
393394
// So does not check if the column name starts with __DORIS_
@@ -405,7 +406,7 @@ private void validateInternal(boolean isOlap, Set<String> keysSet, Set<String> c
405406
}
406407
type.validateDataType();
407408
type = updateCharacterTypeLength(type);
408-
if (!isInternalSession && isOlap && keysType != KeysType.AGG_KEYS && isAggregateTableOnlyType()
409+
if (!isSystemGeneratedTable && isOlap && keysType != KeysType.AGG_KEYS && isAggregateTableOnlyType()
409410
&& !Config.allow_non_aggregate_table_state_types) {
410411
throw new AnalysisException(String.format(
411412
"%s type is only supported in aggregate key tables, column: %s. "

‎fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -803,9 +803,9 @@ public void validate(ConnectContext ctx) {
803803
keysSet.addAll(keys);
804804
Set<String> orderKeySet = Sets.newTreeSet(String.CASE_INSENSITIVE_ORDER);
805805
orderKeySet.addAll(sortOrderFields.stream().map(SortFieldInfo::getColumnName).collect(Collectors.toSet()));
806-
boolean isInternalSession = ctx.getState().isInternal();
806+
boolean isSystemGeneratedTable = ctx.getState().isInternal();
807807
columns.forEach(c -> c.validate(targetIsInternalCatalog, keysSet, orderKeySet, finalEnableMergeOnWrite,
808-
keysType, isInternalSession));
808+
keysType, isSystemGeneratedTable));
809809

810810
// validate index
811811
if (!indexes.isEmpty()) {

‎fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVPlanUtilTest.java‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -340,6 +340,22 @@ public void testAnalyzeQueryNormal() throws Exception {
340340
Assertions.assertTrue(mtmvAnalyzeQueryInfo.getColumnDefinitions().size() == 2);
341341
}
342342

343+
@Test
344+
public void testCreateMTMVWithAggStateColumn() throws Exception {
345+
boolean originalEnableAggState = connectContext.getSessionVariable().enableAggState;
346+
connectContext.getSessionVariable().enableAggState = true;
347+
connectContext.setThreadLocalInfo();
348+
try {
349+
Assertions.assertDoesNotThrow(() -> createMvByNereids(
350+
"create materialized view mv_with_agg_state BUILD DEFERRED REFRESH COMPLETE ON MANUAL\n"
351+
+ "DISTRIBUTED BY RANDOM BUCKETS 1\n"
352+
+ "PROPERTIES ('replication_num' = '1')\n"
353+
+ "as select id, sum_union(sum_state(score)) from test.T1 group by id"));
354+
} finally {
355+
connectContext.getSessionVariable().enableAggState = originalEnableAggState;
356+
}
357+
}
358+
343359
@Test
344360
public void testEnsureMTMVQueryUsable() throws Exception {
345361
createMvByNereids("create materialized view mv1 BUILD DEFERRED REFRESH COMPLETE ON MANUAL\n"

‎fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinitionTest.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -111,12 +111,12 @@ public void testStateTypesRemainSupportedInAggregateKeyTable() {
111111
}
112112

113113
@Test
114-
public void testInternalSessionAllowsStateTypesInNonAggregateTable() {
114+
public void testSystemGeneratedTableAllowsStateTypesInNonAggregateTable() {
115115
for (KeysType keysType : ImmutableList.of(KeysType.DUP_KEYS, KeysType.UNIQUE_KEYS)) {
116116
for (DataType type : aggregateTableOnlyTypes()) {
117117
ColumnDefinition column = new ColumnDefinition(
118118
"v", type, false, null, false, Optional.empty(), "");
119-
Assertions.assertDoesNotThrow(() -> validateInternalColumn(column, keysType));
119+
Assertions.assertDoesNotThrow(() -> validateSystemGeneratedColumn(column, keysType));
120120
}
121121
}
122122
}
@@ -133,7 +133,7 @@ private static void validateColumn(ColumnDefinition column, KeysType keysType) {
133133
column.validate(true, ImmutableSet.of("k"), ImmutableSet.of(), true, keysType);
134134
}
135135

136-
private static void validateInternalColumn(ColumnDefinition column, KeysType keysType) {
136+
private static void validateSystemGeneratedColumn(ColumnDefinition column, KeysType keysType) {
137137
column.validate(true, ImmutableSet.of("k"), ImmutableSet.of(), true, keysType, true);
138138
}
139139
}

0 commit comments

Comments
 (0)