Skip to content

Commit 10e50d5

Browse files
seawindeyujun777
authored andcommitted
[fix](ivm) Ivm validate columns fix when create incremental mv (#3)
* [fix](fe) Fix validateColumns to use correct KeysType for IVM materialized views
1 parent d0162a4 commit 10e50d5

2 files changed

Lines changed: 102 additions & 1 deletion

File tree

‎fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPlanUtil.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -574,6 +574,7 @@ public static MTMVAnalyzeQueryInfo analyzeQuery(ConnectContext ctx, Map<String,
574574
*/
575575
private static void validateColumns(List<ColumnDefinition> columns, Set<String> keysSet,
576576
boolean finalEnableMergeOnWrite) throws UserException {
577+
KeysType keysType = finalEnableMergeOnWrite ? KeysType.UNIQUE_KEYS : KeysType.DUP_KEYS;
577578
Set<String> colSets = Sets.newTreeSet(String.CASE_INSENSITIVE_ORDER);
578579
for (ColumnDefinition col : columns) {
579580
if (!colSets.add(col.getName())) {
@@ -582,7 +583,7 @@ private static void validateColumns(List<ColumnDefinition> columns, Set<String>
582583
if (col.getType().isVarBinaryType()) {
583584
throw new AnalysisException("MTMV do not support varbinary type : " + col.getName());
584585
}
585-
col.validate(true, keysSet, Sets.newHashSet(), finalEnableMergeOnWrite, KeysType.DUP_KEYS);
586+
col.validate(true, keysSet, Sets.newHashSet(), finalEnableMergeOnWrite, keysType);
586587
}
587588
}
588589

‎fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVPlanUtilTest.java‎

Lines changed: 100 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -545,6 +545,32 @@ public void testEnsureMTMVQueryNotEqual() throws Exception {
545545
Assertions.assertTrue(exception.getMessage().contains("changed"));
546546
}
547547

548+
// Background:
549+
// IVM (INCREMENTAL) materialized views are internally stored as UNIQUE_KEYS + Merge-On-Write (MOW)
550+
// tables, while normal (COMPLETE) MVs use DUP_KEYS. The original validateColumns() hardcoded
551+
// KeysType.DUP_KEYS for all MVs, which caused ColumnDefinition.validate() to set incorrect
552+
// aggTypeImplicit values for IVM MV columns.
553+
//
554+
// What is aggTypeImplicit?
555+
// aggTypeImplicit is a boolean field on Column that indicates whether the column's aggregation
556+
// type (e.g. NONE, REPLACE) was implicitly assigned by the system rather than explicitly
557+
// specified by the user. When aggTypeImplicit=true, the aggregation type is considered an
558+
// internal implementation detail and is hidden from user-facing outputs.
559+
//
560+
// How aggTypeImplicit is set (in ColumnDefinition.validate()):
561+
// - DUP_KEYS value columns -> aggTypeImplicit = true (agg type is system-assigned)
562+
// - UNIQUE_KEYS + MOW value columns -> aggTypeImplicit = false (agg type is meaningful)
563+
// - UNIQUE_KEYS non-MOW value columns -> aggTypeImplicit = true
564+
//
565+
// Where aggTypeImplicit is used downstream:
566+
// - Column.toSql() (SHOW CREATE TABLE): when true, the aggregation type is hidden from DDL output
567+
// - Column.equals() / equalsForDistribution(): schema comparison includes this field, so a
568+
// mismatch causes schema inequality even if the actual aggregation type is the same
569+
//
570+
// The fix: derive KeysType from finalEnableMergeOnWrite (true -> UNIQUE_KEYS, false -> DUP_KEYS)
571+
// so validate() produces the correct aggTypeImplicit for each MV type.
572+
573+
// IVM MV must be created as UNIQUE_KEYS + MOW internally, regardless of user input
548574
@Test
549575
public void testIncrementalMvIsUniqueKeyWithMow() throws Exception {
550576
createMvByNereids("create materialized view mv_ivm_unique_key "
@@ -560,6 +586,7 @@ public void testIncrementalMvIsUniqueKeyWithMow() throws Exception {
560586
Assertions.assertTrue(mtmv.getEnableUniqueKeyMergeOnWrite());
561587
}
562588

589+
// Non-IVM (COMPLETE) MV must use DUP_KEYS — the original default behavior
563590
@Test
564591
public void testNonIncrementalMvIsDuplicateKey() throws Exception {
565592
createMvByNereids("create materialized view mv_non_ivm_dup_key "
@@ -574,6 +601,7 @@ public void testNonIncrementalMvIsDuplicateKey() throws Exception {
574601
Assertions.assertEquals(org.apache.doris.catalog.KeysType.DUP_KEYS, mtmv.getKeysType());
575602
}
576603

604+
// User explicitly specifying UNIQUE KEY on IVM MV must fail — the system auto-assigns UNIQUE_KEYS
577605
@Test
578606
public void testIncrementalMvWithUserSpecifiedUniqueKeyFails() {
579607
Assertions.assertThrows(Exception.class, () ->
@@ -585,6 +613,7 @@ public void testIncrementalMvWithUserSpecifiedUniqueKeyFails() {
585613
+ " as select * from test.T4;"));
586614
}
587615

616+
// User explicitly specifying DUPLICATE KEY on IVM MV must fail — same reason as above
588617
@Test
589618
public void testIncrementalMvWithUserSpecifiedDupKeyFails() {
590619
Assertions.assertThrows(Exception.class, () ->
@@ -596,6 +625,77 @@ public void testIncrementalMvWithUserSpecifiedDupKeyFails() {
596625
+ " as select * from test.T4;"));
597626
}
598627

628+
// IVM value columns must have aggTypeImplicit=false (UNIQUE_KEYS+MOW behavior)
629+
// Before the fix, validateColumns() used DUP_KEYS which incorrectly set aggTypeImplicit=true
630+
@Test
631+
public void testIncrementalMvColumnAggTypeNotImplicit() throws Exception {
632+
createMvByNereids("create materialized view mv_ivm_agg_type "
633+
+ "BUILD DEFERRED REFRESH INCREMENTAL ON MANUAL\n"
634+
+ " DISTRIBUTED BY RANDOM BUCKETS 1\n"
635+
+ " PROPERTIES ('replication_num' = '1') \n"
636+
+ " as select * from test.T4;");
637+
638+
Database db = Env.getCurrentEnv().getInternalCatalog().getDbOrAnalysisException("test");
639+
MTMV mtmv = (MTMV) db.getTableOrAnalysisException("mv_ivm_agg_type");
640+
641+
// UNIQUE_KEYS + MOW columns should have aggTypeImplicit = false
642+
for (Column col : mtmv.getBaseSchema()) {
643+
if (!col.isKey() && !col.getName().startsWith("__")) {
644+
Assertions.assertFalse(col.isAggregationTypeImplicit(),
645+
"IVM non-key column '" + col.getName() + "' should have aggTypeImplicit=false");
646+
}
647+
}
648+
}
649+
650+
// Counterpart: non-IVM MV value columns must have aggTypeImplicit=true (DUP_KEYS behavior)
651+
@Test
652+
public void testNonIncrementalMvColumnAggTypeImplicit() throws Exception {
653+
createMvByNereids("create materialized view mv_non_ivm_agg_type "
654+
+ "BUILD DEFERRED REFRESH COMPLETE ON MANUAL\n"
655+
+ " DISTRIBUTED BY RANDOM BUCKETS 1\n"
656+
+ " PROPERTIES ('replication_num' = '1') \n"
657+
+ " as select * from test.T4;");
658+
659+
Database db = Env.getCurrentEnv().getInternalCatalog().getDbOrAnalysisException("test");
660+
MTMV mtmv = (MTMV) db.getTableOrAnalysisException("mv_non_ivm_agg_type");
661+
662+
// DUP_KEYS columns should have aggTypeImplicit = true
663+
for (Column col : mtmv.getBaseSchema()) {
664+
if (!col.isKey()) {
665+
Assertions.assertTrue(col.isAggregationTypeImplicit(),
666+
"Non-IVM non-key column '" + col.getName() + "' should have aggTypeImplicit=true");
667+
}
668+
}
669+
}
670+
671+
// IVM adds a hidden __ivm_row_id column for row-level tracking. Verify it exists in full schema
672+
// but is invisible to users via getBaseSchema(false)
673+
@Test
674+
public void testIncrementalMvRowIdHiddenFromUserSchema() throws Exception {
675+
createMvByNereids("create materialized view mv_ivm_rowid_check "
676+
+ "BUILD DEFERRED REFRESH INCREMENTAL ON MANUAL\n"
677+
+ " DISTRIBUTED BY RANDOM BUCKETS 1\n"
678+
+ " PROPERTIES ('replication_num' = '1') \n"
679+
+ " as select * from test.T4;");
680+
681+
Database db = Env.getCurrentEnv().getInternalCatalog().getDbOrAnalysisException("test");
682+
MTMV mtmv = (MTMV) db.getTableOrAnalysisException("mv_ivm_rowid_check");
683+
684+
// Full schema (including hidden columns) should contain the IVM row-id column
685+
List<Column> fullSchema = mtmv.getBaseSchema(true);
686+
boolean hasRowId = fullSchema.stream()
687+
.anyMatch(c -> Column.IVM_ROW_ID_COL.equals(c.getName()));
688+
Assertions.assertTrue(hasRowId,
689+
"Full schema should contain hidden IVM row-id column");
690+
691+
// Visible (user) schema should NOT contain the IVM row-id column
692+
List<Column> visibleSchema = mtmv.getBaseSchema(false);
693+
boolean visibleHasRowId = visibleSchema.stream()
694+
.anyMatch(c -> Column.IVM_ROW_ID_COL.equals(c.getName()));
695+
Assertions.assertFalse(visibleHasRowId,
696+
"Visible schema should not expose IVM row-id column to user");
697+
}
698+
599699
private static class CountingSessionVariable extends SessionVariable {
600700
private int enableIvmRewriteSetCount;
601701

0 commit comments

Comments
 (0)