Skip to content

Commit 60c0de1

Browse files
seawindeyujun777
authored andcommitted
[fix](ivm) Ivm validate columns fix when create incremental mv (#3)
* [fix](fe) Fix validateColumns to use correct KeysType for IVM materialized views
1 parent 5834fff commit 60c0de1

2 files changed

Lines changed: 102 additions & 1 deletion

File tree

‎fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPlanUtil.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -574,6 +574,7 @@ public static MTMVAnalyzeQueryInfo analyzeQuery(ConnectContext ctx, Map<String,
574574
*/
575575
private static void validateColumns(List<ColumnDefinition> columns, Set<String> keysSet,
576576
boolean finalEnableMergeOnWrite) throws UserException {
577+
KeysType keysType = finalEnableMergeOnWrite ? KeysType.UNIQUE_KEYS : KeysType.DUP_KEYS;
577578
Set<String> colSets = Sets.newTreeSet(String.CASE_INSENSITIVE_ORDER);
578579
for (ColumnDefinition col : columns) {
579580
if (!colSets.add(col.getName())) {
@@ -582,7 +583,7 @@ private static void validateColumns(List<ColumnDefinition> columns, Set<String>
582583
if (col.getType().isVarBinaryType()) {
583584
throw new AnalysisException("MTMV do not support varbinary type : " + col.getName());
584585
}
585-
col.validate(true, keysSet, Sets.newHashSet(), finalEnableMergeOnWrite, KeysType.DUP_KEYS);
586+
col.validate(true, keysSet, Sets.newHashSet(), finalEnableMergeOnWrite, keysType);
586587
}
587588
}
588589

‎fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVPlanUtilTest.java‎

Lines changed: 100 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -603,6 +603,32 @@ public void testEnsureMTMVQueryNotEqual() throws Exception {
603603
Assertions.assertTrue(exception.getMessage().contains("changed"));
604604
}
605605

606+
// Background:
607+
// IVM (INCREMENTAL) materialized views are internally stored as UNIQUE_KEYS + Merge-On-Write (MOW)
608+
// tables, while normal (COMPLETE) MVs use DUP_KEYS. The original validateColumns() hardcoded
609+
// KeysType.DUP_KEYS for all MVs, which caused ColumnDefinition.validate() to set incorrect
610+
// aggTypeImplicit values for IVM MV columns.
611+
//
612+
// What is aggTypeImplicit?
613+
// aggTypeImplicit is a boolean field on Column that indicates whether the column's aggregation
614+
// type (e.g. NONE, REPLACE) was implicitly assigned by the system rather than explicitly
615+
// specified by the user. When aggTypeImplicit=true, the aggregation type is considered an
616+
// internal implementation detail and is hidden from user-facing outputs.
617+
//
618+
// How aggTypeImplicit is set (in ColumnDefinition.validate()):
619+
// - DUP_KEYS value columns -> aggTypeImplicit = true (agg type is system-assigned)
620+
// - UNIQUE_KEYS + MOW value columns -> aggTypeImplicit = false (agg type is meaningful)
621+
// - UNIQUE_KEYS non-MOW value columns -> aggTypeImplicit = true
622+
//
623+
// Where aggTypeImplicit is used downstream:
624+
// - Column.toSql() (SHOW CREATE TABLE): when true, the aggregation type is hidden from DDL output
625+
// - Column.equals() / equalsForDistribution(): schema comparison includes this field, so a
626+
// mismatch causes schema inequality even if the actual aggregation type is the same
627+
//
628+
// The fix: derive KeysType from finalEnableMergeOnWrite (true -> UNIQUE_KEYS, false -> DUP_KEYS)
629+
// so validate() produces the correct aggTypeImplicit for each MV type.
630+
631+
// IVM MV must be created as UNIQUE_KEYS + MOW internally, regardless of user input
606632
@Test
607633
public void testIncrementalMvIsUniqueKeyWithMow() throws Exception {
608634
createMvByNereids("create materialized view mv_ivm_unique_key "
@@ -618,6 +644,7 @@ public void testIncrementalMvIsUniqueKeyWithMow() throws Exception {
618644
Assertions.assertTrue(mtmv.getEnableUniqueKeyMergeOnWrite());
619645
}
620646

647+
// Non-IVM (COMPLETE) MV must use DUP_KEYS — the original default behavior
621648
@Test
622649
public void testNonIncrementalMvIsDuplicateKey() throws Exception {
623650
createMvByNereids("create materialized view mv_non_ivm_dup_key "
@@ -632,6 +659,7 @@ public void testNonIncrementalMvIsDuplicateKey() throws Exception {
632659
Assertions.assertEquals(org.apache.doris.catalog.KeysType.DUP_KEYS, mtmv.getKeysType());
633660
}
634661

662+
// User explicitly specifying UNIQUE KEY on IVM MV must fail — the system auto-assigns UNIQUE_KEYS
635663
@Test
636664
public void testIncrementalMvWithUserSpecifiedUniqueKeyFails() {
637665
Assertions.assertThrows(Exception.class, () ->
@@ -643,6 +671,7 @@ public void testIncrementalMvWithUserSpecifiedUniqueKeyFails() {
643671
+ " as select * from test.T4;"));
644672
}
645673

674+
// User explicitly specifying DUPLICATE KEY on IVM MV must fail — same reason as above
646675
@Test
647676
public void testIncrementalMvWithUserSpecifiedDupKeyFails() {
648677
Assertions.assertThrows(Exception.class, () ->
@@ -654,6 +683,77 @@ public void testIncrementalMvWithUserSpecifiedDupKeyFails() {
654683
+ " as select * from test.T4;"));
655684
}
656685

686+
// IVM value columns must have aggTypeImplicit=false (UNIQUE_KEYS+MOW behavior)
687+
// Before the fix, validateColumns() used DUP_KEYS which incorrectly set aggTypeImplicit=true
688+
@Test
689+
public void testIncrementalMvColumnAggTypeNotImplicit() throws Exception {
690+
createMvByNereids("create materialized view mv_ivm_agg_type "
691+
+ "BUILD DEFERRED REFRESH INCREMENTAL ON MANUAL\n"
692+
+ " DISTRIBUTED BY RANDOM BUCKETS 1\n"
693+
+ " PROPERTIES ('replication_num' = '1') \n"
694+
+ " as select * from test.T4;");
695+
696+
Database db = Env.getCurrentEnv().getInternalCatalog().getDbOrAnalysisException("test");
697+
MTMV mtmv = (MTMV) db.getTableOrAnalysisException("mv_ivm_agg_type");
698+
699+
// UNIQUE_KEYS + MOW columns should have aggTypeImplicit = false
700+
for (Column col : mtmv.getBaseSchema()) {
701+
if (!col.isKey() && !col.getName().startsWith("__")) {
702+
Assertions.assertFalse(col.isAggregationTypeImplicit(),
703+
"IVM non-key column '" + col.getName() + "' should have aggTypeImplicit=false");
704+
}
705+
}
706+
}
707+
708+
// Counterpart: non-IVM MV value columns must have aggTypeImplicit=true (DUP_KEYS behavior)
709+
@Test
710+
public void testNonIncrementalMvColumnAggTypeImplicit() throws Exception {
711+
createMvByNereids("create materialized view mv_non_ivm_agg_type "
712+
+ "BUILD DEFERRED REFRESH COMPLETE ON MANUAL\n"
713+
+ " DISTRIBUTED BY RANDOM BUCKETS 1\n"
714+
+ " PROPERTIES ('replication_num' = '1') \n"
715+
+ " as select * from test.T4;");
716+
717+
Database db = Env.getCurrentEnv().getInternalCatalog().getDbOrAnalysisException("test");
718+
MTMV mtmv = (MTMV) db.getTableOrAnalysisException("mv_non_ivm_agg_type");
719+
720+
// DUP_KEYS columns should have aggTypeImplicit = true
721+
for (Column col : mtmv.getBaseSchema()) {
722+
if (!col.isKey()) {
723+
Assertions.assertTrue(col.isAggregationTypeImplicit(),
724+
"Non-IVM non-key column '" + col.getName() + "' should have aggTypeImplicit=true");
725+
}
726+
}
727+
}
728+
729+
// IVM adds a hidden __ivm_row_id column for row-level tracking. Verify it exists in full schema
730+
// but is invisible to users via getBaseSchema(false)
731+
@Test
732+
public void testIncrementalMvRowIdHiddenFromUserSchema() throws Exception {
733+
createMvByNereids("create materialized view mv_ivm_rowid_check "
734+
+ "BUILD DEFERRED REFRESH INCREMENTAL ON MANUAL\n"
735+
+ " DISTRIBUTED BY RANDOM BUCKETS 1\n"
736+
+ " PROPERTIES ('replication_num' = '1') \n"
737+
+ " as select * from test.T4;");
738+
739+
Database db = Env.getCurrentEnv().getInternalCatalog().getDbOrAnalysisException("test");
740+
MTMV mtmv = (MTMV) db.getTableOrAnalysisException("mv_ivm_rowid_check");
741+
742+
// Full schema (including hidden columns) should contain the IVM row-id column
743+
List<Column> fullSchema = mtmv.getBaseSchema(true);
744+
boolean hasRowId = fullSchema.stream()
745+
.anyMatch(c -> Column.IVM_ROW_ID_COL.equals(c.getName()));
746+
Assertions.assertTrue(hasRowId,
747+
"Full schema should contain hidden IVM row-id column");
748+
749+
// Visible (user) schema should NOT contain the IVM row-id column
750+
List<Column> visibleSchema = mtmv.getBaseSchema(false);
751+
boolean visibleHasRowId = visibleSchema.stream()
752+
.anyMatch(c -> Column.IVM_ROW_ID_COL.equals(c.getName()));
753+
Assertions.assertFalse(visibleHasRowId,
754+
"Visible schema should not expose IVM row-id column to user");
755+
}
756+
657757
private static class CountingSessionVariable extends SessionVariable {
658758
private int enableIvmRewriteSetCount;
659759

0 commit comments

Comments
 (0)