Skip to content

Commit eddf110

Browse files
seawindeyujun777
authored andcommitted
[fix](fe) Validate partitioned IVM MV keys and distribution (apache#19)
IVM materialized views are physically created as UNIQUE_KEYS merge-on-write tables and use the hidden row-id column as the stable dedup identity. The create path needed to align partition keys and distribution validation with that physical layout. Root cause: In `CreateMTMVInfo.analyze()` and `MTMVPlanUtil.analyzeIvmKeys()`, IVM CREATE analysis did not fully reuse ordinary MOW partition validation and user-visible distribution needed to be validated before rewriting the physical distribution to the hidden row-id column. | File | Change Description | |------|--------------------| | `MTMVPlanUtil.java` | Adds IVM partition-aware key analysis, keeps final key columns as the schema prefix, and appends row-id as the stable MOW key. | | `CreateMTMVInfo.java` | Validates user distribution before row-id rewrite, builds physical HASH(row-id) distribution, and reuses MOW partition key validation for IVM. | | `LogicalPlanBuilder.java` | Keeps omitted MTMV distribution as null and only records user-written distribution descriptors. | | `ShowCreateMTMVTest.java` / `CreateMTMVCommandTest.java` | Adds FE coverage for distribution validation, SHOW CREATE replay behavior, and MOW partition validation reuse. | | `regression-test/suites/mtmv_p0/ivm` | Adds and updates regression cases for partitioned IVM MVs and omitted distribution. | Design rationale: IVM must physically distribute by row-id because MOW dedup is tablet-local. User-written distribution is still analyzed before this rewrite so invalid columns and explicit RANDOM follow ordinary UNIQUE table rules. SHOW CREATE omits the hidden physical distribution so generated DDL remains re-executable. ### Release note Fixed IVM materialized view create validation for partition keys and distribution rewrite. ### Check List (For Author) - Test - [x] Regression test - [x] Unit Test - [ ] Manual test (add detailed scripts or steps below) - [ ] No need to test or manual test. Explain why: - [ ] This is a refactor/code format and no logic has been changed. - [ ] Previous test can cover this change. - [ ] No code files have been changed. - [ ] Other reason Unit tests: - `./run-fe-ut.sh --run org.apache.doris.nereids.trees.plans.CreateMTMVCommandTest` - `./run-fe-ut.sh --run org.apache.doris.catalog.ShowCreateMTMVTest` Regression test: - `./run-regression-test.sh --run -d mtmv_p0/ivm -s test_ivm_partition_unique_key`
1 parent d9b6bd9 commit eddf110

12 files changed

Lines changed: 1441 additions & 202 deletions

File tree

‎fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java‎

Lines changed: 32 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -3855,23 +3855,10 @@ public static String getMTMVDdl(MTMV mtmv) throws AnalysisException {
38553855
addColNameAndComment(mtmv, sb, isIvm);
38563856
sb.append("\n");
38573857
sb.append(mtmv.getRefreshInfo());
3858-
if (!isIvm) {
3859-
addMTMVKeyInfo(mtmv, sb);
3860-
}
3858+
addMTMVKeyInfo(mtmv, sb);
38613859
addTableComment(mtmv, sb);
38623860
addMTMVPartitionInfo(mtmv, sb);
3863-
DistributionInfo distributionInfo = mtmv.getDefaultDistributionInfo();
3864-
if (isIvm) {
3865-
// IVM internally rewrites distribution to HASH(__DORIS_IVM_ROW_ID_COL__),
3866-
// which is a hidden column invisible to users. Output DISTRIBUTED BY RANDOM
3867-
// with the same bucket count / auto-bucket setting, so the DDL is re-executable
3868-
// and preserves the bucket configuration. On re-creation, the IVM pipeline
3869-
// will rewrite RANDOM to HASH(row_id) again automatically.
3870-
sb.append("\n").append(new RandomDistributionInfo(
3871-
distributionInfo.getBucketNum(), distributionInfo.getAutoBucket()).toSql());
3872-
} else {
3873-
sb.append("\n").append(distributionInfo.toSql());
3874-
}
3861+
addMTMVDistributionInfo(mtmv, sb, isIvm);
38753862
// properties
38763863
sb.append("\nPROPERTIES (\n");
38773864
addOlapTablePropertyInfo(mtmv, sb, false, false, null);
@@ -3886,19 +3873,41 @@ public static String getMTMVDdl(MTMV mtmv) throws AnalysisException {
38863873
}
38873874

38883875
private static void addMTMVKeyInfo(MTMV mtmv, StringBuilder sb) {
3889-
if (!mtmv.isDuplicateWithoutKey()) {
3890-
String keySql = mtmv.getKeysType().toSql();
3891-
sb.append("\n").append(keySql).append("(");
3892-
List<String> keysColumnNames = Lists.newArrayList();
3893-
for (Column column : mtmv.getBaseSchema()) {
3894-
if (column.isKey()) {
3895-
keysColumnNames.add("`" + column.getName() + "`");
3896-
}
3876+
if (mtmv.isDuplicateWithoutKey()) {
3877+
return;
3878+
}
3879+
List<String> keysColumnNames = Lists.newArrayList();
3880+
for (Column column : mtmv.getBaseSchema(false)) {
3881+
if (column.isKey()) {
3882+
keysColumnNames.add("`" + column.getName() + "`");
38973883
}
3884+
}
3885+
if (!keysColumnNames.isEmpty()) {
3886+
String keySql = mtmv.isIvm() ? "KEY" : mtmv.getKeysType().toSql();
3887+
sb.append("\n").append(keySql).append("(");
38983888
sb.append(Joiner.on(", ").join(keysColumnNames)).append(")");
38993889
}
39003890
}
39013891

3892+
private static void addMTMVDistributionInfo(MTMV mtmv, StringBuilder sb, boolean isIvm) {
3893+
DistributionInfo distributionInfo = mtmv.getDefaultDistributionInfo();
3894+
if (isIvm && isIvmRowIdDistribution(distributionInfo)) {
3895+
sb.append("\n").append(new RandomDistributionInfo(
3896+
distributionInfo.getBucketNum(), distributionInfo.getAutoBucket()).toSql());
3897+
return;
3898+
}
3899+
sb.append("\n").append(distributionInfo.toSql());
3900+
}
3901+
3902+
private static boolean isIvmRowIdDistribution(DistributionInfo distributionInfo) {
3903+
if (!(distributionInfo instanceof HashDistributionInfo)) {
3904+
return false;
3905+
}
3906+
List<Column> distributionColumns = ((HashDistributionInfo) distributionInfo).getDistributionColumns();
3907+
return distributionColumns.size() == 1
3908+
&& Column.IVM_ROW_ID_COL.equalsIgnoreCase(distributionColumns.get(0).getName());
3909+
}
3910+
39023911
private static void addMTMVPartitionInfo(MTMV mtmv, StringBuilder sb) throws AnalysisException {
39033912
MTMVPartitionInfo mvPartitionInfo = mtmv.getMvPartitionInfo();
39043913
if (mvPartitionInfo.getPartitionType() == MTMVPartitionType.SELF_MANAGE) {

‎fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVAnalyzeQueryInfo.java‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -28,13 +28,15 @@ public class MTMVAnalyzeQueryInfo {
2828
private MTMVRelation relation;
2929
private MTMVPartitionInfo mvPartitionInfo;
3030
private List<ColumnDefinition> columnDefinitions;
31+
private List<String> keys;
3132
private Map<String, String> properties;
3233
// set when IVM normalization is enabled; carries normalizedPlan + aggMeta
3334
private IvmNormalizeResult ivmNormalizeResult;
3435

35-
public MTMVAnalyzeQueryInfo(List<ColumnDefinition> columnDefinitions, MTMVPartitionInfo mvPartitionInfo,
36-
MTMVRelation relation, Map<String, String> properties) {
36+
public MTMVAnalyzeQueryInfo(List<ColumnDefinition> columnDefinitions, List<String> keys,
37+
MTMVPartitionInfo mvPartitionInfo, MTMVRelation relation, Map<String, String> properties) {
3738
this.columnDefinitions = columnDefinitions;
39+
this.keys = keys;
3840
this.mvPartitionInfo = mvPartitionInfo;
3941
this.relation = relation;
4042
this.properties = properties;
@@ -44,6 +46,10 @@ public List<ColumnDefinition> getColumnDefinitions() {
4446
return columnDefinitions;
4547
}
4648

49+
public List<String> getKeys() {
50+
return keys;
51+
}
52+
4753
public MTMVPartitionInfo getMvPartitionInfo() {
4854
return mvPartitionInfo;
4955
}

0 commit comments

Comments
 (0)