Skip to content

Commit 31a96e8

Browse files
seawindeyujun777
authored andcommitted
[fix](fe) Validate partitioned IVM MV keys and distribution (apache#19)
IVM materialized views are physically created as UNIQUE_KEYS merge-on-write tables and use the hidden row-id column as the stable dedup identity. The create path needed to align partition keys and distribution validation with that physical layout. Root cause: In `CreateMTMVInfo.analyze()` and `MTMVPlanUtil.analyzeIvmKeys()`, IVM CREATE analysis did not fully reuse ordinary MOW partition validation and user-visible distribution needed to be validated before rewriting the physical distribution to the hidden row-id column. | File | Change Description | |------|--------------------| | `MTMVPlanUtil.java` | Adds IVM partition-aware key analysis, keeps final key columns as the schema prefix, and appends row-id as the stable MOW key. | | `CreateMTMVInfo.java` | Validates user distribution before row-id rewrite, builds physical HASH(row-id) distribution, and reuses MOW partition key validation for IVM. | | `LogicalPlanBuilder.java` | Keeps omitted MTMV distribution as null and only records user-written distribution descriptors. | | `ShowCreateMTMVTest.java` / `CreateMTMVCommandTest.java` | Adds FE coverage for distribution validation, SHOW CREATE replay behavior, and MOW partition validation reuse. | | `regression-test/suites/mtmv_p0/ivm` | Adds and updates regression cases for partitioned IVM MVs and omitted distribution. | Design rationale: IVM must physically distribute by row-id because MOW dedup is tablet-local. User-written distribution is still analyzed before this rewrite so invalid columns and explicit RANDOM follow ordinary UNIQUE table rules. SHOW CREATE omits the hidden physical distribution so generated DDL remains re-executable. ### Release note Fixed IVM materialized view create validation for partition keys and distribution rewrite. ### Check List (For Author) - Test - [x] Regression test - [x] Unit Test - [ ] Manual test (add detailed scripts or steps below) - [ ] No need to test or manual test. Explain why: - [ ] This is a refactor/code format and no logic has been changed. - [ ] Previous test can cover this change. - [ ] No code files have been changed. - [ ] Other reason Unit tests: - `./run-fe-ut.sh --run org.apache.doris.nereids.trees.plans.CreateMTMVCommandTest` - `./run-fe-ut.sh --run org.apache.doris.catalog.ShowCreateMTMVTest` Regression test: - `./run-regression-test.sh --run -d mtmv_p0/ivm -s test_ivm_partition_unique_key`
1 parent 651b014 commit 31a96e8

12 files changed

Lines changed: 1441 additions & 202 deletions

File tree

‎fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java‎

Lines changed: 32 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -3864,23 +3864,10 @@ public static String getMTMVDdl(MTMV mtmv) throws AnalysisException {
38643864
addColNameAndComment(mtmv, sb, isIvm);
38653865
sb.append("\n");
38663866
sb.append(mtmv.getRefreshInfo());
3867-
if (!isIvm) {
3868-
addMTMVKeyInfo(mtmv, sb);
3869-
}
3867+
addMTMVKeyInfo(mtmv, sb);
38703868
addTableComment(mtmv, sb);
38713869
addMTMVPartitionInfo(mtmv, sb);
3872-
DistributionInfo distributionInfo = mtmv.getDefaultDistributionInfo();
3873-
if (isIvm) {
3874-
// IVM internally rewrites distribution to HASH(__DORIS_IVM_ROW_ID_COL__),
3875-
// which is a hidden column invisible to users. Output DISTRIBUTED BY RANDOM
3876-
// with the same bucket count / auto-bucket setting, so the DDL is re-executable
3877-
// and preserves the bucket configuration. On re-creation, the IVM pipeline
3878-
// will rewrite RANDOM to HASH(row_id) again automatically.
3879-
sb.append("\n").append(new RandomDistributionInfo(
3880-
distributionInfo.getBucketNum(), distributionInfo.getAutoBucket()).toSql());
3881-
} else {
3882-
sb.append("\n").append(distributionInfo.toSql());
3883-
}
3870+
addMTMVDistributionInfo(mtmv, sb, isIvm);
38843871
// properties
38853872
sb.append("\nPROPERTIES (\n");
38863873
addOlapTablePropertyInfo(mtmv, sb, false, false, null);
@@ -3895,19 +3882,41 @@ public static String getMTMVDdl(MTMV mtmv) throws AnalysisException {
38953882
}
38963883

38973884
private static void addMTMVKeyInfo(MTMV mtmv, StringBuilder sb) {
3898-
if (!mtmv.isDuplicateWithoutKey()) {
3899-
String keySql = mtmv.getKeysType().toSql();
3900-
sb.append("\n").append(keySql).append("(");
3901-
List<String> keysColumnNames = Lists.newArrayList();
3902-
for (Column column : mtmv.getBaseSchema()) {
3903-
if (column.isKey()) {
3904-
keysColumnNames.add("`" + column.getName() + "`");
3905-
}
3885+
if (mtmv.isDuplicateWithoutKey()) {
3886+
return;
3887+
}
3888+
List<String> keysColumnNames = Lists.newArrayList();
3889+
for (Column column : mtmv.getBaseSchema(false)) {
3890+
if (column.isKey()) {
3891+
keysColumnNames.add("`" + column.getName() + "`");
39063892
}
3893+
}
3894+
if (!keysColumnNames.isEmpty()) {
3895+
String keySql = mtmv.isIvm() ? "KEY" : mtmv.getKeysType().toSql();
3896+
sb.append("\n").append(keySql).append("(");
39073897
sb.append(Joiner.on(", ").join(keysColumnNames)).append(")");
39083898
}
39093899
}
39103900

3901+
private static void addMTMVDistributionInfo(MTMV mtmv, StringBuilder sb, boolean isIvm) {
3902+
DistributionInfo distributionInfo = mtmv.getDefaultDistributionInfo();
3903+
if (isIvm && isIvmRowIdDistribution(distributionInfo)) {
3904+
sb.append("\n").append(new RandomDistributionInfo(
3905+
distributionInfo.getBucketNum(), distributionInfo.getAutoBucket()).toSql());
3906+
return;
3907+
}
3908+
sb.append("\n").append(distributionInfo.toSql());
3909+
}
3910+
3911+
private static boolean isIvmRowIdDistribution(DistributionInfo distributionInfo) {
3912+
if (!(distributionInfo instanceof HashDistributionInfo)) {
3913+
return false;
3914+
}
3915+
List<Column> distributionColumns = ((HashDistributionInfo) distributionInfo).getDistributionColumns();
3916+
return distributionColumns.size() == 1
3917+
&& Column.IVM_ROW_ID_COL.equalsIgnoreCase(distributionColumns.get(0).getName());
3918+
}
3919+
39113920
private static void addMTMVPartitionInfo(MTMV mtmv, StringBuilder sb) throws AnalysisException {
39123921
MTMVPartitionInfo mvPartitionInfo = mtmv.getMvPartitionInfo();
39133922
if (mvPartitionInfo.getPartitionType() == MTMVPartitionType.SELF_MANAGE) {

‎fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVAnalyzeQueryInfo.java‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -28,13 +28,15 @@ public class MTMVAnalyzeQueryInfo {
2828
private MTMVRelation relation;
2929
private MTMVPartitionInfo mvPartitionInfo;
3030
private List<ColumnDefinition> columnDefinitions;
31+
private List<String> keys;
3132
private Map<String, String> properties;
3233
// set when IVM normalization is enabled; carries normalizedPlan + aggMeta
3334
private IvmNormalizeResult ivmNormalizeResult;
3435

35-
public MTMVAnalyzeQueryInfo(List<ColumnDefinition> columnDefinitions, MTMVPartitionInfo mvPartitionInfo,
36-
MTMVRelation relation, Map<String, String> properties) {
36+
public MTMVAnalyzeQueryInfo(List<ColumnDefinition> columnDefinitions, List<String> keys,
37+
MTMVPartitionInfo mvPartitionInfo, MTMVRelation relation, Map<String, String> properties) {
3738
this.columnDefinitions = columnDefinitions;
39+
this.keys = keys;
3840
this.mvPartitionInfo = mvPartitionInfo;
3941
this.relation = relation;
4042
this.properties = properties;
@@ -44,6 +46,10 @@ public List<ColumnDefinition> getColumnDefinitions() {
4446
return columnDefinitions;
4547
}
4648

49+
public List<String> getKeys() {
50+
return keys;
51+
}
52+
4753
public MTMVPartitionInfo getMvPartitionInfo() {
4854
return mvPartitionInfo;
4955
}

0 commit comments

Comments
 (0)