Skip to content

Commit 15ca4a9

Browse files
seawindeyujun777
authored andcommitted
[fix](fe) Hide physical UNIQUE KEY and row-id columns in SHOW CREATE for IVM materialized views (apache#4)
### What problem does this PR solve? Issue Number: close #xxx Problem Summary: For INCREMENTAL (IVM) materialized views, `SHOW CREATE MATERIALIZED VIEW` exposes internal physical details that should be hidden: 1. `UNIQUE KEY(__DORIS_IVM_ROW_ID_COL__)` — the physical key clause 2. `"enable_unique_key_merge_on_write" = "true"` — the MOW property 3. Hidden `__DORIS_IVM_*` columns (defensive filtering) The generated DDL is not replayable — executing it would fail because users cannot create tables with these internal column names and properties. This fix modifies `Env.getMTMVDdl()` to: - Skip `addMTMVKeyInfo()` for IVM MVs (no UNIQUE KEY output) - Filter `__DORIS_IVM_*` columns in `addColNameAndComment()` - Skip MOW property output in `addOlapTablePropertyInfo()` for IVM MVs
1 parent fb15579 commit 15ca4a9

2 files changed

Lines changed: 209 additions & 6 deletions

File tree

‎fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java‎

Lines changed: 20 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -3806,12 +3806,15 @@ public static String getMTMVDdl(MTMV mtmv) throws AnalysisException {
38063806
"get table read lock timeout, database=" + mtmv.getDBName() + ",table=" + mtmv.getName());
38073807
}
38083808
try {
3809+
boolean isIvm = mtmv.isIvm();
38093810
StringBuilder sb = new StringBuilder("CREATE MATERIALIZED VIEW ");
38103811
sb.append(mtmv.getName());
3811-
addColNameAndComment(mtmv, sb);
3812+
addColNameAndComment(mtmv, sb, isIvm);
38123813
sb.append("\n");
38133814
sb.append(mtmv.getRefreshInfo());
3814-
addMTMVKeyInfo(mtmv, sb);
3815+
if (!isIvm) {
3816+
addMTMVKeyInfo(mtmv, sb);
3817+
}
38153818
addTableComment(mtmv, sb);
38163819
addMTMVPartitionInfo(mtmv, sb);
38173820
DistributionInfo distributionInfo = mtmv.getDefaultDistributionInfo();
@@ -3858,13 +3861,22 @@ private static void addMTMVPartitionInfo(MTMV mtmv, StringBuilder sb) throws Ana
38583861
}
38593862

38603863
private static void addColNameAndComment(TableIf tableIf, StringBuilder sb) {
3864+
addColNameAndComment(tableIf, sb, false);
3865+
}
3866+
3867+
private static void addColNameAndComment(TableIf tableIf, StringBuilder sb, boolean filterIvmHiddenCols) {
38613868
sb.append("\n(");
38623869
List<Column> columns = tableIf.getBaseSchema();
3870+
boolean first = true;
38633871
for (int i = 0; i < columns.size(); i++) {
3864-
if (i != 0) {
3872+
Column column = columns.get(i);
3873+
if (filterIvmHiddenCols && column.getName().startsWith("__DORIS_IVM_")) {
3874+
continue;
3875+
}
3876+
if (!first) {
38653877
sb.append(",");
38663878
}
3867-
Column column = columns.get(i);
3879+
first = false;
38683880
sb.append(column.getName());
38693881
if (!StringUtils.isEmpty(column.getComment())) {
38703882
sb.append(" comment '");
@@ -3985,14 +3997,16 @@ private static void addOlapTablePropertyInfo(OlapTable olapTable, StringBuilder
39853997
}
39863998

39873999
// unique key table with merge on write, always print this property for unique table
3988-
if (olapTable.getKeysType() == KeysType.UNIQUE_KEYS) {
4000+
// but hide it for IVM materialized views (internal physical detail)
4001+
boolean isIvmMtmv = olapTable instanceof MTMV && ((MTMV) olapTable).isIvm();
4002+
if (olapTable.getKeysType() == KeysType.UNIQUE_KEYS && !isIvmMtmv) {
39894003
sb.append(",\n\"").append(PropertyAnalyzer.ENABLE_UNIQUE_KEY_MERGE_ON_WRITE).append("\" = \"");
39904004
sb.append(olapTable.getEnableUniqueKeyMergeOnWrite()).append("\"");
39914005
}
39924006

39934007
// enable_unique_key_skip_bitmap, always print this property for merge-on-write unique table
39944008
if (olapTable.getKeysType() == KeysType.UNIQUE_KEYS && olapTable.getEnableUniqueKeyMergeOnWrite()
3995-
&& olapTable.getEnableUniqueKeySkipBitmap()) {
4009+
&& olapTable.getEnableUniqueKeySkipBitmap() && !isIvmMtmv) {
39964010
sb.append(",\n\"").append(PropertyAnalyzer.ENABLE_UNIQUE_KEY_SKIP_BITMAP_COLUMN).append("\" = \"");
39974011
sb.append(olapTable.getEnableUniqueKeySkipBitmap()).append("\"");
39984012
}
Lines changed: 189 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,189 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
package org.apache.doris.catalog;
19+
20+
import org.apache.doris.nereids.sqltest.SqlTestBase;
21+
22+
import org.junit.jupiter.api.Assertions;
23+
import org.junit.jupiter.api.Test;
24+
25+
/**
26+
* Tests for SHOW CREATE MATERIALIZED VIEW DDL output.
27+
*
28+
* PR-4: For IVM (INCREMENTAL) materialized views, the physical model uses
29+
* UNIQUE_KEYS + Merge-On-Write + hidden row-id columns internally. However,
30+
* SHOW CREATE should output a logical DDL that can be re-executed by users,
31+
* hiding all internal physical details:
32+
* - No UNIQUE KEY(...) clause
33+
* - No hidden IVM columns (e.g., __DORIS_IVM_ROW_ID_COL__)
34+
* - No enable_unique_key_merge_on_write property
35+
* - REFRESH INCREMENTAL must appear correctly
36+
*/
37+
public class ShowCreateMTMVTest extends SqlTestBase {
38+
39+
// TC-4-1: INCREMENTAL MV SHOW CREATE must not contain UNIQUE KEY(...)
40+
@Test
41+
public void testShowCreateIncrementalMVNoUniqueKey() throws Exception {
42+
createMvByNereids("create materialized view mv_show_ivm_no_uk "
43+
+ "BUILD DEFERRED REFRESH INCREMENTAL ON MANUAL\n"
44+
+ "DISTRIBUTED BY RANDOM BUCKETS 1\n"
45+
+ "PROPERTIES ('replication_num' = '1')\n"
46+
+ "as select * from test.T4;");
47+
48+
Database db = Env.getCurrentEnv().getInternalCatalog().getDbOrAnalysisException("test");
49+
MTMV mtmv = (MTMV) db.getTableOrAnalysisException("mv_show_ivm_no_uk");
50+
String ddl = Env.getMTMVDdl(mtmv);
51+
52+
Assertions.assertFalse(ddl.contains("UNIQUE KEY"),
53+
"IVM SHOW CREATE should not contain UNIQUE KEY, but got:\n" + ddl);
54+
}
55+
56+
// TC-4-2: INCREMENTAL MV SHOW CREATE must not expose hidden row-id column
57+
@Test
58+
public void testShowCreateIncrementalMVNoRowIdColumn() throws Exception {
59+
createMvByNereids("create materialized view mv_show_ivm_no_rowid "
60+
+ "BUILD DEFERRED REFRESH INCREMENTAL ON MANUAL\n"
61+
+ "DISTRIBUTED BY RANDOM BUCKETS 1\n"
62+
+ "PROPERTIES ('replication_num' = '1')\n"
63+
+ "as select * from test.T4;");
64+
65+
Database db = Env.getCurrentEnv().getInternalCatalog().getDbOrAnalysisException("test");
66+
MTMV mtmv = (MTMV) db.getTableOrAnalysisException("mv_show_ivm_no_rowid");
67+
String ddl = Env.getMTMVDdl(mtmv);
68+
69+
Assertions.assertFalse(ddl.contains("__DORIS_IVM_"),
70+
"IVM SHOW CREATE should not contain any __DORIS_IVM_ columns, but got:\n" + ddl);
71+
}
72+
73+
// TC-4-3: INCREMENTAL MV SHOW CREATE must contain REFRESH INCREMENTAL
74+
@Test
75+
public void testShowCreateIncrementalMVContainsRefreshIncremental() throws Exception {
76+
createMvByNereids("create materialized view mv_show_ivm_refresh "
77+
+ "BUILD DEFERRED REFRESH INCREMENTAL ON MANUAL\n"
78+
+ "DISTRIBUTED BY RANDOM BUCKETS 1\n"
79+
+ "PROPERTIES ('replication_num' = '1')\n"
80+
+ "as select * from test.T4;");
81+
82+
Database db = Env.getCurrentEnv().getInternalCatalog().getDbOrAnalysisException("test");
83+
MTMV mtmv = (MTMV) db.getTableOrAnalysisException("mv_show_ivm_refresh");
84+
String ddl = Env.getMTMVDdl(mtmv);
85+
86+
Assertions.assertTrue(ddl.contains("REFRESH INCREMENTAL"),
87+
"IVM SHOW CREATE should contain 'REFRESH INCREMENTAL', but got:\n" + ddl);
88+
}
89+
90+
// TC-4-4: Non-INCREMENTAL MV SHOW CREATE behavior must be unchanged
91+
@Test
92+
public void testShowCreateCompleteMVUnchanged() throws Exception {
93+
createMvByNereids("create materialized view mv_show_complete "
94+
+ "BUILD DEFERRED REFRESH COMPLETE ON MANUAL\n"
95+
+ "DISTRIBUTED BY RANDOM BUCKETS 1\n"
96+
+ "PROPERTIES ('replication_num' = '1')\n"
97+
+ "as select * from test.T4;");
98+
99+
Database db = Env.getCurrentEnv().getInternalCatalog().getDbOrAnalysisException("test");
100+
MTMV mtmv = (MTMV) db.getTableOrAnalysisException("mv_show_complete");
101+
String ddl = Env.getMTMVDdl(mtmv);
102+
103+
Assertions.assertTrue(ddl.contains("REFRESH COMPLETE"),
104+
"COMPLETE MV SHOW CREATE should contain 'REFRESH COMPLETE', but got:\n" + ddl);
105+
// Non-IVM MV uses DUP_KEYS without key columns, so no KEY clause output
106+
// (isDuplicateWithoutKey() returns true when keysNum == 0)
107+
Assertions.assertFalse(ddl.contains("__DORIS_IVM_"),
108+
"COMPLETE MV should not have any IVM columns:\n" + ddl);
109+
}
110+
111+
// TC-4-5: SHOW CREATE output of IVM MV must be replayable (can be re-executed)
112+
@Test
113+
public void testShowCreateIncrementalMVIsReplayable() throws Exception {
114+
createMvByNereids("create materialized view mv_show_ivm_replay "
115+
+ "BUILD DEFERRED REFRESH INCREMENTAL ON MANUAL\n"
116+
+ "DISTRIBUTED BY RANDOM BUCKETS 1\n"
117+
+ "PROPERTIES ('replication_num' = '1')\n"
118+
+ "as select * from test.T4;");
119+
120+
Database db = Env.getCurrentEnv().getInternalCatalog().getDbOrAnalysisException("test");
121+
MTMV mtmv = (MTMV) db.getTableOrAnalysisException("mv_show_ivm_replay");
122+
String ddl = Env.getMTMVDdl(mtmv);
123+
124+
// The DDL should not contain physical details that would cause re-execution to fail
125+
Assertions.assertFalse(ddl.contains("UNIQUE KEY"),
126+
"Replayable DDL must not contain UNIQUE KEY:\n" + ddl);
127+
Assertions.assertFalse(ddl.contains("enable_unique_key_merge_on_write"),
128+
"Replayable DDL must not contain MOW property:\n" + ddl);
129+
Assertions.assertFalse(ddl.contains("__DORIS_IVM_"),
130+
"Replayable DDL must not contain IVM hidden columns:\n" + ddl);
131+
Assertions.assertTrue(ddl.contains("REFRESH INCREMENTAL"),
132+
"Replayable DDL must contain REFRESH INCREMENTAL:\n" + ddl);
133+
}
134+
135+
// TC-4-6: Even with show_hidden_columns=true, IVM row-id should not be exposed in SHOW CREATE
136+
@Test
137+
public void testShowCreateIncrementalMVNoRowIdEvenWithShowHidden() throws Exception {
138+
createMvByNereids("create materialized view mv_show_ivm_hidden "
139+
+ "BUILD DEFERRED REFRESH INCREMENTAL ON MANUAL\n"
140+
+ "DISTRIBUTED BY RANDOM BUCKETS 1\n"
141+
+ "PROPERTIES ('replication_num' = '1')\n"
142+
+ "as select * from test.T4;");
143+
144+
Database db = Env.getCurrentEnv().getInternalCatalog().getDbOrAnalysisException("test");
145+
MTMV mtmv = (MTMV) db.getTableOrAnalysisException("mv_show_ivm_hidden");
146+
147+
// getMTMVDdl uses getBaseSchema() (visible only) and our filter,
148+
// so show_hidden_columns session variable should not affect the output
149+
boolean originalShowHidden = connectContext.getSessionVariable().showHiddenColumns();
150+
try {
151+
connectContext.getSessionVariable().setShowHiddenColumns(true);
152+
String ddl = Env.getMTMVDdl(mtmv);
153+
154+
Assertions.assertFalse(ddl.contains("__DORIS_IVM_"),
155+
"Even with show_hidden_columns=true, IVM columns should be hidden:\n" + ddl);
156+
Assertions.assertFalse(ddl.contains("UNIQUE KEY"),
157+
"Even with show_hidden_columns=true, UNIQUE KEY should be hidden:\n" + ddl);
158+
} finally {
159+
connectContext.getSessionVariable().setShowHiddenColumns(originalShowHidden);
160+
}
161+
}
162+
163+
// TC-4-7: DUP_KEYS MV (non-IVM) preserves its normal KEY clause in SHOW CREATE
164+
@Test
165+
public void testShowCreateDupKeysMVNoKeyOutput() throws Exception {
166+
createMvByNereids("create materialized view mv_show_dup_keys "
167+
+ "BUILD DEFERRED REFRESH COMPLETE ON MANUAL\n"
168+
+ "DISTRIBUTED BY RANDOM BUCKETS 1\n"
169+
+ "PROPERTIES ('replication_num' = '1')\n"
170+
+ "as select * from test.T4;");
171+
172+
Database db = Env.getCurrentEnv().getInternalCatalog().getDbOrAnalysisException("test");
173+
MTMV mtmv = (MTMV) db.getTableOrAnalysisException("mv_show_dup_keys");
174+
String ddl = Env.getMTMVDdl(mtmv);
175+
176+
// Non-IVM COMPLETE MV uses DUP_KEYS and may output DUPLICATE KEY(...)
177+
// The important thing is it should NOT have UNIQUE KEY or MOW properties
178+
Assertions.assertFalse(ddl.contains("UNIQUE KEY"),
179+
"DUP_KEYS MV should not contain UNIQUE KEY:\n" + ddl);
180+
Assertions.assertFalse(ddl.contains("enable_unique_key_merge_on_write"),
181+
"DUP_KEYS MV should not contain MOW property:\n" + ddl);
182+
// Should contain DUPLICATE KEY if it has key columns
183+
if (ddl.contains("DUPLICATE KEY")) {
184+
// Verify the key columns are from the query, not IVM internal columns
185+
Assertions.assertFalse(ddl.contains("__DORIS_IVM_"),
186+
"DUP_KEYS MV key clause should not reference IVM columns:\n" + ddl);
187+
}
188+
}
189+
}

0 commit comments

Comments
 (0)