Skip to content

Commit 22c95eb

Browse files
authored
[fix](ivm) Carry the row-binlog hidden columns in the analyzed MTMV schema (#67853)
An MTMV re-validates its schema against a fresh analysis of its own query whenever a base table changes (`MTMVPlanUtil.ensureMTMVQueryUsable` -> `checkColumnIfChange`). A table created with row binlog carries hidden columns that the query itself never produces (`__DORIS_COMMIT_TSO_COL__`, plus `__DORIS_ROW_LSN_COL__` for duplicate keys), and an MV ends up with them in its physical schema when its own properties enable row binlog. The analyzed column list did not contain them, so for those MVs the comparison was permanently short by a column and every refresh failed with `column length not equals, please check whether columns of base table have changed` -- including `COMPLETE`, which is the only way out of a stale baseline. The MV then became unrecoverable without dropping and recreating it. Cascade IVM always hits this, because a cascade source has to keep row binlog for its downstream level. Reported shape and trace issue: #65418 The root cause is that the hidden-column rules were split across two classes: `CreateTableInfo` owns the OLAP ones (delete sign / row store / version / skip bitmap), while the row-binlog ones are added by `InternalCatalog#createOlapTable` right before the table is built. The analyzed schema path only mirrored the first half. ### What changed - Extracted the row-binlog hidden-column rule into `CreateTableInfo.addRowBinlogHiddenColumns`, and let `createRowBinlogHiddenColumnsIfNecessary` delegate to it. The helper is idempotent. - Applied the same rule when the analyzed MTMV column list is built, so that list matches the physical schema of the table it produces. The binlog config is derived from the properties passed in: for a refresh those are the MV table's own properties, where `InternalCatalog` already stores the resolved (database-merged) binlog config. - The physical schema is unchanged: both paths append the same columns at the same position, so a table built from the analyzed list still ends up with exactly one of each. `checkColumnIfChange` compares types index by index, so the fix also pins the column order, not just the count.
1 parent 859a19c commit 22c95eb

5 files changed

Lines changed: 318 additions & 4 deletions

File tree

‎fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPlanUtil.java‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
import org.apache.doris.analysis.ToSqlParams;
2424
import org.apache.doris.analysis.UserIdentity;
2525
import org.apache.doris.catalog.AggregateType;
26+
import org.apache.doris.catalog.BinlogConfig;
2627
import org.apache.doris.catalog.Column;
2728
import org.apache.doris.catalog.DatabaseIf;
2829
import org.apache.doris.catalog.DistributionInfo;
@@ -625,6 +626,14 @@ private static MTMVAnalyzeQueryInfo analyzeQueryInternal(ConnectContext ctx, Map
625626
properties = CreateTableInfo.addOlapHiddenColumns(
626627
columns, isIvm ? KeysType.UNIQUE_KEYS : KeysType.DUP_KEYS,
627628
isIvm, properties, false);
629+
// A row-binlog table carries hidden columns on top of the OLAP ones above, added by
630+
// InternalCatalog#createOlapTable just before the table is built. The analyzed list has
631+
// to carry them too: an MTMV re-validates its schema against it whenever a base table
632+
// changes (MTMVPlanUtil#checkColumnIfChange), and a column missing here is
633+
// indistinguishable from a real schema change. Idempotent, so the table still gets one.
634+
CreateTableInfo.addRowBinlogHiddenColumns(columns,
635+
isIvm ? KeysType.UNIQUE_KEYS : KeysType.DUP_KEYS, isIvm,
636+
BinlogConfig.fromProperties(properties));
628637
// analyze column
629638
final boolean finalEnableMergeOnWrite = isIvm;
630639
Set<String> keysSet = Sets.newTreeSet(String.CASE_INSENSITIVE_ORDER);

‎fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java‎

Lines changed: 24 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1646,14 +1646,34 @@ public void checkLegalityOfPartitionExprs(PartitionTableInfo partitionTableInfo)
16461646
* Add hidden columns required by row binlog.
16471647
*/
16481648
public void createRowBinlogHiddenColumnsIfNecessary(BinlogConfig binlogConfig) {
1649-
if (!binlogConfig.isRowFormat()) {
1649+
addRowBinlogHiddenColumns(columns, keysType, isEnableMergeOnWrite, binlogConfig);
1650+
}
1651+
1652+
/**
1653+
* Append the hidden columns a row-binlog table carries. Callers that build the column list
1654+
* outside the create-table flow (an analyzed MTMV schema) go through here as well, so both
1655+
* sides of {@code MTMVPlanUtil#checkColumnIfChange} agree on the physical layout.
1656+
*
1657+
* <p>Idempotent: a column that is already present is kept once.
1658+
*/
1659+
public static void addRowBinlogHiddenColumns(List<ColumnDefinition> columns, KeysType keysType,
1660+
boolean isEnableMergeOnWrite, BinlogConfig binlogConfig) {
1661+
if (binlogConfig == null || !binlogConfig.isRowFormat()) {
16501662
return;
16511663
}
16521664
if (keysType.equals(KeysType.DUP_KEYS)) {
1653-
columns.add(ColumnDefinition.newCommitTsoColumnDefinition(AggregateType.NONE));
1654-
columns.add(ColumnDefinition.newRowLsnColumnDefinition(AggregateType.NONE));
1665+
addIfAbsent(columns, ColumnDefinition.newCommitTsoColumnDefinition(AggregateType.NONE));
1666+
addIfAbsent(columns, ColumnDefinition.newRowLsnColumnDefinition(AggregateType.NONE));
16551667
} else if (keysType.equals(KeysType.UNIQUE_KEYS) && isEnableMergeOnWrite) {
1656-
columns.add(ColumnDefinition.newCommitTsoColumnDefinition(AggregateType.NONE));
1668+
addIfAbsent(columns, ColumnDefinition.newCommitTsoColumnDefinition(AggregateType.NONE));
1669+
}
1670+
}
1671+
1672+
private static void addIfAbsent(List<ColumnDefinition> columns, ColumnDefinition columnDefinition) {
1673+
boolean present = columns.stream()
1674+
.anyMatch(column -> column.getName().equalsIgnoreCase(columnDefinition.getName()));
1675+
if (!present) {
1676+
columns.add(columnDefinition);
16571677
}
16581678
}
16591679
}

‎fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVPlanUtilTest.java‎

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -490,6 +490,70 @@ public void testEnsureMTMVQueryUsableEnableIvmRewriteByRefreshMethod() throws Ex
490490
incrementalCtx.getStatementContext().getIvmRewriteContext().orElseThrow().getMode());
491491
}
492492

493+
@Test
494+
public void testEnsureMTMVQueryUsableWithRowBinlogHiddenColumns() throws Exception {
495+
// A table created with row binlog carries hidden columns its query never produces:
496+
// __DORIS_COMMIT_TSO_COL__ for merge-on-write unique keys, plus __DORIS_ROW_LSN_COL__ for
497+
// duplicate keys. InternalCatalog#createOlapTable adds them while the table is built, so an
498+
// MV that enables row binlog itself ends up with them in its physical schema.
499+
// ensureMTMVQueryUsable re-derives the schema from the query and compares the two
500+
// (checkColumnIfChange), so the analyzed column list has to carry them too -- otherwise
501+
// every refresh of such an MV fails with a spurious "column length not equals".
502+
createTable("CREATE TABLE IF NOT EXISTS row_binlog_schema_base (\n"
503+
+ " k1 int,\n"
504+
+ " v1 int\n"
505+
+ ")\n"
506+
+ "DUPLICATE KEY(k1)\n"
507+
+ "DISTRIBUTED BY HASH(k1) BUCKETS 1\n"
508+
+ "PROPERTIES ('replication_num' = '1', 'binlog.enable' = 'true', 'binlog.format' = 'ROW')\n");
509+
510+
createMvByNereids("create materialized view row_binlog_schema_ivm "
511+
+ "BUILD DEFERRED REFRESH INCREMENTAL ON MANUAL\n"
512+
+ " DISTRIBUTED BY RANDOM BUCKETS 1\n"
513+
+ " PROPERTIES ('replication_num' = '1', 'binlog.enable' = 'true', "
514+
+ "'binlog.format' = 'ROW') \n"
515+
+ " as select k1, v1 from test.row_binlog_schema_base;");
516+
createMvByNereids("create materialized view row_binlog_schema_dup "
517+
+ "BUILD DEFERRED REFRESH COMPLETE ON MANUAL\n"
518+
+ " DISTRIBUTED BY RANDOM BUCKETS 1\n"
519+
+ " PROPERTIES ('replication_num' = '1', 'binlog.enable' = 'true', "
520+
+ "'binlog.format' = 'ROW') \n"
521+
+ " as select k1, v1 from test.row_binlog_schema_base;");
522+
createMvByNereids("create materialized view row_binlog_schema_plain "
523+
+ "BUILD DEFERRED REFRESH COMPLETE ON MANUAL\n"
524+
+ " DISTRIBUTED BY RANDOM BUCKETS 1\n"
525+
+ " PROPERTIES ('replication_num' = '1') \n"
526+
+ " as select k1, v1 from test.row_binlog_schema_base;");
527+
528+
Database db = Env.getCurrentEnv().getInternalCatalog().getDbOrAnalysisException("test");
529+
MTMV ivmMv = (MTMV) db.getTableOrAnalysisException("row_binlog_schema_ivm");
530+
MTMV dupMv = (MTMV) db.getTableOrAnalysisException("row_binlog_schema_dup");
531+
MTMV plainMv = (MTMV) db.getTableOrAnalysisException("row_binlog_schema_plain");
532+
533+
Assertions.assertEquals(Lists.newArrayList(Column.COMMIT_TSO_COL), rowBinlogHiddenColumns(ivmMv));
534+
Assertions.assertEquals(Lists.newArrayList(Column.COMMIT_TSO_COL, Column.ROW_LSN_COL),
535+
rowBinlogHiddenColumns(dupMv));
536+
Assertions.assertTrue(rowBinlogHiddenColumns(plainMv).isEmpty());
537+
538+
for (MTMV mtmv : Lists.newArrayList(ivmMv, dupMv, plainMv)) {
539+
ConnectContext ctx = MTMVPlanUtil.createMTMVContext(mtmv,
540+
MTMVPlanUtil.DISABLE_RULES_WHEN_GENERATE_MTMV_CACHE);
541+
Assertions.assertDoesNotThrow(() -> MTMVPlanUtil.ensureMTMVQueryUsable(mtmv, ctx),
542+
"analyzed schema must match the physical schema of " + mtmv.getName());
543+
}
544+
}
545+
546+
private static List<String> rowBinlogHiddenColumns(MTMV mtmv) {
547+
List<String> hidden = Lists.newArrayList();
548+
for (Column column : mtmv.getBaseSchema(true)) {
549+
if (column.getName().equalsIgnoreCase(Column.COMMIT_TSO_COL)
550+
|| column.getName().equalsIgnoreCase(Column.ROW_LSN_COL)) {
551+
hidden.add(column.getName().toUpperCase());
552+
}
553+
}
554+
return hidden;
555+
}
556+
493557
@Test
494558
public void testEnsureMTMVQueryAnalyzeFailed() throws Exception {
495559
createTable("CREATE TABLE IF NOT EXISTS analyze_faild_t_partition (\n"
Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
-- This file is automatically generated. You should know what you did if you want to edit this
2+
-- !rb_ivm_rows --
3+
1 10
4+
2 20
5+
6+
-- !rb_ivm_no_binlog_rows --
7+
1 10
8+
2 20
9+
10+
-- !rb_dup_rows --
11+
1 10
12+
2 20
13+
14+
-- !rb_dup_no_binlog_rows --
15+
1 10
16+
2 20
17+
18+
-- !cascade_l2_baseline --
19+
10 2 300
20+
20 1 300
21+
22+
-- !cascade_l1_after_aba --
23+
0 3 600
24+
25+
-- !cascade_l2_after_aba --
26+
0 3 600
27+
Lines changed: 194 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,194 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
import org.awaitility.Awaitility
19+
import static java.util.concurrent.TimeUnit.SECONDS
20+
21+
// A materialized view re-validates its schema against a fresh analysis of its own query whenever a
22+
// base table changes (MTMVPlanUtil.ensureMTMVQueryUsable -> checkColumnIfChange). A table created
23+
// with row binlog carries hidden columns that the query never produces, and an MV carries them too
24+
// when its own properties enable row binlog (which cascade IVM requires). The analyzed schema has to
25+
// contain them as well, otherwise every refresh of such an MV fails with
26+
// "column length not equals, please check whether columns of base table have changed" -- including a
27+
// COMPLETE refresh, which is the only way out of a stale baseline.
28+
//
29+
// Covered here: the four MV shapes, and the DORIS-28306 shape where a cascade L1 could not recover
30+
// through COMPLETE after a schema ABA on a referenced column.
31+
suite("test_ivm_row_binlog_schema_validation") {
32+
def rowBinlogProps = "'replication_num' = '1', 'binlog.enable' = 'true', 'binlog.format' = 'ROW'"
33+
def plainProps = "'replication_num' = '1'"
34+
// A cascade source keeps historical values so its own downstream MV can read the binlog.
35+
def cascadeProps = rowBinlogProps + ", 'binlog.need_historical_value' = 'true'"
36+
37+
def ddlJobCount = { String table ->
38+
return sql("""SHOW ALTER TABLE COLUMN WHERE TableName = '${table}'""").size()
39+
}
40+
41+
// `SHOW ALTER TABLE COLUMN` keeps finished jobs, so wait for a *new* job that is FINISHED.
42+
def waitDdlFinished = { String table, int previousJobCount ->
43+
Awaitility.await().atMost(300, SECONDS).pollInterval(2, SECONDS).until({
44+
def jobs = sql """SHOW ALTER TABLE COLUMN WHERE TableName = '${table}'"""
45+
return jobs.size() > previousJobCount
46+
&& jobs.every({ row -> row[9].toString() == 'FINISHED' })
47+
})
48+
}
49+
50+
def lastTaskId = null
51+
// tasks('type'='mv') can briefly miss the just-finished task, so require a *new* TaskId.
52+
def waitTerminalTask = { String mv ->
53+
def taskResult
54+
Awaitility.await().atMost(300, SECONDS).pollInterval(2, SECONDS).until({
55+
taskResult = sql_return_maparray("""
56+
SELECT TaskId, Status, RefreshMode, ErrorMsg
57+
FROM tasks('type'='mv')
58+
WHERE MvDatabaseName = '${context.dbName}' AND MvName = '${mv}'
59+
ORDER BY CreateTime DESC, TaskId DESC LIMIT 1
60+
""")
61+
return !taskResult.isEmpty()
62+
&& taskResult[0].TaskId.toString() != lastTaskId
63+
&& taskResult[0].Status.toString() != 'PENDING'
64+
&& taskResult[0].Status.toString() != 'RUNNING'
65+
})
66+
lastTaskId = taskResult[0].TaskId.toString()
67+
return taskResult[0]
68+
}
69+
70+
def refresh = { String mv, String mode ->
71+
sql """REFRESH MATERIALIZED VIEW ${mv} ${mode}"""
72+
return waitTerminalTask(mv)
73+
}
74+
75+
// ---------------------------------------------------------------- 1. four MV shapes
76+
// Same query and same base table for all four; what differs is whether the MV itself enables
77+
// row binlog, and (for the IVM ones) whether the MV is a MOW unique table.
78+
def cases = [
79+
[name: "rb_ivm", mvProps: rowBinlogProps, ivm: true],
80+
[name: "rb_ivm_no_binlog", mvProps: plainProps, ivm: true],
81+
[name: "rb_dup", mvProps: rowBinlogProps, ivm: false],
82+
[name: "rb_dup_no_binlog", mvProps: plainProps, ivm: false],
83+
]
84+
85+
for (def c : cases) {
86+
def table = c.name + "_base"
87+
def mv = c.name + "_mv"
88+
89+
sql """DROP MATERIALIZED VIEW IF EXISTS ${mv}"""
90+
sql """DROP TABLE IF EXISTS ${table}"""
91+
sql """
92+
CREATE TABLE ${table} (
93+
k1 INT NOT NULL,
94+
v1 INT NULL,
95+
spare INT NULL
96+
)
97+
UNIQUE KEY(k1)
98+
DISTRIBUTED BY HASH(k1) BUCKETS 1
99+
PROPERTIES ('replication_num' = '1', 'enable_unique_key_merge_on_write' = 'true',
100+
'binlog.enable' = 'true', 'binlog.format' = 'ROW',
101+
'binlog.need_historical_value' = 'true')
102+
"""
103+
sql """INSERT INTO ${table} VALUES (1, 10, 1), (2, 20, 1)"""
104+
105+
sql """
106+
CREATE MATERIALIZED VIEW ${mv}
107+
BUILD DEFERRED REFRESH ${c.ivm ? 'INCREMENTAL' : 'COMPLETE'} ON MANUAL
108+
KEY(k1)
109+
DISTRIBUTED BY HASH(k1) BUCKETS 1
110+
PROPERTIES (${c.mvProps})
111+
AS SELECT k1, SUM(v1) AS total FROM ${table} GROUP BY k1
112+
"""
113+
def task = refresh(mv, "COMPLETE")
114+
assertEquals("SUCCESS", task.Status.toString(), "${c.name} baseline refresh: " + task.ErrorMsg)
115+
116+
// A schema change that leaves the query intact: the MV must stay refreshable. The dropped
117+
// column is not referenced, so nothing about the MV's result changes.
118+
def before = ddlJobCount(table)
119+
sql """ALTER TABLE ${table} DROP COLUMN spare"""
120+
waitDdlFinished(table, before)
121+
122+
task = refresh(mv, c.ivm ? "INCREMENTAL" : "COMPLETE")
123+
assertEquals("SUCCESS", task.Status.toString(),
124+
"${c.name} refresh after an unrelated DROP COLUMN: " + task.ErrorMsg)
125+
}
126+
127+
// The dropped column was never referenced, so all four MVs must still hold their baseline rows.
128+
order_qt_rb_ivm_rows "SELECT k1, total FROM rb_ivm_mv"
129+
order_qt_rb_ivm_no_binlog_rows "SELECT k1, total FROM rb_ivm_no_binlog_mv"
130+
order_qt_rb_dup_rows "SELECT k1, total FROM rb_dup_mv"
131+
order_qt_rb_dup_no_binlog_rows "SELECT k1, total FROM rb_dup_no_binlog_mv"
132+
133+
// ------------------------------------- 2. DORIS-28306: cascade L1 must recover via COMPLETE
134+
// A cascade forces row binlog onto L1, so L1 is exactly the shape that used to make every
135+
// refresh of L1 fail after a schema change. An ABA on a referenced column must still leave the
136+
// explicit COMPLETE recovery path usable.
137+
def cTable = "cascade_aba_base"
138+
sql """DROP MATERIALIZED VIEW IF EXISTS cascade_l2"""
139+
sql """DROP MATERIALIZED VIEW IF EXISTS cascade_l1"""
140+
sql """DROP TABLE IF EXISTS ${cTable}"""
141+
sql """
142+
CREATE TABLE ${cTable} (
143+
id BIGINT NOT NULL,
144+
grp INT NULL,
145+
amount BIGINT NULL
146+
)
147+
UNIQUE KEY(id)
148+
DISTRIBUTED BY HASH(id) BUCKETS 1
149+
PROPERTIES ('replication_num' = '1', 'enable_unique_key_merge_on_write' = 'true',
150+
'binlog.enable' = 'true', 'binlog.format' = 'ROW',
151+
'binlog.need_historical_value' = 'true')
152+
"""
153+
sql """INSERT INTO ${cTable} VALUES (1, 10, 100), (2, 10, 200), (3, 20, 300)"""
154+
155+
sql """
156+
CREATE MATERIALIZED VIEW cascade_l1
157+
BUILD DEFERRED REFRESH INCREMENTAL ON MANUAL KEY(id)
158+
PROPERTIES (${cascadeProps})
159+
AS SELECT id, grp, amount FROM ${cTable}
160+
"""
161+
def cTask = refresh("cascade_l1", "COMPLETE")
162+
assertEquals("SUCCESS", cTask.Status.toString(), "cascade L1 baseline: " + cTask.ErrorMsg)
163+
164+
sql """
165+
CREATE MATERIALIZED VIEW cascade_l2
166+
BUILD DEFERRED REFRESH INCREMENTAL ON MANUAL KEY(grp)
167+
PROPERTIES (${cascadeProps})
168+
AS SELECT grp, COUNT(*) AS row_count, SUM(amount) AS total_amount FROM cascade_l1 GROUP BY grp
169+
"""
170+
cTask = refresh("cascade_l2", "COMPLETE")
171+
assertEquals("SUCCESS", cTask.Status.toString(), "cascade L2 baseline: " + cTask.ErrorMsg)
172+
order_qt_cascade_l2_baseline "SELECT grp, row_count, total_amount FROM cascade_l2"
173+
174+
def cBefore = ddlJobCount(cTable)
175+
sql """ALTER TABLE ${cTable} DROP COLUMN grp"""
176+
waitDdlFinished(cTable, cBefore)
177+
cBefore = ddlJobCount(cTable)
178+
sql """ALTER TABLE ${cTable} ADD COLUMN grp INT NULL DEFAULT '0'"""
179+
waitDdlFinished(cTable, cBefore)
180+
181+
// COMPLETE is the recovery path: it has to rebuild L1 from the current base-table semantics,
182+
// where every pre-existing row now reads the re-added column's default value. Whether the
183+
// strict INCREMENTAL in between is rejected is a separate contract, owned by the suite for the
184+
// baseline invalidation itself, so it is deliberately not asserted here -- this suite has to
185+
// hold with or without that change.
186+
cTask = refresh("cascade_l1", "COMPLETE")
187+
assertEquals("SUCCESS", cTask.Status.toString(), "COMPLETE after schema ABA: " + cTask.ErrorMsg)
188+
order_qt_cascade_l1_after_aba "SELECT grp, COUNT(*) AS c, SUM(amount) AS s FROM cascade_l1 GROUP BY grp"
189+
190+
// ... and the downstream level stays refreshable afterwards.
191+
cTask = refresh("cascade_l2", "COMPLETE")
192+
assertEquals("SUCCESS", cTask.Status.toString(), "cascade L2 after L1 recovery: " + cTask.ErrorMsg)
193+
order_qt_cascade_l2_after_aba "SELECT grp, row_count, total_amount FROM cascade_l2"
194+
}

0 commit comments

Comments
 (0)