Skip to content

Commit d93ceac

Browse files
seawindeyujun777
authored andcommitted
[fix](fe) Mark IVM binlog broken on metadata data changes (apache#15)
### What problem does this PR solve? Issue Number: None Related PR: None Problem Summary: IVM incremental refresh depends on consumable row binlog deltas, but base table truncate, drop partition, replace partition, and recover partition can change visible data without producing row-level deltas. Mark dependent IVM materialized views as binlogBroken before those metadata data changes so the next refresh falls back to full refresh instead of using an incomplete incremental stream. ### Release note IVM refresh now falls back to full refresh after supported base table metadata data changes that break row-binlog continuity. ### Check List (For Author) - Test: Unit Test - ./run-fe-ut.sh --run org.apache.doris.mtmv.ivm.IvmInfoTest,org.apache.doris.mtmv.ivm.IvmStreamRefTest - ./run-fe-ut.sh --run org.apache.doris.mtmv.ivm.IvmBinlogBrokenTest - Behavior changed: Yes. Dependent IVM MVs are marked binlogBroken after truncate/drop partition/replace partition/recover partition on base tables, so later refresh uses fallback full refresh. - Does this need documentation: No
1 parent 69f1af0 commit d93ceac

9 files changed

Lines changed: 313 additions & 0 deletions

File tree

‎fe/fe-core/src/main/java/org/apache/doris/catalog/CatalogRecycleBin.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@
2929
import org.apache.doris.common.util.DynamicPartitionUtil;
3030
import org.apache.doris.common.util.MasterDaemon;
3131
import org.apache.doris.common.util.TimeUtils;
32+
import org.apache.doris.mtmv.BaseTableInfo;
3233
import org.apache.doris.persist.RecoverInfo;
3334
import org.apache.doris.persist.gson.GsonUtils;
3435
import org.apache.doris.qe.GlobalVariable;
@@ -1041,6 +1042,9 @@ public void recoverPartition(long dbId, OlapTable table, String partitionName,
10411042
recoverPartition.setName(newPartitionName);
10421043
}
10431044

1045+
// Recovering a partition restores visible rows from recycle metadata without row binlog entries.
1046+
Env.getCurrentEnv().getMtmvService().getRelationManager().markIvmBinlogBroken(
1047+
new BaseTableInfo(table), "Base table partition was recovered without row binlog");
10441048
// recover partition
10451049
table.addPartition(recoverPartition);
10461050

‎fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -150,6 +150,7 @@
150150
import org.apache.doris.master.PartitionInfoCollector;
151151
import org.apache.doris.meta.MetaContext;
152152
import org.apache.doris.metric.MetricRepo;
153+
import org.apache.doris.mtmv.BaseTableInfo;
153154
import org.apache.doris.mtmv.MTMVAlterOpType;
154155
import org.apache.doris.mtmv.MTMVPartitionExprFactory;
155156
import org.apache.doris.mtmv.MTMVPartitionInfo;
@@ -6999,6 +7000,9 @@ public void replaceTempPartition(Database db, OlapTable olapTable, ReplacePartit
69997000
throw new DdlException("Temp partition[" + partName + "] does not exist");
70007001
}
70017002
}
7003+
// Replacing partitions swaps visible data through metadata, so row-binlog continuity is broken.
7004+
getMtmvService().getRelationManager().markIvmBinlogBroken(
7005+
new BaseTableInfo(olapTable), "Base table partitions were replaced without row binlog");
70027006
List<Long> replacedPartitionIds = olapTable.replaceTempPartitions(db.getId(), partitionNames,
70037007
tempPartitionNames, isStrictRange,
70047008
useTempPartitionName, isForceDropOld);

‎fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1969,6 +1969,11 @@ public void dropPartition(Database db, OlapTable olapTable, DropPartitionOp drop
19691969
}
19701970
}
19711971

1972+
if (!isTempPartition) {
1973+
// Dropping a normal partition removes visible rows through metadata, not row binlog entries.
1974+
Env.getCurrentEnv().getMtmvService().getRelationManager().markIvmBinlogBroken(
1975+
new BaseTableInfo(olapTable), "Base table partition was dropped without row binlog");
1976+
}
19721977
dropPartitionWithoutCheck(db, olapTable, partitionName, isTempPartition, isForceDrop);
19731978
}
19741979

@@ -3685,6 +3690,11 @@ public void truncateTable(String dbName, String tableName, PartitionNamesInfo pa
36853690
throw new DdlException("Table[" + copiedTbl.getName() + "]'s meta has been changed. try again.");
36863691
}
36873692

3693+
if (!origPartitions.isEmpty()) {
3694+
// Truncate replaces partitions through metadata, so existing row-binlog streams become incomplete.
3695+
Env.getCurrentEnv().getMtmvService().getRelationManager().markIvmBinlogBroken(
3696+
new BaseTableInfo(olapTable), "Base table was truncated without row binlog");
3697+
}
36883698
//replace
36893699
Map<Long, RecyclePartitionParam> recyclePartitionParamMap = new HashMap<>();
36903700
oldPartitions = truncateTableInternal(olapTable, newPartitions,

‎fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelationManager.java‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@
2929
import org.apache.doris.job.exception.JobException;
3030
import org.apache.doris.job.extensions.mtmv.MTMVTask;
3131
import org.apache.doris.mtmv.MTMVRefreshEnum.MTMVState;
32+
import org.apache.doris.mtmv.ivm.IvmInfo;
3233
import org.apache.doris.nereids.rules.exploration.mv.PartitionCompensator;
3334
import org.apache.doris.nereids.trees.plans.commands.info.CancelMTMVTaskInfo;
3435
import org.apache.doris.nereids.trees.plans.commands.info.PauseMTMVInfo;
@@ -83,6 +84,46 @@ public Set<BaseTableInfo> getMtmvsByBaseTableOneLevelAndFromView(BaseTableInfo t
8384
return tableMTMVsOneLevelAndFromView.getOrDefault(table, ImmutableSet.of());
8485
}
8586

87+
/**
88+
* Mark dependent IVM MVs as binlog-broken before a base-table metadata data change.
89+
* These operations can change visible rows without producing consumable row binlog deltas.
90+
*/
91+
public void markIvmBinlogBroken(BaseTableInfo baseTableInfo, String reason) {
92+
Set<BaseTableInfo> mtmvsByBaseTable = getMtmvsByBaseTableOneLevelAndFromView(baseTableInfo);
93+
if (CollectionUtils.isEmpty(mtmvsByBaseTable)) {
94+
return;
95+
}
96+
for (BaseTableInfo mtmvInfo : mtmvsByBaseTable) {
97+
MTMV mtmv;
98+
try {
99+
mtmv = MTMVUtil.getMTMV(mtmvInfo);
100+
} catch (AnalysisException e) {
101+
LOG.warn("Skip marking IVM binlog broken because dependent MTMV is missing, "
102+
+ "baseTable={}, mtmv={}, reason={}", baseTableInfo, mtmvInfo, reason, e);
103+
continue;
104+
}
105+
if (!mtmv.isIvm()) {
106+
continue;
107+
}
108+
IvmInfo currentInfo = mtmv.getIvmInfo();
109+
if (currentInfo.isBinlogBroken()) {
110+
continue;
111+
}
112+
// Persist a copied metadata object so callers do not mutate the current MTMV state in place.
113+
IvmInfo newInfo = new IvmInfo(currentInfo);
114+
newInfo.setBinlogBroken(true);
115+
try {
116+
TableNameInfo tableNameInfo = new TableNameInfo(mtmv.getQualifiedDbName(), mtmv.getName());
117+
Env.getCurrentEnv().alterMTMVIvmInfo(tableNameInfo, newInfo);
118+
LOG.info("Marked IVM binlog broken, baseTable={}, mtmv={}, reason={}",
119+
baseTableInfo, mtmvInfo, reason);
120+
} catch (Exception e) {
121+
LOG.error("Failed to mark IVM binlog broken, baseTable={}, mtmv={}, reason={}",
122+
baseTableInfo, mtmvInfo, reason, e);
123+
}
124+
}
125+
}
126+
86127
/**
87128
* if At least one partition is available, return this mtmv
88129
*

‎fe/fe-core/src/main/java/org/apache/doris/mtmv/ivm/IvmInfo.java‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,19 @@ public IvmInfo() {
4646
this.baseTableStreams = Maps.newHashMap();
4747
}
4848

49+
/** Deep-copy persisted IVM metadata before publishing an updated edit-log entry. */
50+
public IvmInfo(IvmInfo other) {
51+
this.enableIvm = other.enableIvm;
52+
this.binlogBroken = other.binlogBroken;
53+
this.runningIvmRefresh = other.runningIvmRefresh;
54+
this.baseTableStreams = Maps.newHashMap();
55+
if (other.baseTableStreams != null) {
56+
for (Map.Entry<BaseTableInfo, IvmStreamRef> entry : other.baseTableStreams.entrySet()) {
57+
this.baseTableStreams.put(entry.getKey(), new IvmStreamRef(entry.getValue()));
58+
}
59+
}
60+
}
61+
4962
public boolean isEnableIvm() {
5063
return enableIvm;
5164
}

‎fe/fe-core/src/main/java/org/apache/doris/mtmv/ivm/IvmStreamRef.java‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,12 @@ public IvmStreamRef(long consumedTso) {
4545
this.latestTso = 0;
4646
}
4747

48+
/** Copy both persisted and transient stream positions for IVM metadata updates. */
49+
public IvmStreamRef(IvmStreamRef other) {
50+
this.consumedTso = other.consumedTso;
51+
this.latestTso = other.latestTso;
52+
}
53+
4854
public long getConsumedTso() {
4955
return consumedTso;
5056
}
Lines changed: 153 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,153 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
package org.apache.doris.mtmv.ivm;
19+
20+
import org.apache.doris.catalog.Env;
21+
import org.apache.doris.catalog.MTMV;
22+
import org.apache.doris.catalog.info.TableNameInfo;
23+
import org.apache.doris.utframe.TestWithFeService;
24+
25+
import org.junit.jupiter.api.Assertions;
26+
import org.junit.jupiter.api.Test;
27+
28+
public class IvmBinlogBrokenTest extends TestWithFeService {
29+
30+
@Test
31+
public void testTruncateMarksBinlogBroken() throws Exception {
32+
String db = "ivm_broken_truncate";
33+
createPartitionedIvmTableAndMv(db);
34+
35+
executeSql("TRUNCATE TABLE ivm_base");
36+
37+
Assertions.assertTrue(getMtmv(db).getIvmInfo().isBinlogBroken());
38+
}
39+
40+
@Test
41+
public void testTruncatePartitionMarksBinlogBroken() throws Exception {
42+
String db = "ivm_broken_truncate_partition";
43+
createPartitionedIvmTableAndMv(db);
44+
45+
executeSql("TRUNCATE TABLE ivm_base PARTITION(p202001)");
46+
47+
Assertions.assertTrue(getMtmv(db).getIvmInfo().isBinlogBroken());
48+
}
49+
50+
@Test
51+
public void testDropPartitionMarksBinlogBroken() throws Exception {
52+
String db = "ivm_broken_drop_partition";
53+
createPartitionedIvmTableAndMv(db);
54+
55+
executeSql("ALTER TABLE ivm_base DROP PARTITION p202001");
56+
57+
Assertions.assertTrue(getMtmv(db).getIvmInfo().isBinlogBroken());
58+
}
59+
60+
@Test
61+
public void testReplacePartitionMarksBinlogBroken() throws Exception {
62+
String db = "ivm_broken_replace_partition";
63+
createPartitionedIvmTableAndMv(db);
64+
65+
executeSql("ALTER TABLE ivm_base ADD TEMPORARY PARTITION tp202001 "
66+
+ "VALUES [('2020-01-01'), ('2020-02-01'))");
67+
executeSql("ALTER TABLE ivm_base REPLACE PARTITION (p202001) "
68+
+ "WITH TEMPORARY PARTITION (tp202001)");
69+
70+
Assertions.assertTrue(getMtmv(db).getIvmInfo().isBinlogBroken());
71+
}
72+
73+
@Test
74+
public void testRecoverPartitionMarksBinlogBroken() throws Exception {
75+
String db = "ivm_broken_recover_partition";
76+
createPartitionedIvmTableAndMv(db);
77+
executeSql("ALTER TABLE ivm_base DROP PARTITION p202001");
78+
setBinlogBroken(getMtmv(db), false);
79+
80+
executeSql("RECOVER PARTITION p202001 FROM ivm_base");
81+
82+
Assertions.assertTrue(getMtmv(db).getIvmInfo().isBinlogBroken());
83+
}
84+
85+
@Test
86+
public void testAddPartitionDoesNotMarkBinlogBroken() throws Exception {
87+
String db = "ivm_broken_add_partition";
88+
createPartitionedIvmTableAndMv(db);
89+
90+
executeSql("ALTER TABLE ivm_base ADD PARTITION p202003 "
91+
+ "VALUES [('2020-03-01'), ('2020-04-01'))");
92+
93+
Assertions.assertFalse(getMtmv(db).getIvmInfo().isBinlogBroken());
94+
}
95+
96+
@Test
97+
public void testDropTempPartitionDoesNotMarkBinlogBroken() throws Exception {
98+
String db = "ivm_broken_drop_temp_partition";
99+
createPartitionedIvmTableAndMv(db);
100+
executeSql("ALTER TABLE ivm_base ADD TEMPORARY PARTITION tp202001 "
101+
+ "VALUES [('2020-01-01'), ('2020-02-01'))");
102+
103+
executeSql("ALTER TABLE ivm_base DROP TEMPORARY PARTITION tp202001");
104+
105+
Assertions.assertFalse(getMtmv(db).getIvmInfo().isBinlogBroken());
106+
}
107+
108+
@Test
109+
public void testDropMissingPartitionIfExistsDoesNotMarkBinlogBroken() throws Exception {
110+
String db = "ivm_broken_drop_missing_partition";
111+
createPartitionedIvmTableAndMv(db);
112+
113+
executeSql("ALTER TABLE ivm_base DROP PARTITION IF EXISTS p_missing");
114+
115+
Assertions.assertFalse(getMtmv(db).getIvmInfo().isBinlogBroken());
116+
}
117+
118+
private void createPartitionedIvmTableAndMv(String db) throws Exception {
119+
createDatabaseAndUse(db);
120+
createTable("CREATE TABLE " + db + ".ivm_base (\n"
121+
+ " dt date NOT NULL,\n"
122+
+ " k1 int,\n"
123+
+ " v1 int\n"
124+
+ ")\n"
125+
+ "DUPLICATE KEY(dt, k1)\n"
126+
+ "PARTITION BY RANGE(dt) (\n"
127+
+ " PARTITION p202001 VALUES [('2020-01-01'), ('2020-02-01')),\n"
128+
+ " PARTITION p202002 VALUES [('2020-02-01'), ('2020-03-01'))\n"
129+
+ ")\n"
130+
+ "DISTRIBUTED BY HASH(k1) BUCKETS 1\n"
131+
+ "PROPERTIES ('replication_num' = '1', 'binlog.enable' = 'true', 'binlog.format' = 'ROW')");
132+
createMvByNereids("CREATE MATERIALIZED VIEW ivm_mv\n"
133+
+ "BUILD DEFERRED REFRESH INCREMENTAL ON MANUAL\n"
134+
+ "DISTRIBUTED BY RANDOM BUCKETS 1\n"
135+
+ "PROPERTIES ('replication_num' = '1')\n"
136+
+ "AS SELECT dt, k1, v1 FROM ivm_base");
137+
Assertions.assertTrue(getMtmv(db).isIvm());
138+
Assertions.assertFalse(getMtmv(db).getIvmInfo().isBinlogBroken());
139+
}
140+
141+
private MTMV getMtmv(String db) throws Exception {
142+
return (MTMV) Env.getCurrentInternalCatalog()
143+
.getDb(db).get()
144+
.getTableOrMetaException("ivm_mv");
145+
}
146+
147+
private void setBinlogBroken(MTMV mtmv, boolean binlogBroken) {
148+
IvmInfo info = new IvmInfo(mtmv.getIvmInfo());
149+
info.setBinlogBroken(binlogBroken);
150+
Env.getCurrentEnv().alterMTMVIvmInfo(
151+
new TableNameInfo(mtmv.getQualifiedDbName(), mtmv.getName()), info);
152+
}
153+
}
Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
package org.apache.doris.mtmv.ivm;
19+
20+
import org.apache.doris.mtmv.BaseTableInfo;
21+
22+
import org.junit.jupiter.api.Assertions;
23+
import org.junit.jupiter.api.Test;
24+
import org.mockito.Mockito;
25+
26+
import java.util.HashMap;
27+
import java.util.Map;
28+
29+
public class IvmInfoTest {
30+
31+
@Test
32+
public void testCopyConstructorDeepCopiesStreams() {
33+
IvmInfo info = new IvmInfo();
34+
info.setEnableIvm(true);
35+
info.setBinlogBroken(true);
36+
info.setRunningIvmRefresh(true);
37+
BaseTableInfo baseTableInfo = Mockito.mock(BaseTableInfo.class);
38+
IvmStreamRef streamRef = new IvmStreamRef(42L);
39+
streamRef.setLatestTso(100L);
40+
Map<BaseTableInfo, IvmStreamRef> streams = new HashMap<>();
41+
streams.put(baseTableInfo, streamRef);
42+
info.setBaseTableStreams(streams);
43+
44+
IvmInfo copy = new IvmInfo(info);
45+
46+
Assertions.assertTrue(copy.isEnableIvm());
47+
Assertions.assertTrue(copy.isBinlogBroken());
48+
Assertions.assertTrue(copy.isRunningIvmRefresh());
49+
Assertions.assertEquals(1, copy.getBaseTableStreams().size());
50+
IvmStreamRef copiedRef = copy.getBaseTableStreams().get(baseTableInfo);
51+
Assertions.assertNotSame(streamRef, copiedRef);
52+
Assertions.assertEquals(42L, copiedRef.getConsumedTso());
53+
Assertions.assertEquals(100L, copiedRef.getLatestTso());
54+
55+
copiedRef.setConsumedTso(200L);
56+
Assertions.assertEquals(42L, streamRef.getConsumedTso());
57+
}
58+
59+
@Test
60+
public void testCopyConstructorHandlesNullStreams() {
61+
IvmInfo info = new IvmInfo();
62+
info.setBaseTableStreams(null);
63+
64+
IvmInfo copy = new IvmInfo(info);
65+
66+
Assertions.assertNotNull(copy.getBaseTableStreams());
67+
Assertions.assertTrue(copy.getBaseTableStreams().isEmpty());
68+
}
69+
}

‎fe/fe-core/src/test/java/org/apache/doris/mtmv/ivm/IvmStreamRefTest.java‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,19 @@ public void testConstructorWithConsumedTso() {
3636
Assertions.assertEquals(0, ref.getLatestTso());
3737
}
3838

39+
@Test
40+
public void testCopyConstructor() {
41+
IvmStreamRef ref = new IvmStreamRef(42L);
42+
ref.setLatestTso(100L);
43+
44+
IvmStreamRef copy = new IvmStreamRef(ref);
45+
46+
Assertions.assertEquals(42L, copy.getConsumedTso());
47+
Assertions.assertEquals(100L, copy.getLatestTso());
48+
copy.setConsumedTso(200L);
49+
Assertions.assertEquals(42L, ref.getConsumedTso());
50+
}
51+
3952
@Test
4053
public void testIsUpToDateWhenEqual() {
4154
IvmStreamRef ref = new IvmStreamRef(10L);

0 commit comments

Comments
 (0)