Skip to content

Commit 79fbe1c

Browse files
seawindeyujun777
authored andcommitted
[fix](fe) Derive IVM binlog broken from DDL (apache#17)
* [fix](fe) Derive IVM binlog broken from DDL ### What problem does this PR solve? Issue Number: None Related PR: apache#15 Problem Summary: DDL operations such as truncate, drop partition, replace partition, and recover partition can change base table visible rows through metadata without row binlog deltas. Persisting the IVM binlogBroken mark through a separate ALTER_IVM_INFO edit log makes the mark independent from the base-table DDL replay sequence. This change treats DDL-caused binlogBroken as DDL-derived memory state: master DDL paths mark the in-memory MTMV state, and DDL replay paths derive the same mark from the base-table DDL log. ALTER_IVM_INFO remains available for refresh-owned IVM state such as refresh progress. ### Release note None ### Check List (For Author) - Test: Unit Test - ../run-fe-ut.sh --run org.apache.doris.mtmv.ivm.IvmBinlogBrokenTest - Behavior changed: Yes. DDL-caused IVM binlogBroken state is derived from base-table DDL logs instead of being persisted as a separate MTMV ALTER_IVM_INFO log. - Does this need documentation: No * [fix](fe) Use explicit db id for recycle bin recover
1 parent c57d471 commit 79fbe1c

7 files changed

Lines changed: 118 additions & 15 deletions

File tree

‎fe/fe-core/src/main/java/org/apache/doris/catalog/CatalogRecycleBin.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1105,6 +1105,9 @@ public void replayRecoverPartition(OlapTable table, long partitionId,
11051105
throw new DdlException("Partition name[" + newPartitionName + "] is already used");
11061106
}
11071107
}
1108+
Env.getCurrentEnv().getMtmvService().getRelationManager().markIvmBinlogBroken(
1109+
new BaseTableInfo(table, recyclePartitionInfo.getDbId()),
1110+
"Base table partition was recovered without row binlog");
11081111
table.addPartition(recyclePartitionInfo.getPartition());
11091112
if (!Strings.isNullOrEmpty(newPartitionName)) {
11101113
table.renamePartition(recyclePartitionInfo.getPartition().getName(), newPartitionName);

‎fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7054,6 +7054,8 @@ public void replayReplaceTempPartition(ReplacePartitionOperationLog replaceTempP
70547054
.getTableOrMetaException(tableId, Lists.newArrayList(TableType.OLAP, TableType.MATERIALIZED_VIEW));
70557055
olapTable.writeLock();
70567056
try {
7057+
getMtmvService().getRelationManager().markIvmBinlogBroken(
7058+
new BaseTableInfo(olapTable), "Base table partitions were replaced without row binlog");
70577059
olapTable.replaceTempPartitions(dbId, replaceTempPartitionLog.getPartitions(),
70587060
replaceTempPartitionLog.getTempPartitions(), replaceTempPartitionLog.isStrictRange(),
70597061
replaceTempPartitionLog.useTempPartitionName(), replaceTempPartitionLog.isForce());

‎fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -500,6 +500,22 @@ public void alterIvmInfo(IvmInfo ivmInfo) {
500500
}
501501
}
502502

503+
public boolean markIvmBinlogBroken() {
504+
writeMvLock();
505+
try {
506+
if (ivmInfo == null) {
507+
ivmInfo = new IvmInfo();
508+
}
509+
if (ivmInfo.isBinlogBroken()) {
510+
return false;
511+
}
512+
ivmInfo.setBinlogBroken(true);
513+
return true;
514+
} finally {
515+
writeMvUnlock();
516+
}
517+
}
518+
503519
public List<String> getInsertedColumnNames() {
504520
List<Column> columns = getBaseSchema(true);
505521
List<String> columnNames = Lists.newArrayListWithExpectedSize(columns.size());

‎fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2041,6 +2041,8 @@ public void replayDropPartition(DropPartitionInfo info) throws MetaNotFoundExcep
20412041
if (info.isTempPartition()) {
20422042
olapTable.dropTempPartition(info.getPartitionName(), true);
20432043
} else {
2044+
Env.getCurrentEnv().getMtmvService().getRelationManager().markIvmBinlogBroken(
2045+
new BaseTableInfo(olapTable), "Base table partition was dropped without row binlog");
20442046
partition = olapTable.dropPartition(info.getDbId(), info.getPartitionName(), info.isForceDrop());
20452047
if (!info.isForceDrop() && partition != null && info.getRecycleTime() != 0) {
20462048
Env.getCurrentRecycleBin().setRecycleTimeByIdForReplay(partition.getId(), info.getRecycleTime());
@@ -3771,6 +3773,10 @@ public void replayTruncateTable(TruncateTableInfo info) throws MetaNotFoundExcep
37713773
olapTable.writeLock();
37723774
try {
37733775
Map<Long, RecyclePartitionParam> recyclePartitionParamMap = new HashMap<>();
3776+
if (!info.getPartitions().isEmpty()) {
3777+
Env.getCurrentEnv().getMtmvService().getRelationManager().markIvmBinlogBroken(
3778+
new BaseTableInfo(olapTable), "Base table was truncated without row binlog");
3779+
}
37743780
truncateTableInternal(olapTable, info.getPartitions(), info.isEntireTable(),
37753781
recyclePartitionParamMap, isForceDrop);
37763782

‎fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelationManager.java‎

Lines changed: 1 addition & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,6 @@
2929
import org.apache.doris.job.exception.JobException;
3030
import org.apache.doris.job.extensions.mtmv.MTMVTask;
3131
import org.apache.doris.mtmv.MTMVRefreshEnum.MTMVState;
32-
import org.apache.doris.mtmv.ivm.IvmInfo;
3332
import org.apache.doris.nereids.rules.exploration.mv.PartitionCompensator;
3433
import org.apache.doris.nereids.trees.plans.commands.info.CancelMTMVTaskInfo;
3534
import org.apache.doris.nereids.trees.plans.commands.info.PauseMTMVInfo;
@@ -105,21 +104,9 @@ public void markIvmBinlogBroken(BaseTableInfo baseTableInfo, String reason) {
105104
if (!mtmv.isIvm()) {
106105
continue;
107106
}
108-
IvmInfo currentInfo = mtmv.getIvmInfo();
109-
if (currentInfo.isBinlogBroken()) {
110-
continue;
111-
}
112-
// Persist a copied metadata object so callers do not mutate the current MTMV state in place.
113-
IvmInfo newInfo = new IvmInfo(currentInfo);
114-
newInfo.setBinlogBroken(true);
115-
try {
116-
TableNameInfo tableNameInfo = new TableNameInfo(mtmv.getQualifiedDbName(), mtmv.getName());
117-
Env.getCurrentEnv().alterMTMVIvmInfo(tableNameInfo, newInfo);
107+
if (mtmv.markIvmBinlogBroken()) {
118108
LOG.info("Marked IVM binlog broken, baseTable={}, mtmv={}, reason={}",
119109
baseTableInfo, mtmvInfo, reason);
120-
} catch (Exception e) {
121-
LOG.error("Failed to mark IVM binlog broken, baseTable={}, mtmv={}, reason={}",
122-
baseTableInfo, mtmvInfo, reason, e);
123110
}
124111
}
125112
}

‎fe/fe-core/src/test/java/org/apache/doris/mtmv/ivm/IvmBinlogBrokenTest.java‎

Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,14 +17,24 @@
1717

1818
package org.apache.doris.mtmv.ivm;
1919

20+
import org.apache.doris.catalog.Database;
2021
import org.apache.doris.catalog.Env;
2122
import org.apache.doris.catalog.MTMV;
23+
import org.apache.doris.catalog.MaterializedIndex;
24+
import org.apache.doris.catalog.OlapTable;
25+
import org.apache.doris.catalog.Partition;
2226
import org.apache.doris.catalog.info.TableNameInfo;
27+
import org.apache.doris.persist.DropPartitionInfo;
28+
import org.apache.doris.persist.RecoverInfo;
29+
import org.apache.doris.persist.ReplacePartitionOperationLog;
30+
import org.apache.doris.persist.TruncateTableInfo;
2331
import org.apache.doris.utframe.TestWithFeService;
2432

2533
import org.junit.jupiter.api.Assertions;
2634
import org.junit.jupiter.api.Test;
2735

36+
import java.util.Collections;
37+
2838
public class IvmBinlogBrokenTest extends TestWithFeService {
2939

3040
@Test
@@ -51,9 +61,11 @@ public void testTruncatePartitionMarksBinlogBroken() throws Exception {
5161
public void testDropPartitionMarksBinlogBroken() throws Exception {
5262
String db = "ivm_broken_drop_partition";
5363
createPartitionedIvmTableAndMv(db);
64+
IvmInfo ivmInfo = getMtmv(db).getIvmInfo();
5465

5566
executeSql("ALTER TABLE ivm_base DROP PARTITION p202001");
5667

68+
Assertions.assertSame(ivmInfo, getMtmv(db).getIvmInfo());
5769
Assertions.assertTrue(getMtmv(db).getIvmInfo().isBinlogBroken());
5870
}
5971

@@ -115,6 +127,76 @@ public void testDropMissingPartitionIfExistsDoesNotMarkBinlogBroken() throws Exc
115127
Assertions.assertFalse(getMtmv(db).getIvmInfo().isBinlogBroken());
116128
}
117129

130+
@Test
131+
public void testReplayDropPartitionMarksBinlogBroken() throws Exception {
132+
String db = "ivm_broken_replay_drop_partition";
133+
createPartitionedIvmTableAndMv(db);
134+
Database database = getDb(db);
135+
OlapTable table = getBaseTable(db);
136+
Partition partition = table.getPartition("p202001");
137+
138+
DropPartitionInfo info = new DropPartitionInfo(database.getId(), table.getId(), partition.getId(),
139+
"p202001", false, false, 0L, table.getVisibleVersion(), table.getVisibleVersionTime());
140+
Env.getCurrentInternalCatalog().replayDropPartition(info);
141+
142+
Assertions.assertTrue(getMtmv(db).getIvmInfo().isBinlogBroken());
143+
}
144+
145+
@Test
146+
public void testReplayTruncateMarksBinlogBroken() throws Exception {
147+
String db = "ivm_broken_replay_truncate";
148+
createPartitionedIvmTableAndMv(db);
149+
Database database = getDb(db);
150+
OlapTable table = getBaseTable(db);
151+
Partition oldPartition = table.getPartition("p202001");
152+
Partition newPartition = new Partition(Env.getCurrentEnv().getNextId(), oldPartition.getName(),
153+
new MaterializedIndex(table.getBaseIndexId(), MaterializedIndex.IndexState.NORMAL),
154+
oldPartition.getDistributionInfo());
155+
156+
TruncateTableInfo info = new TruncateTableInfo(database.getId(), database.getFullName(), table.getId(),
157+
table.getName(), Collections.singletonList(newPartition), false,
158+
"TRUNCATE TABLE ivm_base PARTITION(p202001)", Collections.singletonList(oldPartition), true,
159+
Collections.emptyMap());
160+
Env.getCurrentInternalCatalog().replayTruncateTable(info);
161+
162+
Assertions.assertTrue(getMtmv(db).getIvmInfo().isBinlogBroken());
163+
}
164+
165+
@Test
166+
public void testReplayReplacePartitionMarksBinlogBroken() throws Exception {
167+
String db = "ivm_broken_replay_replace_partition";
168+
createPartitionedIvmTableAndMv(db);
169+
executeSql("ALTER TABLE ivm_base ADD TEMPORARY PARTITION tp202001 "
170+
+ "VALUES [('2020-01-01'), ('2020-02-01'))");
171+
Database database = getDb(db);
172+
OlapTable table = getBaseTable(db);
173+
174+
ReplacePartitionOperationLog log = new ReplacePartitionOperationLog(database.getId(), database.getFullName(),
175+
table.getId(), table.getName(), Collections.singletonList("p202001"),
176+
Collections.singletonList("tp202001"), Collections.emptyList(), false, false,
177+
table.getVisibleVersion(), table.getVisibleVersionTime(), false);
178+
Env.getCurrentEnv().replayReplaceTempPartition(log);
179+
180+
Assertions.assertTrue(getMtmv(db).getIvmInfo().isBinlogBroken());
181+
}
182+
183+
@Test
184+
public void testReplayRecoverPartitionMarksBinlogBroken() throws Exception {
185+
String db = "ivm_broken_replay_recover_partition";
186+
createPartitionedIvmTableAndMv(db);
187+
Database database = getDb(db);
188+
OlapTable table = getBaseTable(db);
189+
long partitionId = table.getPartition("p202001").getId();
190+
executeSql("ALTER TABLE ivm_base DROP PARTITION p202001");
191+
setBinlogBroken(getMtmv(db), false);
192+
193+
RecoverInfo info = new RecoverInfo(database.getId(), table.getId(), partitionId, "", table.getName(),
194+
"", "p202001", null);
195+
Env.getCurrentInternalCatalog().replayRecoverPartition(info);
196+
197+
Assertions.assertTrue(getMtmv(db).getIvmInfo().isBinlogBroken());
198+
}
199+
118200
private void createPartitionedIvmTableAndMv(String db) throws Exception {
119201
createDatabaseAndUse(db);
120202
createTable("CREATE TABLE " + db + ".ivm_base (\n"
@@ -144,6 +226,14 @@ private MTMV getMtmv(String db) throws Exception {
144226
.getTableOrMetaException("ivm_mv");
145227
}
146228

229+
private Database getDb(String db) {
230+
return Env.getCurrentInternalCatalog().getDb(db).get();
231+
}
232+
233+
private OlapTable getBaseTable(String db) throws Exception {
234+
return (OlapTable) getDb(db).getTableOrMetaException("ivm_base");
235+
}
236+
147237
private void setBinlogBroken(MTMV mtmv, boolean binlogBroken) {
148238
IvmInfo info = new IvmInfo(mtmv.getIvmInfo());
149239
info.setBinlogBroken(binlogBroken);

‎fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/IvmNormalizeMtmvTest.java‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,6 @@
4141
import org.apache.doris.nereids.jobs.JobContext;
4242
import org.apache.doris.nereids.properties.PhysicalProperties;
4343
import org.apache.doris.nereids.trees.expressions.Alias;
44-
import org.apache.doris.nereids.trees.expressions.Cast;
4544
import org.apache.doris.nereids.trees.expressions.ExprId;
4645
import org.apache.doris.nereids.trees.expressions.Expression;
4746
import org.apache.doris.nereids.trees.expressions.NamedExpression;

0 commit comments

Comments
 (0)