Skip to content

Commit ff3f291

Browse files
committed
[fix](ivm) Rebuild the whole MV when the changed partitions may be outside the sync window
The mapping that invalidation reads is filtered by partition_sync_limit, so an empty selection does not by itself say that no MV partition holds the changed rows: a base partition the window has dropped can still have its rows in an MV partition, because shrinking the window does not touch the MV's own partitions and widening it again makes partition sync keep them. Reading the unfiltered lineage instead is precise, but it turns every partition of the base table's history into MV partitions while the read locks the selection takes are held, and the DDL that triggered it holds the changed table's write lock. Key changes: - Rebuild the whole MV when the selection comes out empty while a partition_sync_limit is in effect; without one the windowed and the full mapping are the same set, so an empty selection still means that no MV partition holds those rows - Read that from the generator's own parsing, exposed as MTMVPartitionUtil.isPartitionSyncLimitActive, so the property is interpreted in one place - The regression asserts the terminal task's status rather than only the MV's rows: partition sync drops the MV partition first, so a failed refresh could leave a row set that matches Unit Test: - IvmBaselineRebuildTest: a changed partition outside the shrunk window rebuilds the whole MV (the test fails without the change) - Regression: mtmv_p0/ivm/test_ivm_baseline_marker_scope pins SUCCESS on the narrowed refresh, and the suite runs no differently otherwise
1 parent 7ad093e commit ff3f291

5 files changed

Lines changed: 46 additions & 83 deletions

File tree

‎fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java‎

Lines changed: 16 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -749,12 +749,8 @@ private Optional<Set<String>> selectAffectedMvPartitions(BaseTableInfo baseTable
749749
if (!pctTable.getPartitionNames().containsAll(changedBasePartitions.keySet())) {
750750
return Optional.empty();
751751
}
752-
// By lineage, not by the partition_sync_limit window: the window can move after a base
753-
// partition was read, and a partition it does not cover right now can still have its rows in
754-
// an MV partition. Partition sync keeps that MV partition once the window covers the base
755-
// partition again, and the change itself left no binlog to repair those rows with.
756752
Map<String, Map<MTMVRelatedTableIf, Set<String>>> partitionMappings =
757-
calculatePartitionMappingsByLineage();
753+
calculatePartitionMappings(Maps.newHashMap());
758754
Set<String> res = Sets.newHashSet();
759755
boolean pctTableMapped = false;
760756
for (Entry<String, Map<MTMVRelatedTableIf, Set<String>>> mapping : partitionMappings.entrySet()) {
@@ -782,6 +778,18 @@ private Optional<Set<String>> selectAffectedMvPartitions(BaseTableInfo baseTable
782778
+ "baseTable={}, mv={}", baseTableInfo, name);
783779
return Optional.empty();
784780
}
781+
// An empty selection is only trustworthy while the mapping covers every base partition. With a
782+
// partition_sync_limit in effect it does not: the window leaves out the partitions it dropped,
783+
// and one of those can still have its rows in an MV partition -- shrinking the window does not
784+
// touch the MV's own partitions, and widening it again makes partition sync keep them. Those
785+
// two readings cannot be told apart here, so the whole MV is rebuilt instead; without a limit
786+
// the mapping is complete, and an empty selection really does mean no MV partition holds them.
787+
if (res.isEmpty() && MTMVPartitionUtil.isPartitionSyncLimitActive(mvProperties)) {
788+
LOG.info("Changed base partitions are outside the partition_sync_limit window and the MV may "
789+
+ "still hold their rows, rebuild the whole MV. baseTable={}, changedPartitions={}, "
790+
+ "mv={}", baseTableInfo, changedBasePartitions.keySet(), name);
791+
return Optional.empty();
792+
}
785793
return Optional.of(res);
786794
} catch (Exception e) {
787795
// The base table change is applied either way, so this must not fail the DDL: warn and take
@@ -1001,24 +1009,6 @@ public Map<String, Map<MTMVRelatedTableIf, Set<String>>> calculatePartitionMappi
10011009
public Map<String, Map<MTMVRelatedTableIf, Set<String>>> calculatePartitionMappings(
10021010
Map<List<String>, Set<String>> queryUsedBaseTablePartitionMap,
10031011
Map<MvccTableInfo, MvccSnapshot> pinnedSnapshots) throws AnalysisException {
1004-
return calculatePartitionMappings(queryUsedBaseTablePartitionMap, pinnedSnapshots, false);
1005-
}
1006-
1007-
/**
1008-
* The MV partitions each base partition belongs to by lineage, ignoring the partition_sync_limit
1009-
* window. Invalidating MV partitions has to use this: the window can move after a base partition was
1010-
* read, and a changed partition that the window does not cover right now can still have its rows in an
1011-
* MV partition -- partition sync keeps that partition once the window covers the base partition again,
1012-
* and the change itself left no binlog to repair it with.
1013-
*/
1014-
public Map<String, Map<MTMVRelatedTableIf, Set<String>>> calculatePartitionMappingsByLineage()
1015-
throws AnalysisException {
1016-
return calculatePartitionMappings(Maps.newHashMap(), null, true);
1017-
}
1018-
1019-
private Map<String, Map<MTMVRelatedTableIf, Set<String>>> calculatePartitionMappings(
1020-
Map<List<String>, Set<String>> queryUsedBaseTablePartitionMap,
1021-
Map<MvccTableInfo, MvccSnapshot> pinnedSnapshots, boolean byLineage) throws AnalysisException {
10221012
if (mvPartitionInfo.getPartitionType() == MTMVPartitionType.SELF_MANAGE) {
10231013
return Maps.newHashMap();
10241014
}
@@ -1034,11 +1024,9 @@ private Map<String, Map<MTMVRelatedTableIf, Set<String>>> calculatePartitionMapp
10341024
= getEffectiveQueryUsedBaseTablePartitionMap(
10351025
queryUsedBaseTablePartitionMap, mvPartitionItems, pinnedSnapshots);
10361026
Map<String, Map<MTMVRelatedTableIf, Set<String>>> res = Maps.newHashMap();
1037-
Map<PartitionKeyDesc, Map<MTMVRelatedTableIf, Set<String>>> pctPartitionDescs = byLineage
1038-
? MTMVPartitionUtil.generateRelatedPartitionDescsByLineage(mvPartitionInfo, mvProperties,
1039-
getPartitionColumns(), effectiveFilter, pinnedSnapshots)
1040-
: MTMVPartitionUtil.generateRelatedPartitionDescs(mvPartitionInfo, mvProperties,
1041-
getPartitionColumns(), effectiveFilter, pinnedSnapshots);
1027+
Map<PartitionKeyDesc, Map<MTMVRelatedTableIf, Set<String>>> pctPartitionDescs = MTMVPartitionUtil
1028+
.generateRelatedPartitionDescs(mvPartitionInfo, mvProperties, getPartitionColumns(),
1029+
effectiveFilter, pinnedSnapshots);
10421030
for (Entry<String, PartitionItem> entry : mvPartitionItems.entrySet()) {
10431031
res.put(entry.getKey(),
10441032
pctPartitionDescs.getOrDefault(entry.getValue().toPartitionKeyDesc(), Maps.newHashMap()));

‎fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionUtil.java‎

Lines changed: 9 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -91,13 +91,14 @@ public class MTMVPartitionUtil {
9191
new MTMVRelatedPartitionDescTransferGenerator()
9292
);
9393

94-
// The same chain without the partition_sync_limit step, for callers that ask what a base partition
95-
// belongs to by lineage rather than what the window still covers. Derived from the chain above so
96-
// that a generator added there cannot be forgotten here.
97-
private static final List<MTMVRelatedPartitionDescGeneratorService> lineagePartitionDescGenerators =
98-
partitionDescGenerators.stream()
99-
.filter(generator -> !(generator instanceof MTMVRelatedPartitionDescSyncLimitGenerator))
100-
.collect(ImmutableList.toImmutableList());
94+
/**
95+
* Whether a partition_sync_limit is in effect, i.e. whether the windowed mapping leaves base partitions
96+
* out. Only that mapping is bounded by the window, so only there does its completeness depend on it.
97+
*/
98+
public static boolean isPartitionSyncLimitActive(Map<String, String> mvProperties) {
99+
return MTMVRelatedPartitionDescSyncLimitGenerator
100+
.generateMTMVPartitionSyncConfigByProperties(mvProperties).getSyncLimit() > 0;
101+
}
101102

102103
/**
103104
* Determine whether the partition is sync with retated partition and other baseTables
@@ -233,39 +234,9 @@ public static Map<PartitionKeyDesc, Map<MTMVRelatedTableIf, Set<String>>> genera
233234
Map<String, String> mvProperties, List<Column> partitionColumns,
234235
Map<List<String>, Set<String>> queryUsedPartitions,
235236
Map<MvccTableInfo, MvccSnapshot> pinnedSnapshots) throws AnalysisException {
236-
return generateRelatedPartitionDescs(mvPartitionInfo, mvProperties, partitionColumns,
237-
queryUsedPartitions, pinnedSnapshots, partitionDescGenerators);
238-
}
239-
240-
/**
241-
* The same partitions, mapped by lineage: a base partition is related to an MV partition when the MV's
242-
* partition definition puts it there, whether or not the partition_sync_limit window covers it now.
243-
*
244-
* <p>Invalidating MV partitions needs this rather than the windowed mapping, because the window can
245-
* move after a base partition was read. A partition the window no longer covers, or did not cover when
246-
* the change happened, can still have its rows in an MV partition: the MV partition is only dropped
247-
* when partition sync regenerates the MV's partition set, and a sync limit that is later widened puts
248-
* the base partition back into that set, leaving the MV partition -- and the rows of the change that
249-
* no binlog recorded -- in place.
250-
*/
251-
public static Map<PartitionKeyDesc, Map<MTMVRelatedTableIf, Set<String>>> generateRelatedPartitionDescsByLineage(
252-
MTMVPartitionInfo mvPartitionInfo,
253-
Map<String, String> mvProperties, List<Column> partitionColumns,
254-
Map<List<String>, Set<String>> queryUsedPartitions,
255-
Map<MvccTableInfo, MvccSnapshot> pinnedSnapshots) throws AnalysisException {
256-
return generateRelatedPartitionDescs(mvPartitionInfo, mvProperties, partitionColumns,
257-
queryUsedPartitions, pinnedSnapshots, lineagePartitionDescGenerators);
258-
}
259-
260-
private static Map<PartitionKeyDesc, Map<MTMVRelatedTableIf, Set<String>>> generateRelatedPartitionDescs(
261-
MTMVPartitionInfo mvPartitionInfo,
262-
Map<String, String> mvProperties, List<Column> partitionColumns,
263-
Map<List<String>, Set<String>> queryUsedPartitions,
264-
Map<MvccTableInfo, MvccSnapshot> pinnedSnapshots,
265-
List<MTMVRelatedPartitionDescGeneratorService> generators) throws AnalysisException {
266237
long start = System.currentTimeMillis();
267238
RelatedPartitionDescResult result = new RelatedPartitionDescResult(pinnedSnapshots);
268-
for (MTMVRelatedPartitionDescGeneratorService service : generators) {
239+
for (MTMVRelatedPartitionDescGeneratorService service : partitionDescGenerators) {
269240
service.apply(mvPartitionInfo, mvProperties, result, partitionColumns, queryUsedPartitions);
270241
}
271242
if (LOG.isDebugEnabled()) {

‎fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescSyncLimitGenerator.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -77,7 +77,7 @@ public void apply(MTMVPartitionInfo mvPartitionInfo, Map<String, String> mvPrope
7777
* @param mvProperties
7878
* @return
7979
*/
80-
public MTMVPartitionSyncConfig generateMTMVPartitionSyncConfigByProperties(
80+
public static MTMVPartitionSyncConfig generateMTMVPartitionSyncConfigByProperties(
8181
Map<String, String> mvProperties) {
8282
int syncLimit = StringUtils.isEmpty(mvProperties.get(PropertyAnalyzer.PROPERTIES_PARTITION_SYNC_LIMIT)) ? -1
8383
: Integer.parseInt(mvProperties.get(PropertyAnalyzer.PROPERTIES_PARTITION_SYNC_LIMIT));

‎fe/fe-core/src/test/java/org/apache/doris/mtmv/ivm/IvmBaselineRebuildTest.java‎

Lines changed: 10 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -168,14 +168,14 @@ public void testDropPartitionOutsideMvPartitionsMarksNothing() throws Exception
168168
}
169169

170170
/**
171-
* Invalidation reads the mapping by lineage, not by the partition_sync_limit window. A base partition
172-
* the window no longer covers can still have its rows in an MV partition -- the MV was built while the
173-
* partition was inside the window, and partition sync keeps the MV partition once a widened window
174-
* covers the base partition again -- so a windowed mapping would report no MV partition for the change
175-
* and leave those rows behind, with no binlog to repair them.
171+
* A base partition the partition_sync_limit window no longer covers can still have its rows in an MV
172+
* partition: the MV was built while that partition was inside the window, shrinking the window does not
173+
* touch the MV's own partitions, and widening it again makes partition sync keep the partition holding
174+
* those rows. TRUNCATE emits no binlog, so nothing incremental can repair them -- the whole MV has to be
175+
* rebuilt rather than a partition being guessed at.
176176
*/
177177
@Test
178-
public void testChangedPartitionOutsideTheSyncWindowStillMarksItsMvPartition() throws Exception {
178+
public void testChangedPartitionOutsideTheSyncWindowRebuildsTheWholeMv() throws Exception {
179179
String db = "ivm_baseline_sync_window";
180180
String thisYear = LocalDate.now().withDayOfYear(1).toString();
181181
String nextYear = LocalDate.now().withDayOfYear(1).plusYears(1).toString();
@@ -201,21 +201,15 @@ public void testChangedPartitionOutsideTheSyncWindowStillMarksItsMvPartition() t
201201
+ "AS SELECT dt, k1, v1 FROM ivm_base");
202202
MTMV mtmv = getMtmv(db);
203203
Assertions.assertEquals(3, mtmv.getPartitionNames().size());
204-
Set<String> expected = mvPartitionsWithSameRange(mtmv, getBaseTable(db), "p202001");
205-
Assertions.assertEquals(1, expected.size());
206204

207-
// The window now keeps only this year's partition, so the mapping still describes this base table
208-
// while p202001 drops out of it: the lookup for the changed partition is empty, and only the
209-
// lineage mapping knows which MV partition holds its rows.
205+
// The window now keeps only this year's partition, so p202001 leaves the mapping while the MV's own
206+
// partition for it stays. TRUNCATE leaves the base partition in place, so partition sync would keep
207+
// that MV partition too -- the rows it still holds are exactly what the rebuild has to remove.
210208
executeSql("ALTER MATERIALIZED VIEW ivm_mv SET ('partition_sync_limit' = '1',"
211209
+ " 'partition_sync_time_unit' = 'YEAR')");
212-
213-
// TRUNCATE leaves the base partition in place, so partition sync keeps the MV partition holding its
214-
// rows. A DROP would take that MV partition with it and hide the problem.
215210
executeSql("TRUNCATE TABLE ivm_base PARTITION(p202001)");
216211

217-
Assertions.assertFalse(mtmv.getIvmInfo().requiresCompleteBaselineRebuild());
218-
Assertions.assertEquals(expected, mtmv.getIvmInfo().getPendingBaselineRebuildPartitions());
212+
Assertions.assertTrue(mtmv.getIvmInfo().requiresCompleteBaselineRebuild());
219213
}
220214

221215
/**

‎regression-test/suites/mtmv_p0/ivm/test_ivm_baseline_marker_scope.groovy‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,13 @@ suite("test_ivm_baseline_marker_scope") {
6565
return taskResult[0].TaskId.toString()
6666
}
6767

68+
def taskStatus = { String id ->
69+
def rows = sql_return_maparray("""
70+
SELECT Status FROM tasks('type'='mv') WHERE TaskId = '${id}'
71+
""")
72+
return rows[0].Status.toString()
73+
}
74+
6875
// Unset RefreshMode / IvmFallbackReason come back as the literal two-character string "\N",
6976
// which does not survive the .out round trip, so fold the unset value into a printable token.
7077
def taskQuery = { String taskId ->
@@ -174,6 +181,9 @@ suite("test_ivm_baseline_marker_scope") {
174181
sql """ALTER TABLE ${factTable} DROP PARTITION p202601"""
175182
sql """REFRESH MATERIALIZED VIEW ${mvName} AUTO"""
176183
taskId = waitForNewTask(taskId)
184+
// waitForNewTask returns on any terminal state, and partition sync already dropped the MV partition,
185+
// so the row set below would match even if this refresh failed. The task itself has to be asserted.
186+
assertEquals("SUCCESS", taskStatus(taskId))
177187
order_qt_narrowed_mv """SELECT order_id, dt, amount, dimension_name
178188
FROM ${mvName}"""
179189
}

0 commit comments

Comments
 (0)