Skip to content

Commit 7ad093e

Browse files
committed
[fix](ivm) Invalidate MV partitions by lineage, not by the sync window
Invalidation read the partition mapping through the partition_sync_limit window. A base partition the window no longer covers can still have its rows in an MV partition: the MV was built while that partition was inside the window, shrinking the window does not touch the MV's own partitions, and widening it again makes partition sync keep the MV partition that holds those rows. A change to such a partition was then read as "no MV partition reads this" -- and TRUNCATE, like the other silent changes, emits no binlog -- so no barrier was recorded and the rows stayed in the MV for good. Key changes: - Add a lineage variant of the partition mapping, the generator chain without the sync-limit step, and read it when invalidating, so a changed partition is matched to the MV partitions that hold its rows regardless of the window - Read an empty result as "nothing to rebuild" only when the MV has partitions at all: an MV without one cannot hold the changed rows, and treating it as unknown recorded a COMPLETE barrier that made a strict INCREMENTAL fail for nothing - Report from invalidateIvmBaseline whether a barrier was recorded, and log the no-op case as such instead of claiming the baseline was invalidated Unit Test: - IvmBaselineRebuildTest: a base partition outside the shrunk window still marks the MV partition holding its rows (the test fails without the change), and the contention case now uses a bounded worker that is asserted to end - MetaLockUtilsTest: tryReadLockTables releases the locks it took when a later table is busy
1 parent 8e563c7 commit 7ad093e

5 files changed

Lines changed: 195 additions & 12 deletions

File tree

‎fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java‎

Lines changed: 43 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -626,7 +626,11 @@ public void invalidateIvmBaseline() {
626626
* Mark the MV partitions that may hold rows read from the changed base table partitions as needing a
627627
* rebuild. When those partitions cannot be determined, the whole MV is marked instead.
628628
*/
629-
public void invalidateIvmBaseline(BaseTableInfo baseTableInfo, Map<String, Long> changedPartitions) {
629+
/**
630+
* @return whether a barrier was recorded. The caller reports the two outcomes differently: a change
631+
* that no MV partition reads leaves nothing to rebuild and must not be logged as one.
632+
*/
633+
public boolean invalidateIvmBaseline(BaseTableInfo baseTableInfo, Map<String, Long> changedPartitions) {
630634
// Computed before the MV lock is taken, not inside it: the mapping reads the partition items of the
631635
// MV and of every PCT table, so it takes those tables' locks, and the MV lock has to stay a leaf
632636
// (nothing may be acquired under it) the way the rest of this class assumes. The selection does not
@@ -641,7 +645,7 @@ public void invalidateIvmBaseline(BaseTableInfo baseTableInfo, Map<String, Long>
641645
// keeps it from discarding the result of a task that is already running.
642646
LOG.debug("No MV partition is affected by changed base partitions, mv={}, baseTable={}, "
643647
+ "changedPartitions={}", name, baseTableInfo, changedPartitions);
644-
return;
648+
return false;
645649
}
646650
EditLogItem editLogItem;
647651
writeMvLock();
@@ -662,6 +666,7 @@ public void invalidateIvmBaseline(BaseTableInfo baseTableInfo, Map<String, Long>
662666
writeMvUnlock();
663667
}
664668
editLogItem.await();
669+
return true;
665670
}
666671

667672
/**
@@ -744,8 +749,12 @@ private Optional<Set<String>> selectAffectedMvPartitions(BaseTableInfo baseTable
744749
if (!pctTable.getPartitionNames().containsAll(changedBasePartitions.keySet())) {
745750
return Optional.empty();
746751
}
752+
// By lineage, not by the partition_sync_limit window: the window can move after a base
753+
// partition was read, and a partition it does not cover right now can still have its rows in
754+
// an MV partition. Partition sync keeps that MV partition once the window covers the base
755+
// partition again, and the change itself left no binlog to repair those rows with.
747756
Map<String, Map<MTMVRelatedTableIf, Set<String>>> partitionMappings =
748-
calculatePartitionMappings(Maps.newHashMap());
757+
calculatePartitionMappingsByLineage();
749758
Set<String> res = Sets.newHashSet();
750759
boolean pctTableMapped = false;
751760
for (Entry<String, Map<MTMVRelatedTableIf, Set<String>>> mapping : partitionMappings.entrySet()) {
@@ -760,8 +769,15 @@ private Optional<Set<String>> selectAffectedMvPartitions(BaseTableInfo baseTable
760769
}
761770
}
762771
// The mapping does not describe this base table at all. That contradicts the PCT check above,
763-
// so it is safer to rebuild everything than to trust a selection that never saw the table.
772+
// so it is safer to rebuild everything than to trust a selection that never saw the table --
773+
// unless the MV has no partition of its own yet, which is the one shape where the missing
774+
// entries are not a surprise: an MV without partitions holds no rows.
764775
if (!pctTableMapped) {
776+
if (getPartitionNames().isEmpty()) {
777+
LOG.info("MV has no partition yet, nothing can hold the changed base partitions. "
778+
+ "baseTable={}, mv={}", baseTableInfo, name);
779+
return Optional.of(Sets.newHashSet());
780+
}
765781
LOG.warn("Base table is not described by the partition mapping, rebuild the whole MV. "
766782
+ "baseTable={}, mv={}", baseTableInfo, name);
767783
return Optional.empty();
@@ -985,6 +1001,24 @@ public Map<String, Map<MTMVRelatedTableIf, Set<String>>> calculatePartitionMappi
9851001
public Map<String, Map<MTMVRelatedTableIf, Set<String>>> calculatePartitionMappings(
9861002
Map<List<String>, Set<String>> queryUsedBaseTablePartitionMap,
9871003
Map<MvccTableInfo, MvccSnapshot> pinnedSnapshots) throws AnalysisException {
1004+
return calculatePartitionMappings(queryUsedBaseTablePartitionMap, pinnedSnapshots, false);
1005+
}
1006+
1007+
/**
1008+
* The MV partitions each base partition belongs to by lineage, ignoring the partition_sync_limit
1009+
* window. Invalidating MV partitions has to use this: the window can move after a base partition was
1010+
* read, and a changed partition that the window does not cover right now can still have its rows in an
1011+
* MV partition -- partition sync keeps that partition once the window covers the base partition again,
1012+
* and the change itself left no binlog to repair it with.
1013+
*/
1014+
public Map<String, Map<MTMVRelatedTableIf, Set<String>>> calculatePartitionMappingsByLineage()
1015+
throws AnalysisException {
1016+
return calculatePartitionMappings(Maps.newHashMap(), null, true);
1017+
}
1018+
1019+
private Map<String, Map<MTMVRelatedTableIf, Set<String>>> calculatePartitionMappings(
1020+
Map<List<String>, Set<String>> queryUsedBaseTablePartitionMap,
1021+
Map<MvccTableInfo, MvccSnapshot> pinnedSnapshots, boolean byLineage) throws AnalysisException {
9881022
if (mvPartitionInfo.getPartitionType() == MTMVPartitionType.SELF_MANAGE) {
9891023
return Maps.newHashMap();
9901024
}
@@ -1000,9 +1034,11 @@ public Map<String, Map<MTMVRelatedTableIf, Set<String>>> calculatePartitionMappi
10001034
= getEffectiveQueryUsedBaseTablePartitionMap(
10011035
queryUsedBaseTablePartitionMap, mvPartitionItems, pinnedSnapshots);
10021036
Map<String, Map<MTMVRelatedTableIf, Set<String>>> res = Maps.newHashMap();
1003-
Map<PartitionKeyDesc, Map<MTMVRelatedTableIf, Set<String>>> pctPartitionDescs = MTMVPartitionUtil
1004-
.generateRelatedPartitionDescs(mvPartitionInfo, mvProperties, getPartitionColumns(),
1005-
effectiveFilter, pinnedSnapshots);
1037+
Map<PartitionKeyDesc, Map<MTMVRelatedTableIf, Set<String>>> pctPartitionDescs = byLineage
1038+
? MTMVPartitionUtil.generateRelatedPartitionDescsByLineage(mvPartitionInfo, mvProperties,
1039+
getPartitionColumns(), effectiveFilter, pinnedSnapshots)
1040+
: MTMVPartitionUtil.generateRelatedPartitionDescs(mvPartitionInfo, mvProperties,
1041+
getPartitionColumns(), effectiveFilter, pinnedSnapshots);
10061042
for (Entry<String, PartitionItem> entry : mvPartitionItems.entrySet()) {
10071043
res.put(entry.getKey(),
10081044
pctPartitionDescs.getOrDefault(entry.getValue().toPartitionKeyDesc(), Maps.newHashMap()));

‎fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionUtil.java‎

Lines changed: 39 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -91,6 +91,14 @@ public class MTMVPartitionUtil {
9191
new MTMVRelatedPartitionDescTransferGenerator()
9292
);
9393

94+
// The same chain without the partition_sync_limit step, for callers that ask what a base partition
95+
// belongs to by lineage rather than what the window still covers. Derived from the chain above so
96+
// that a generator added there cannot be forgotten here.
97+
private static final List<MTMVRelatedPartitionDescGeneratorService> lineagePartitionDescGenerators =
98+
partitionDescGenerators.stream()
99+
.filter(generator -> !(generator instanceof MTMVRelatedPartitionDescSyncLimitGenerator))
100+
.collect(ImmutableList.toImmutableList());
101+
94102
/**
95103
* Determine whether the partition is sync with retated partition and other baseTables
96104
*
@@ -225,9 +233,39 @@ public static Map<PartitionKeyDesc, Map<MTMVRelatedTableIf, Set<String>>> genera
225233
Map<String, String> mvProperties, List<Column> partitionColumns,
226234
Map<List<String>, Set<String>> queryUsedPartitions,
227235
Map<MvccTableInfo, MvccSnapshot> pinnedSnapshots) throws AnalysisException {
236+
return generateRelatedPartitionDescs(mvPartitionInfo, mvProperties, partitionColumns,
237+
queryUsedPartitions, pinnedSnapshots, partitionDescGenerators);
238+
}
239+
240+
/**
241+
* The same partitions, mapped by lineage: a base partition is related to an MV partition when the MV's
242+
* partition definition puts it there, whether or not the partition_sync_limit window covers it now.
243+
*
244+
* <p>Invalidating MV partitions needs this rather than the windowed mapping, because the window can
245+
* move after a base partition was read. A partition the window no longer covers, or did not cover when
246+
* the change happened, can still have its rows in an MV partition: the MV partition is only dropped
247+
* when partition sync regenerates the MV's partition set, and a sync limit that is later widened puts
248+
* the base partition back into that set, leaving the MV partition -- and the rows of the change that
249+
* no binlog recorded -- in place.
250+
*/
251+
public static Map<PartitionKeyDesc, Map<MTMVRelatedTableIf, Set<String>>> generateRelatedPartitionDescsByLineage(
252+
MTMVPartitionInfo mvPartitionInfo,
253+
Map<String, String> mvProperties, List<Column> partitionColumns,
254+
Map<List<String>, Set<String>> queryUsedPartitions,
255+
Map<MvccTableInfo, MvccSnapshot> pinnedSnapshots) throws AnalysisException {
256+
return generateRelatedPartitionDescs(mvPartitionInfo, mvProperties, partitionColumns,
257+
queryUsedPartitions, pinnedSnapshots, lineagePartitionDescGenerators);
258+
}
259+
260+
private static Map<PartitionKeyDesc, Map<MTMVRelatedTableIf, Set<String>>> generateRelatedPartitionDescs(
261+
MTMVPartitionInfo mvPartitionInfo,
262+
Map<String, String> mvProperties, List<Column> partitionColumns,
263+
Map<List<String>, Set<String>> queryUsedPartitions,
264+
Map<MvccTableInfo, MvccSnapshot> pinnedSnapshots,
265+
List<MTMVRelatedPartitionDescGeneratorService> generators) throws AnalysisException {
228266
long start = System.currentTimeMillis();
229267
RelatedPartitionDescResult result = new RelatedPartitionDescResult(pinnedSnapshots);
230-
for (MTMVRelatedPartitionDescGeneratorService service : partitionDescGenerators) {
268+
for (MTMVRelatedPartitionDescGeneratorService service : generators) {
231269
service.apply(mvPartitionInfo, mvProperties, result, partitionColumns, queryUsedPartitions);
232270
}
233271
if (LOG.isDebugEnabled()) {

‎fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelationManager.java‎

Lines changed: 12 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -115,13 +115,22 @@ private void markIvmBaselineRebuild(BaseTableInfo baseTableInfo, boolean allPart
115115
if (MTMVPartitionUtil.isTableExcluded(mtmv.getExcludedTriggerTables(), baseTableName)) {
116116
continue;
117117
}
118+
boolean invalidated;
118119
if (allPartitionsChanged) {
119120
mtmv.invalidateIvmBaseline();
121+
invalidated = true;
120122
} else {
121-
mtmv.invalidateIvmBaseline(baseTableInfo, changedPartitions);
123+
invalidated = mtmv.invalidateIvmBaseline(baseTableInfo, changedPartitions);
124+
}
125+
// A partition change that no MV partition reads leaves nothing to rebuild, and saying that it
126+
// invalidated the baseline would claim a persisted barrier that does not exist.
127+
if (invalidated) {
128+
LOG.info("Invalidated IVM baseline, baseTable={}, mtmv={}, reason={}",
129+
baseTableInfo, mtmvInfo, reason);
130+
} else {
131+
LOG.info("No MV partition reads the changed base partitions, nothing to invalidate. "
132+
+ "baseTable={}, mtmv={}, reason={}", baseTableInfo, mtmvInfo, reason);
122133
}
123-
LOG.info("Invalidated IVM baseline, baseTable={}, mtmv={}, reason={}",
124-
baseTableInfo, mtmvInfo, reason);
125134
}
126135
}
127136

‎fe/fe-core/src/test/java/org/apache/doris/common/util/MetaLockUtilsTest.java‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@
2828
import org.junit.jupiter.api.Test;
2929

3030
import java.util.List;
31+
import java.util.concurrent.CountDownLatch;
3132
import java.util.concurrent.TimeUnit;
3233

3334
public class MetaLockUtilsTest {
@@ -60,6 +61,47 @@ public void testReadLockTables() {
6061
tableList.get(1).writeUnlock();
6162
}
6263

64+
/**
65+
* The read batch gives back what it took when a later table stays busy, so the caller holds nothing
66+
* while it decides what to do. Probed the same way as the tests above: a read lock still held would
67+
* block the write lock of the table that was acquired first.
68+
*/
69+
@Test
70+
public void testTryReadLockTablesReleasesWhatItTookWhenALaterTableIsBusy() throws Exception {
71+
List<Table> tables = ImmutableList.of(TableTest.newOlapTable(0, "readable", 0),
72+
TableTest.newOlapTable(1, "busy", 0));
73+
Table busy = tables.get(1);
74+
// Another thread has to hold the write lock: the same thread could take a read lock underneath
75+
// its own write lock (that direction is allowed), which is not the case under test. Both
76+
// acquisitions are bounded and the holder is asserted to end, so a failure cannot strand it.
77+
CountDownLatch locked = new CountDownLatch(1);
78+
CountDownLatch released = new CountDownLatch(1);
79+
Thread holder = new Thread(() -> {
80+
Assertions.assertTrue(busy.tryWriteLock(30, TimeUnit.SECONDS));
81+
locked.countDown();
82+
try {
83+
released.await(30, TimeUnit.SECONDS);
84+
} catch (InterruptedException e) {
85+
Thread.currentThread().interrupt();
86+
} finally {
87+
busy.writeUnlock();
88+
}
89+
});
90+
holder.start();
91+
try {
92+
Assertions.assertTrue(locked.await(30, TimeUnit.SECONDS));
93+
Assertions.assertFalse(MetaLockUtils.tryReadLockTables(tables, 100, TimeUnit.MILLISECONDS),
94+
"a busy table must fail the batch");
95+
Assertions.assertTrue(tables.get(0).tryWriteLock(100, TimeUnit.MILLISECONDS),
96+
"the read lock taken for the first table must have been released");
97+
tables.get(0).writeUnlock();
98+
} finally {
99+
released.countDown();
100+
holder.join(TimeUnit.SECONDS.toMillis(30));
101+
}
102+
Assertions.assertFalse(holder.isAlive(), "the holder should have released the table");
103+
}
104+
63105
@Test
64106
public void testWriteLockTables() throws MetaNotFoundException {
65107
MetaLockUtils.writeLockTables(tableList);

‎fe/fe-core/src/test/java/org/apache/doris/mtmv/ivm/IvmBaselineRebuildTest.java‎

Lines changed: 59 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,7 @@
4848
import org.junit.jupiter.api.Assertions;
4949
import org.junit.jupiter.api.Test;
5050

51+
import java.time.LocalDate;
5152
import java.util.Collections;
5253
import java.util.Map.Entry;
5354
import java.util.Set;
@@ -166,6 +167,57 @@ public void testDropPartitionOutsideMvPartitionsMarksNothing() throws Exception
166167
Assertions.assertFalse(mtmv.getIvmInfo().isBaselineRebuildRequired());
167168
}
168169

170+
/**
171+
* Invalidation reads the mapping by lineage, not by the partition_sync_limit window. A base partition
172+
* the window no longer covers can still have its rows in an MV partition -- the MV was built while the
173+
* partition was inside the window, and partition sync keeps the MV partition once a widened window
174+
* covers the base partition again -- so a windowed mapping would report no MV partition for the change
175+
* and leave those rows behind, with no binlog to repair them.
176+
*/
177+
@Test
178+
public void testChangedPartitionOutsideTheSyncWindowStillMarksItsMvPartition() throws Exception {
179+
String db = "ivm_baseline_sync_window";
180+
String thisYear = LocalDate.now().withDayOfYear(1).toString();
181+
String nextYear = LocalDate.now().withDayOfYear(1).plusYears(1).toString();
182+
createDatabaseAndUse(db);
183+
createTable("CREATE TABLE " + db + ".ivm_base (\n"
184+
+ " dt date NOT NULL,\n"
185+
+ " k1 int,\n"
186+
+ " v1 int\n"
187+
+ ")\n"
188+
+ "DUPLICATE KEY(dt, k1)\n"
189+
+ "PARTITION BY RANGE(dt) (\n"
190+
+ " PARTITION p202001 VALUES [('2020-01-01'), ('2020-02-01')),\n"
191+
+ " PARTITION p202002 VALUES [('2020-02-01'), ('2020-03-01')),\n"
192+
+ " PARTITION pThisYear VALUES [('" + thisYear + "'), ('" + nextYear + "'))\n"
193+
+ ")\n"
194+
+ "DISTRIBUTED BY HASH(k1) BUCKETS 1\n"
195+
+ "PROPERTIES ('replication_num' = '1', 'binlog.enable' = 'true', 'binlog.format' = 'ROW')");
196+
createMvByNereids("CREATE MATERIALIZED VIEW ivm_mv\n"
197+
+ "BUILD DEFERRED REFRESH INCREMENTAL ON MANUAL\n"
198+
+ "PARTITION BY(dt)\n"
199+
+ "DISTRIBUTED BY RANDOM BUCKETS 1\n"
200+
+ "PROPERTIES ('replication_num' = '1')\n"
201+
+ "AS SELECT dt, k1, v1 FROM ivm_base");
202+
MTMV mtmv = getMtmv(db);
203+
Assertions.assertEquals(3, mtmv.getPartitionNames().size());
204+
Set<String> expected = mvPartitionsWithSameRange(mtmv, getBaseTable(db), "p202001");
205+
Assertions.assertEquals(1, expected.size());
206+
207+
// The window now keeps only this year's partition, so the mapping still describes this base table
208+
// while p202001 drops out of it: the lookup for the changed partition is empty, and only the
209+
// lineage mapping knows which MV partition holds its rows.
210+
executeSql("ALTER MATERIALIZED VIEW ivm_mv SET ('partition_sync_limit' = '1',"
211+
+ " 'partition_sync_time_unit' = 'YEAR')");
212+
213+
// TRUNCATE leaves the base partition in place, so partition sync keeps the MV partition holding its
214+
// rows. A DROP would take that MV partition with it and hide the problem.
215+
executeSql("TRUNCATE TABLE ivm_base PARTITION(p202001)");
216+
217+
Assertions.assertFalse(mtmv.getIvmInfo().requiresCompleteBaselineRebuild());
218+
Assertions.assertEquals(expected, mtmv.getIvmInfo().getPendingBaselineRebuildPartitions());
219+
}
220+
169221
/**
170222
* The partition mapping is built from the MV's PCT tables only. A changed partition of a joined table
171223
* the MV's partition column does not reach is invisible to it, and missing such a change leaves rows
@@ -240,8 +292,11 @@ public void testMultiPctTableBusyOtherTableRebuildsWholeMv() throws Exception {
240292
OlapTable otherPctTable = (OlapTable) getDb(db).getTableOrMetaException("ivm_dim");
241293
CountDownLatch locked = new CountDownLatch(1);
242294
CountDownLatch released = new CountDownLatch(1);
295+
// The acquisitions are bounded on both sides so that a failure cannot leave this worker holding
296+
// the table forever: it is not a daemon, and the test asserts that it ends.
243297
Thread writer = new Thread(() -> {
244-
otherPctTable.writeLock();
298+
Assertions.assertTrue(otherPctTable.tryWriteLock(30, TimeUnit.SECONDS),
299+
"the test worker should be able to take the other PCT table");
245300
locked.countDown();
246301
try {
247302
released.await(30, TimeUnit.SECONDS);
@@ -259,7 +314,10 @@ public void testMultiPctTableBusyOtherTableRebuildsWholeMv() throws Exception {
259314
released.countDown();
260315
}
261316
writer.join(TimeUnit.SECONDS.toMillis(30));
317+
Assertions.assertFalse(writer.isAlive(), "the test worker should have released the table");
262318

319+
// The batch fails on its first table here, so what this covers is the whole-MV fallback; the
320+
// release of the locks taken before the busy one is covered in MetaLockUtilsTest.
263321
Assertions.assertTrue(mtmv.getIvmInfo().requiresCompleteBaselineRebuild());
264322
}
265323

0 commit comments

Comments
 (0)