Skip to content

Commit 9214167

Browse files
committed
[fix](ivm) Treat a partition_sync_limit window change as a baseline change
A partition_sync_limit window decides which base partitions the MV maintains, so changing it changes which rows the MV owes: a partition that leaves the window keeps its MV partition, the deltas of the range it skipped were never applied, and a later widening makes partition sync keep it. The invalidation path tried to answer that after the fact -- whether the mapping it had just built was filtered by a window -- by reading the window from mutable properties. A window set and cleared while that mapping was built is seen by neither read, so the mapping could be windowed while the guard read no limit, the selection came out empty, and no barrier was written for a change whose rows no row binlog can repair. The window now gets the answer the two properties around it already get: an effective change to it forces a complete baseline rebuild, recorded by the ALTER itself, in the same critical section that applies it, so no ordering of the ALTER against a partition change can leave a windowed mapping trusted. Key changes: - Compare the three window properties across alterMvProperties, as they are stored, so restating the same window forces nothing, and require a complete baseline rebuild when they change, for IVM MVs - Keep the property vocabulary where the rest of it lives: MTMVPropertyUtil gains the key list and the two helpers, next to hasPartitionSyncLimit - Release the barrier the ALTER now records in the three tests that set the window and then test what happens while it is in effect, so their assertions keep holding for the reason they were written for Unit Test: - IvmBaselineRebuildTest.testChangingTheSyncWindowRequiresCompleteBaselineRebuild: setting the window requires a complete rebuild, restating the same window does not, widening it does. Without the rule it fails with expected: <true> but was: <false> - Regression: mtmv_p0/ivm test_ivm_partition_sync_limit and test_ivm_partition_sync_limit_with_window (both set the window at CREATE, so the rule does not reach them) plus the eight marker suites of the previous commit green on a local cluster built from this head; IvmBaselineRebuildTest 37/37, MTMVPropertyUtilTest 13/13, MTMVTest 23/23; checkstyle clean
1 parent 57a1b3d commit 9214167

3 files changed

Lines changed: 83 additions & 0 deletions

File tree

‎fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -384,6 +384,15 @@ public void alterMvProperties(AlterMTMV alterMTMV, boolean isReplay) {
384384
Map<TableNameInfo, Integer> oldWindowLimits = containsPartitionWindowLimit
385385
? MTMVPropertyUtil.getIvmPartitionWindowLimit(this.mvProperties)
386386
: Maps.newHashMap();
387+
// A partition_sync_limit window decides which base partitions the MV maintains, so changing
388+
// it changes which rows the MV owes: a partition that leaves the window keeps its MV
389+
// partition, the deltas of the range it skipped were never applied, and a later widening
390+
// makes partition sync keep it. That is the same trade as the two properties around it and
391+
// gets the same answer, rather than leaving a reader of the window to decide afterwards
392+
// whether the mapping it just built was filtered by a window that is no longer there.
393+
boolean containsSyncWindow = MTMVPropertyUtil.containsPartitionSyncWindow(mvProperties);
394+
Map<String, String> oldSyncWindow = containsSyncWindow
395+
? MTMVPropertyUtil.partitionSyncWindowOf(this.mvProperties) : null;
387396
this.mvProperties.putAll(mvProperties);
388397
// Both excluded_trigger_tables changes and window limit enlargement/removal
389398
// change the refresh baseline semantics: partitions previously skipped become
@@ -428,6 +437,10 @@ public void alterMvProperties(AlterMTMV alterMTMV, boolean isReplay) {
428437
}
429438
}
430439
}
440+
if (containsSyncWindow && ivmInfo != null && ivmInfo.isEnableIvm()
441+
&& !MTMVPropertyUtil.partitionSyncWindowOf(this.mvProperties).equals(oldSyncWindow)) {
442+
requireCompleteBaselineRebuild = true;
443+
}
431444
if (invalidateRefreshSnapshot || requireCompleteBaselineRebuild) {
432445
this.schemaChangeVersion++;
433446
this.refreshSnapshot = new MTMVRefreshSnapshot();

‎fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPropertyUtil.java‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,12 @@
4545
import java.util.Set;
4646

4747
public class MTMVPropertyUtil {
48+
/** The properties a partition_sync_limit window is built from; see MTMV#alterMvProperties. */
49+
private static final List<String> PARTITION_SYNC_WINDOW_KEYS = Lists.newArrayList(
50+
PropertyAnalyzer.PROPERTIES_PARTITION_SYNC_LIMIT,
51+
PropertyAnalyzer.PROPERTIES_PARTITION_TIME_UNIT,
52+
PropertyAnalyzer.PROPERTIES_PARTITION_DATE_FORMAT);
53+
4854
public static final Set<String> MV_PROPERTY_KEYS = Sets.newHashSet(
4955
PropertyAnalyzer.PROPERTIES_GRACE_PERIOD,
5056
PropertyAnalyzer.PROPERTIES_EXCLUDED_TRIGGER_TABLES,
@@ -231,6 +237,29 @@ public static boolean hasPartitionSyncLimit(Map<String, String> mvProperties) {
231237
return !StringUtils.isEmpty(value) && Integer.parseInt(value) > 0;
232238
}
233239

240+
/** Whether the given (altered) properties touch the partition_sync_limit window. */
241+
public static boolean containsPartitionSyncWindow(Map<String, String> properties) {
242+
for (String property : PARTITION_SYNC_WINDOW_KEYS) {
243+
if (properties.containsKey(property)) {
244+
return true;
245+
}
246+
}
247+
return false;
248+
}
249+
250+
/**
251+
* The window the given properties describe, for comparing it across an ALTER. Only the properties the
252+
* window is built from are read, and the values are compared as they are stored: setting the same
253+
* window again changes nothing about which rows the MV owes and must not force a rebuild.
254+
*/
255+
public static Map<String, String> partitionSyncWindowOf(Map<String, String> properties) {
256+
Map<String, String> res = Maps.newHashMap();
257+
for (String property : PARTITION_SYNC_WINDOW_KEYS) {
258+
res.put(property, properties.get(property));
259+
}
260+
return res;
261+
}
262+
234263
/**
235264
* Look up the window limit configured for a base table, mirroring the
236265
* excluded_trigger_tables name-matching semantics (empty db/ctl wildcard).

‎fe/fe-core/src/test/java/org/apache/doris/mtmv/ivm/IvmBaselineRebuildTest.java‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -216,6 +216,10 @@ public void testChangedPartitionOutsideTheSyncWindowRebuildsTheWholeMv() throws
216216
// that MV partition too -- the rows it still holds are exactly what the rebuild has to remove.
217217
executeSql("ALTER MATERIALIZED VIEW ivm_mv SET ('partition_sync_limit' = '1',"
218218
+ " 'partition_sync_time_unit' = 'YEAR')");
219+
// Setting the window is itself a change of which rows the MV owes, so it records a complete
220+
// baseline rebuild (see testChangingTheSyncWindowRequiresCompleteBaselineRebuild). Release it:
221+
// this test is about a change that happens while the window is in effect.
222+
clearBaselineRebuild(mtmv);
219223
executeSql("TRUNCATE TABLE ivm_base PARTITION(p202001)");
220224

221225
Assertions.assertTrue(mtmv.getIvmInfo().requiresCompleteBaselineRebuild());
@@ -263,13 +267,46 @@ public void testChangeThatMixesInWindowAndOutOfWindowPartitionsRebuildsTheWholeM
263267

264268
executeSql("ALTER MATERIALIZED VIEW ivm_mv SET ('partition_sync_limit' = '1',"
265269
+ " 'partition_sync_time_unit' = 'YEAR')");
270+
// Setting the window is itself a change of which rows the MV owes, so it records a complete
271+
// baseline rebuild (see testChangingTheSyncWindowRequiresCompleteBaselineRebuild). Release it:
272+
// this test is about a change that happens while the window is in effect.
273+
clearBaselineRebuild(mtmv);
266274
// One statement, so the marker sees both partitions together: pThisYear is inside the window while
267275
// p202001 is not, which is exactly the mix a non-empty selection must not be allowed to hide.
268276
executeSql("TRUNCATE TABLE ivm_base PARTITION(p202001, pThisYear)");
269277

270278
Assertions.assertTrue(mtmv.getIvmInfo().requiresCompleteBaselineRebuild());
271279
}
272280

281+
/**
282+
* A window property decides which base partitions the MV maintains, so changing it changes which
283+
* rows the MV owes: the partitions that leave the window keep their MV partition, the deltas of the
284+
* range they skipped were never applied, and widening the window again makes partition sync keep
285+
* them. That is the trade excluded_trigger_tables and ivm_partition_window_limit already make, and it
286+
* gets the same answer -- a complete baseline rebuild -- rather than leaving the mapping's reader to
287+
* decide afterwards whether the window it just applied is still there.
288+
*/
289+
@Test
290+
public void testChangingTheSyncWindowRequiresCompleteBaselineRebuild() throws Exception {
291+
String db = "ivm_sync_window_property_change";
292+
createPartitionedIvmTableAndPartitionedMv(db);
293+
MTMV mtmv = getMtmv(db);
294+
Assertions.assertFalse(mtmv.getIvmInfo().isBaselineRebuildRequired());
295+
296+
executeSql("ALTER MATERIALIZED VIEW ivm_mv SET ('partition_sync_limit' = '1',"
297+
+ " 'partition_sync_time_unit' = 'YEAR')");
298+
Assertions.assertTrue(mtmv.getIvmInfo().requiresCompleteBaselineRebuild());
299+
clearBaselineRebuild(mtmv);
300+
301+
// The same window, restated: it decides the same rows, so it forces nothing.
302+
executeSql("ALTER MATERIALIZED VIEW ivm_mv SET ('partition_sync_limit' = '1')");
303+
Assertions.assertFalse(mtmv.getIvmInfo().isBaselineRebuildRequired());
304+
305+
// A wider window brings back partitions whose deltas were skipped.
306+
executeSql("ALTER MATERIALIZED VIEW ivm_mv SET ('partition_sync_limit' = '10')");
307+
Assertions.assertTrue(mtmv.getIvmInfo().requiresCompleteBaselineRebuild());
308+
}
309+
273310
/**
274311
* The limit is read on both sides of the mapping the selection judges: the mapping is built under
275312
* whatever window the properties hold at that moment, and MV properties are mutable in between --
@@ -290,6 +327,10 @@ public void testTheSyncLimitIsReadOnBothSidesOfTheMapping() throws Exception {
290327
OlapTable baseTable = getBaseTable(db);
291328
executeSql("ALTER MATERIALIZED VIEW ivm_mv SET ('partition_sync_limit' = '1',"
292329
+ " 'partition_sync_time_unit' = 'YEAR')");
330+
// Setting the window is itself a change of which rows the MV owes, so it records a complete
331+
// baseline rebuild (see testChangingTheSyncWindowRequiresCompleteBaselineRebuild). Release it:
332+
// this test is about a change that happens while the window is in effect.
333+
clearBaselineRebuild(mtmv);
293334

294335
try (MockedStatic<MTMVPartitionUtil> partitionUtil = Mockito.mockStatic(MTMVPartitionUtil.class,
295336
Mockito.CALLS_REAL_METHODS)) {

0 commit comments

Comments
 (0)