Skip to content

Commit 9adfd2d

Browse files
authored
[fix](mv) Compensate complete invalid roll-up partition buckets (apache#67882)
## Problem During partition-union rewrite, a roll-up materialized-view partition can cover multiple base-table partitions. When that MV partition is invalid and removed as a whole, compensating only the base partitions that overlap the raw query partition set can leave the replacement scope smaller than the removed MV bucket and produce incorrect aggregate results. ## Root cause The compensation calculation used the intersection between an invalid MV partition's complete base-partition mapping and the raw query partitions as both the relevance test and the compensation set. This lost the atomic relationship between one MV partition and every base partition mapped to it. ## Reproduction Use one valid MV partition mapped to `p3`, one invalid roll-up partition mapped to `{p1, p2}`, and another invalid partition mapped to `{p4, p5}`. For a raw query partition set `{p1, p3}`, the old calculation compensated only `p1`, even though the complete intersecting roll-up bucket was removed. The disjoint `{p4, p5}` bucket must still remain outside compensation. ## Fix Use the intersection only to decide whether an invalid MV partition is relevant to the query. Once relevant, add its complete base-partition mapping to compensation. Invalid MV partitions with no intersection remain excluded, preserving the existing query-range guard. The existing multi-PCT-table merge behavior is unchanged. ## Tests - `./run-fe-ut.sh --run org.apache.doris.nereids.rules.exploration.mv.PartitionCompensatorTest` (13 tests passed) - `DISABLE_BUILD_UI=ON ./build.sh --fe` (passed; the host's Node.js 16 cannot build the current UI, so the supported no-UI FE build path was used) - `./run-regression-test.sh --run -s mtmv_range_date_part_up_rewrite ...` (1 suite passed)
1 parent 62b26c2 commit 9adfd2d

2 files changed

Lines changed: 83 additions & 2 deletions

File tree

‎fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PartitionCompensator.java‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -200,8 +200,11 @@ private static Pair<Pair<BaseTableInfo, Set<String>>, Pair<BaseColInfo, Set<Stri
200200
// Base table partition maybe deleted, need not union
201201
continue;
202202
}
203-
Sets.intersection(baseTablePartitions, queryUsedBaseTablePartitionNameSet)
204-
.copyInto(baseTableNeedUnionPartitionNameSet);
203+
if (!Sets.intersection(baseTablePartitions, queryUsedBaseTablePartitionNameSet).isEmpty()) {
204+
// An MV partition is the atomic unit removed from the rewritten plan. If any base
205+
// partition in its roll-up bucket is used by the query, compensate the whole bucket.
206+
baseTableNeedUnionPartitionNameSet.addAll(baseTablePartitions);
207+
}
205208
}
206209
// If related base table creates partitions or mv is created with ttl, need base table union
207210
Sets.difference(queryUsedBaseTablePartitionNameSet, mvValidBaseTablePartitionNameSet)

‎fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/PartitionCompensatorTest.java‎

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -487,6 +487,84 @@ public void testCalcInvalidPartitionsNoConcurrentModificationWithTwoRelatedTable
487487
.forEach(v -> Assertions.assertEquals(expectedUnion, v));
488488
}
489489

490+
@SuppressWarnings("unchecked")
491+
@Test
492+
public void testCalcInvalidPartitionsCompensatesWholeIntersectingRollupBucket()
493+
throws Exception {
494+
DatabaseIf<?> baseDb = mockDatabase("cat", 1L, "db", 2L);
495+
MTMVRelatedTableIf relatedTable = mockRelatedTableIf(
496+
"base_t", 10L, ImmutableList.of("cat", "db", "base_t"), baseDb);
497+
BaseColInfo colInfo = new BaseColInfo("dt", new BaseTableInfo(relatedTable));
498+
499+
DatabaseIf<?> mvDb = mockDatabase("internal", 3L, "mv_db", 4L);
500+
MTMV mtmv = Mockito.mock(MTMV.class);
501+
Mockito.when(mtmv.getName()).thenReturn("mv1");
502+
Mockito.when(mtmv.getId()).thenReturn(100L);
503+
Mockito.when(mtmv.getDatabase()).thenReturn(mvDb);
504+
Mockito.when(mtmv.selectNonEmptyPartitionIds(ArgumentMatchers.any(), ArgumentMatchers.any()))
505+
.thenReturn(ImmutableList.of(1L));
506+
507+
long validMvPartitionId = 101L;
508+
long partiallyStaleMvPartitionId = 102L;
509+
long disjointMvPartitionId = 103L;
510+
Partition validMvPartition = mockPartition(validMvPartitionId, "mv_valid");
511+
Partition partiallyStaleMvPartition = mockPartition(partiallyStaleMvPartitionId, "mv_partially_stale");
512+
Partition disjointMvPartition = mockPartition(disjointMvPartitionId, "mv_disjoint");
513+
Mockito.when(mtmv.getPartition(validMvPartitionId)).thenReturn(validMvPartition);
514+
Mockito.when(mtmv.getPartition(partiallyStaleMvPartitionId)).thenReturn(partiallyStaleMvPartition);
515+
Mockito.when(mtmv.getPartition(disjointMvPartitionId)).thenReturn(disjointMvPartition);
516+
517+
PartitionInfo mvPartitionInfo = Mockito.mock(PartitionInfo.class);
518+
Mockito.when(mtmv.getPartitionInfo()).thenReturn(mvPartitionInfo);
519+
Mockito.when(mvPartitionInfo.getType()).thenReturn(PartitionType.RANGE);
520+
MTMVPartitionInfo mvPctInfo = Mockito.mock(MTMVPartitionInfo.class);
521+
Mockito.when(mtmv.getMvPartitionInfo()).thenReturn(mvPctInfo);
522+
Mockito.when(mvPctInfo.getPctTables()).thenReturn(ImmutableSet.of(relatedTable));
523+
Mockito.when(mvPctInfo.getPctInfos()).thenReturn(ImmutableList.of(colInfo));
524+
525+
Map<String, Set<String>> relatedPartitionMapping = new HashMap<>();
526+
relatedPartitionMapping.put("mv_valid", ImmutableSet.of("p3"));
527+
relatedPartitionMapping.put("mv_partially_stale", ImmutableSet.of("p1", "p2"));
528+
relatedPartitionMapping.put("mv_disjoint", ImmutableSet.of("p4", "p5"));
529+
Map<MTMVRelatedTableIf, Map<String, Set<String>>> partitionMappings = new HashMap<>();
530+
partitionMappings.put(relatedTable, relatedPartitionMapping);
531+
532+
AsyncMaterializationContext matCtx = Mockito.mock(AsyncMaterializationContext.class);
533+
Mockito.when(matCtx.getMtmv()).thenReturn(mtmv);
534+
Mockito.when(matCtx.calculatePartitionMappings(ArgumentMatchers.any())).thenReturn(partitionMappings);
535+
536+
Map<BaseTableInfo, Collection<Partition>> canRewriteMap = new HashMap<>();
537+
canRewriteMap.put(new BaseTableInfo(mtmv), ImmutableList.of(validMvPartition));
538+
StatementContext stmtCtx = Mockito.mock(StatementContext.class);
539+
Mockito.when(stmtCtx.getMvCanRewritePartitionsMap()).thenReturn(canRewriteMap);
540+
CascadesContext cascadesCtx = Mockito.mock(CascadesContext.class);
541+
Mockito.when(cascadesCtx.getStatementContext()).thenReturn(stmtCtx);
542+
543+
LogicalOlapScan selectedMvScan = Mockito.mock(LogicalOlapScan.class);
544+
Mockito.when(selectedMvScan.getTable()).thenReturn(mtmv);
545+
Mockito.when(selectedMvScan.getSelectedPartitionIds())
546+
.thenReturn(ImmutableList.of(
547+
validMvPartitionId, partiallyStaleMvPartitionId, disjointMvPartitionId));
548+
Plan rewrittenPlan = Mockito.mock(Plan.class);
549+
Mockito.when(rewrittenPlan.collectToList(ArgumentMatchers.any()))
550+
.thenReturn(ImmutableList.of(selectedMvScan));
551+
552+
// The query touches p1 in the stale roll-up bucket and p3 in a valid bucket. Once the stale
553+
// MV partition is removed, its complete {p1, p2} mapping must be compensated atomically.
554+
// The completely disjoint {p4, p5} bucket is removed from the MV scan but is not compensated.
555+
Map<List<String>, Set<String>> queryUsedPartitions = new HashMap<>();
556+
queryUsedPartitions.put(relatedTable.getFullQualifiers(), ImmutableSet.of("p1", "p3"));
557+
558+
Pair<Map<BaseTableInfo, Set<String>>, Map<BaseColInfo, Set<String>>> result =
559+
PartitionCompensator.calcInvalidPartitions(
560+
queryUsedPartitions, rewrittenPlan, matCtx, cascadesCtx);
561+
562+
Assertions.assertNotNull(result);
563+
Assertions.assertEquals(ImmutableSet.of("mv_partially_stale", "mv_disjoint"),
564+
result.key().get(new BaseTableInfo(mtmv)));
565+
Assertions.assertEquals(ImmutableSet.of("p1", "p2"), result.value().get(colInfo));
566+
}
567+
490568
@SuppressWarnings("unchecked")
491569
@Test
492570
public void testCalcInvalidPartitionsDoesNotCompensateBasePartitionsUnusedByQuery()

0 commit comments

Comments
 (0)