Skip to content

Commit 3238a51

Browse files
committed
[fix](fe) Fix MTMV refresh predicates for date_add hour partition
### What problem does this PR solve? Issue Number: apache#62395 Related PR: apache#62410 Problem Summary: MTMV partition refresh builds base-table predicates directly from MV partition bounds. When MV partition expression is date_trunc(date_add/sub(col, INTERVAL N HOUR), unit), the bounds must be shifted by the inverse hour offset; otherwise refresh can miss rows or fail with no-partition errors. ### Release note Supports MTMV partition refresh when partition expression includes date_trunc(date_add/sub(..., INTERVAL N HOUR), ...). ### Check List (For Author) - Test: Regression test - ./run-regression-test.sh --run -d mtmv_p0 -s test_rollup_partition_mtmv_date_add (OrbStack Linux) - Behavior changed: Yes (MV refresh predicate mapping for hour-offset partition expressions) - Does this need documentation: No
1 parent 1bec4bc commit 3238a51

2 files changed

Lines changed: 377 additions & 10 deletions

File tree

‎fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommand.java‎

Lines changed: 94 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,10 @@
1717

1818
package org.apache.doris.nereids.trees.plans.commands;
1919

20+
import org.apache.doris.analysis.Expr;
21+
import org.apache.doris.analysis.FunctionCallExpr;
22+
import org.apache.doris.analysis.LiteralExpr;
23+
import org.apache.doris.analysis.TimestampArithmeticExpr;
2024
import org.apache.doris.catalog.Env;
2125
import org.apache.doris.catalog.ListPartitionItem;
2226
import org.apache.doris.catalog.MTMV;
@@ -45,6 +49,10 @@
4549
import org.apache.doris.nereids.trees.expressions.LessThan;
4650
import org.apache.doris.nereids.trees.expressions.Slot;
4751
import org.apache.doris.nereids.trees.expressions.literal.BooleanLiteral;
52+
import org.apache.doris.nereids.trees.expressions.literal.DateLiteral;
53+
import org.apache.doris.nereids.trees.expressions.literal.DateTimeLiteral;
54+
import org.apache.doris.nereids.trees.expressions.literal.DateTimeV2Literal;
55+
import org.apache.doris.nereids.trees.expressions.literal.DateV2Literal;
4856
import org.apache.doris.nereids.trees.expressions.literal.Literal;
4957
import org.apache.doris.nereids.trees.expressions.literal.NullLiteral;
5058
import org.apache.doris.nereids.trees.plans.Plan;
@@ -136,9 +144,10 @@ private static Map<TableIf, Set<Expression>> constructTableWithPredicates(MTMV m
136144
PartitionItem partitionItem = mv.getPartitionItemOrAnalysisException(partitionName);
137145
items.add(partitionItem);
138146
}
147+
Optional<Long> inverseOffsetHours = getInverseHourOffsetForBaseFilter(mv);
139148
ImmutableMap.Builder<TableIf, Set<Expression>> builder = new ImmutableMap.Builder<>();
140149
tableWithPartKey.forEach((table, colName) ->
141-
builder.put(table, constructPredicates(items, colName))
150+
builder.put(table, constructPredicates(items, colName, inverseOffsetHours))
142151
);
143152
return builder.build();
144153
}
@@ -153,23 +162,40 @@ public static Set<Expression> constructPredicates(Set<PartitionItem> partitions,
153162
return constructPredicates(partitions, slot);
154163
}
155164

165+
@VisibleForTesting
166+
public static Set<Expression> constructPredicates(Set<PartitionItem> partitions,
167+
String colName, Optional<Long> hourOffset) {
168+
UnboundSlot slot = new UnboundSlot(colName);
169+
return constructPredicates(partitions, slot, hourOffset);
170+
}
171+
156172
/**
157173
* construct predicates for partition items, the min key is the min key of range items.
158174
* For list partition or less than partition items, the min key is null.
159175
*/
160176
@VisibleForTesting
161177
public static Set<Expression> constructPredicates(Set<PartitionItem> partitions, Slot colSlot) {
178+
return constructPredicates(partitions, colSlot, Optional.empty());
179+
}
180+
181+
/**
182+
* Construct predicates from partition items and apply optional hour shift to date-like bounds.
183+
*/
184+
@VisibleForTesting
185+
public static Set<Expression> constructPredicates(Set<PartitionItem> partitions, Slot colSlot,
186+
Optional<Long> hourOffset) {
162187
Set<Expression> predicates = new HashSet<>();
163188
if (partitions.isEmpty()) {
164189
return Sets.newHashSet(BooleanLiteral.TRUE);
165190
}
191+
long shiftHours = hourOffset.orElse(0L);
166192
if (partitions.iterator().next() instanceof ListPartitionItem) {
167193
for (PartitionItem item : partitions) {
168-
predicates.add(convertListPartitionToIn(item, colSlot));
194+
predicates.add(convertListPartitionToIn(item, colSlot, shiftHours));
169195
}
170196
} else {
171197
for (PartitionItem item : partitions) {
172-
predicates.add(convertRangePartitionToCompare(item, colSlot));
198+
predicates.add(convertRangePartitionToCompare(item, colSlot, shiftHours));
173199
}
174200
}
175201
return predicates;
@@ -180,9 +206,10 @@ private static Expression convertPartitionKeyToLiteral(PartitionKey key) {
180206
Type.fromPrimitiveType(key.getTypes().get(0)));
181207
}
182208

183-
private static Expression convertListPartitionToIn(PartitionItem item, Slot col) {
209+
private static Expression convertListPartitionToIn(PartitionItem item, Slot col, long shiftHours) {
184210
List<Expression> inValues = ((ListPartitionItem) item).getItems().stream()
185211
.map(UpdateMvByPartitionCommand::convertPartitionKeyToLiteral)
212+
.map(literal -> shiftDateLikeLiteralByHours(literal, shiftHours))
186213
.collect(ImmutableList.toImmutableList());
187214
List<Expression> predicates = new ArrayList<>();
188215
if (inValues.stream().anyMatch(NullLiteral.class::isInstance)) {
@@ -201,16 +228,18 @@ private static Expression convertListPartitionToIn(PartitionItem item, Slot col)
201228
return ExpressionUtils.or(predicates);
202229
}
203230

204-
private static Expression convertRangePartitionToCompare(PartitionItem item, Slot col) {
231+
private static Expression convertRangePartitionToCompare(PartitionItem item, Slot col, long shiftHours) {
205232
Range<PartitionKey> range = item.getItems();
206233
List<Expression> expressions = new ArrayList<>();
207234
if (range.hasLowerBound() && !range.lowerEndpoint().isMinValue()) {
208235
PartitionKey key = range.lowerEndpoint();
209-
expressions.add(new GreaterThanEqual(col, convertPartitionKeyToLiteral(key)));
236+
expressions.add(new GreaterThanEqual(col,
237+
shiftDateLikeLiteralByHours(convertPartitionKeyToLiteral(key), shiftHours)));
210238
}
211239
if (range.hasUpperBound() && !range.upperEndpoint().isMaxValue()) {
212240
PartitionKey key = range.upperEndpoint();
213-
expressions.add(new LessThan(col, convertPartitionKeyToLiteral(key)));
241+
expressions.add(new LessThan(col,
242+
shiftDateLikeLiteralByHours(convertPartitionKeyToLiteral(key), shiftHours)));
214243
}
215244
if (expressions.isEmpty()) {
216245
return BooleanLiteral.of(true);
@@ -224,6 +253,64 @@ private static Expression convertRangePartitionToCompare(PartitionItem item, Slo
224253
return predicate;
225254
}
226255

256+
/**
257+
* For partition expression like date_trunc(date_add/sub(base_col, INTERVAL N HOUR), unit),
258+
* the base-table filter should use inverse offset (target - N / + N).
259+
*/
260+
private static Optional<Long> getInverseHourOffsetForBaseFilter(MTMV mv) {
261+
Expr partitionExpr = mv.getMvPartitionInfo().getExpr();
262+
if (!(partitionExpr instanceof FunctionCallExpr)) {
263+
return Optional.empty();
264+
}
265+
FunctionCallExpr functionCallExpr = (FunctionCallExpr) partitionExpr;
266+
if (!"date_trunc".equalsIgnoreCase(functionCallExpr.getFnName().getFunction())) {
267+
return Optional.empty();
268+
}
269+
List<Expr> params = functionCallExpr.getParams().exprs();
270+
if (params.size() != 2 || !(params.get(0) instanceof TimestampArithmeticExpr)) {
271+
return Optional.empty();
272+
}
273+
TimestampArithmeticExpr timestampArithmeticExpr = (TimestampArithmeticExpr) params.get(0);
274+
if (timestampArithmeticExpr.getTimeUnit() != TimestampArithmeticExpr.TimeUnit.HOUR
275+
|| !(timestampArithmeticExpr.getChild(1) instanceof LiteralExpr)) {
276+
return Optional.empty();
277+
}
278+
long offsetHours;
279+
try {
280+
offsetHours = Long.parseLong(((LiteralExpr) timestampArithmeticExpr.getChild(1)).getStringValue());
281+
} catch (NumberFormatException e) {
282+
return Optional.empty();
283+
}
284+
String funcName = timestampArithmeticExpr.getFuncName().toLowerCase();
285+
if ("date_sub".equals(funcName)) {
286+
offsetHours = -offsetHours;
287+
} else if (!"date_add".equals(funcName)) {
288+
return Optional.empty();
289+
}
290+
return Optional.of(-offsetHours);
291+
}
292+
293+
private static Expression shiftDateLikeLiteralByHours(Expression expression, long shiftHours) {
294+
if (shiftHours == 0) {
295+
return expression;
296+
}
297+
if (expression instanceof DateTimeV2Literal) {
298+
return ((DateTimeV2Literal) expression).plusHours(shiftHours);
299+
}
300+
if (expression instanceof DateTimeLiteral) {
301+
return ((DateTimeLiteral) expression).plusHours(shiftHours);
302+
}
303+
if (expression instanceof DateV2Literal) {
304+
return ((DateV2Literal) expression).toBeginOfTheDay().plusHours(shiftHours);
305+
}
306+
if (expression instanceof DateLiteral) {
307+
DateLiteral dateLiteral = (DateLiteral) expression;
308+
return new DateTimeLiteral(dateLiteral.getYear(), dateLiteral.getMonth(), dateLiteral.getDay(),
309+
0, 0, 0).plusHours(shiftHours);
310+
}
311+
return expression;
312+
}
313+
227314
/**
228315
* Add predicates on base table when mv can partition update, Also support plan that contain cte and view
229316
*/

0 commit comments

Comments
 (0)