Skip to content

Commit 80daf50

Browse files
committed
[improvement](fe) Support hour-offset date_add/date_sub in MTMV partition rollup
### What problem does this PR solve? Issue Number: close apache#62395 Related PR: None Problem Summary: MTMV partition rollup only handled `date_trunc(partition_col, unit)`. It could not track partition lineage when the MV partition expression uses hour offset conversion such as `date_trunc(date_add(k2, INTERVAL 3 HOUR), 'day')` or `date_trunc(date_sub(k2, INTERVAL 3 HOUR), 'day')`, which is needed for UTC-to-local-day partitioning and incremental maintenance. ### Release note Support MTMV partition rollup and partition increment check for `date_trunc(date_add/date_sub(partition_col, INTERVAL N HOUR), unit)`. ### Check List (For Author) - Test: Regression test / Unit Test - Unit Test: `./run-fe-ut.sh --run org.apache.doris.mtmv.MTMVRelatedPartitionDescRollUpGeneratorTest` - Build: `MVN_OPT='-Dmaven.build.cache.enabled=false' ./build.sh --fe --clean -j$(nproc)` - Regression test: `./run-regression-test.sh --run -d mtmv_p0 -s test_rollup_partition_mtmv_date_add` (failed locally because FE/BE test cluster was not started; connection refused) - Behavior changed: Yes (MTMV partition expression now supports hour-offset `date_add/date_sub` lineage under `date_trunc`) - Does this need documentation: No
1 parent 4e81ace commit 80daf50

6 files changed

Lines changed: 524 additions & 1 deletion

File tree

Lines changed: 299 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,299 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
package org.apache.doris.mtmv;
19+
20+
import org.apache.doris.analysis.Expr;
21+
import org.apache.doris.analysis.FunctionCallExpr;
22+
import org.apache.doris.analysis.LiteralExpr;
23+
import org.apache.doris.analysis.PartitionExprUtil;
24+
import org.apache.doris.analysis.PartitionKeyDesc;
25+
import org.apache.doris.analysis.PartitionValue;
26+
import org.apache.doris.analysis.StringLiteral;
27+
import org.apache.doris.analysis.TimestampArithmeticExpr;
28+
import org.apache.doris.catalog.PartitionType;
29+
import org.apache.doris.catalog.Type;
30+
import org.apache.doris.common.AnalysisException;
31+
import org.apache.doris.common.util.PropertyAnalyzer;
32+
import org.apache.doris.datasource.mvcc.MvccUtil;
33+
import org.apache.doris.nereids.trees.expressions.Expression;
34+
import org.apache.doris.nereids.trees.expressions.functions.executable.DateTimeArithmetic;
35+
import org.apache.doris.nereids.trees.expressions.functions.executable.DateTimeExtractAndTransform;
36+
import org.apache.doris.nereids.trees.expressions.literal.BigIntLiteral;
37+
import org.apache.doris.nereids.trees.expressions.literal.DateTimeV2Literal;
38+
import org.apache.doris.nereids.trees.expressions.literal.DateV2Literal;
39+
import org.apache.doris.nereids.trees.expressions.literal.VarcharLiteral;
40+
41+
import com.google.common.base.Preconditions;
42+
import com.google.common.collect.ImmutableSet;
43+
import org.apache.commons.lang3.StringUtils;
44+
45+
import java.util.Collections;
46+
import java.util.List;
47+
import java.util.Map;
48+
import java.util.Objects;
49+
import java.util.Optional;
50+
import java.util.Set;
51+
52+
/**
53+
* Rollup service for expressions:
54+
* date_trunc(date_add/date_sub(partition_col, interval N hour), 'unit')
55+
*/
56+
public class MTMVPartitionExprDateTruncDateAddSub implements MTMVPartitionExprService {
57+
private static final Set<String> TIME_UNITS = ImmutableSet.of(
58+
"year", "quarter", "week", "month", "day", "hour");
59+
private final String timeUnit;
60+
private final long offsetHours;
61+
62+
public MTMVPartitionExprDateTruncDateAddSub(FunctionCallExpr functionCallExpr) throws AnalysisException {
63+
List<Expr> paramsExprs = functionCallExpr.getParams().exprs();
64+
if (paramsExprs.size() != 2) {
65+
throw new AnalysisException("date_trunc params exprs size should be 2.");
66+
}
67+
Expr dateTruncUnit = paramsExprs.get(1);
68+
if (!(dateTruncUnit instanceof StringLiteral)) {
69+
throw new AnalysisException("date_trunc param of time unit is not string literal.");
70+
}
71+
this.timeUnit = dateTruncUnit.getStringValue().toLowerCase();
72+
73+
Expr dateTruncArg = paramsExprs.get(0);
74+
if (!(dateTruncArg instanceof TimestampArithmeticExpr)) {
75+
throw new AnalysisException("date_trunc first argument should be date_add/date_sub in mtmv partition");
76+
}
77+
TimestampArithmeticExpr timestampArithmeticExpr = (TimestampArithmeticExpr) dateTruncArg;
78+
if (timestampArithmeticExpr.getTimeUnit() != TimestampArithmeticExpr.TimeUnit.HOUR) {
79+
throw new AnalysisException("only HOUR unit is supported in date_add/date_sub for mtmv partition");
80+
}
81+
if (!(timestampArithmeticExpr.getChild(1) instanceof LiteralExpr)) {
82+
throw new AnalysisException("date_add/date_sub offset should be a literal");
83+
}
84+
long offset = parseOffsetHours((LiteralExpr) timestampArithmeticExpr.getChild(1));
85+
String functionName = timestampArithmeticExpr.getFuncName().toLowerCase();
86+
if ("date_sub".equals(functionName)) {
87+
this.offsetHours = -offset;
88+
} else if ("date_add".equals(functionName)) {
89+
this.offsetHours = offset;
90+
} else {
91+
throw new AnalysisException("only date_add/date_sub is supported in mtmv partition");
92+
}
93+
}
94+
95+
@Override
96+
public String getRollUpIdentity(PartitionKeyDesc partitionKeyDesc, Map<String, String> mvProperties)
97+
throws AnalysisException {
98+
String res = null;
99+
Optional<String> dateFormat = getDateFormat(mvProperties);
100+
List<List<PartitionValue>> inValues = partitionKeyDesc.getInValues();
101+
for (int i = 0; i < inValues.size(); i++) {
102+
PartitionValue partitionValue = inValues.get(i).get(0);
103+
if (partitionValue.isNullPartition()) {
104+
throw new AnalysisException("date_trunc + date_add/date_sub not support null partition value");
105+
}
106+
String identity = dateTruncByOffset(partitionValue.getStringValue(), dateFormat, false).toString();
107+
if (i == 0) {
108+
res = identity;
109+
} else if (!Objects.equals(res, identity)) {
110+
throw new AnalysisException(
111+
String.format("partition values not equal, res: %s, identity: %s", res, identity));
112+
}
113+
}
114+
return res;
115+
}
116+
117+
@Override
118+
public PartitionKeyDesc generateRollUpPartitionKeyDesc(PartitionKeyDesc partitionKeyDesc,
119+
MTMVPartitionInfo mvPartitionInfo, MTMVRelatedTableIf pctTable) throws AnalysisException {
120+
Type partitionColumnType = MTMVPartitionUtil
121+
.getPartitionColumnType(pctTable, mvPartitionInfo.getPartitionColByPctTable(pctTable));
122+
Preconditions.checkState(partitionKeyDesc.getLowerValues().size() == 1,
123+
"only support one partition column");
124+
DateTimeV2Literal beginTime = dateTruncByOffset(
125+
partitionKeyDesc.getLowerValues().get(0).getStringValue(), Optional.empty(), false);
126+
PartitionValue lowerValue = new PartitionValue(dateTimeToStr(beginTime, partitionColumnType));
127+
PartitionValue upperValue = getUpperValue(partitionKeyDesc.getUpperValues().get(0), beginTime,
128+
partitionColumnType);
129+
return PartitionKeyDesc.createFixed(
130+
Collections.singletonList(lowerValue),
131+
Collections.singletonList(upperValue));
132+
}
133+
134+
@Override
135+
public void analyze(MTMVPartitionInfo mvPartitionInfo) throws AnalysisException {
136+
if (!TIME_UNITS.contains(this.timeUnit)) {
137+
throw new AnalysisException(
138+
String.format("timeUnit not support: %s, only support: %s", this.timeUnit, TIME_UNITS));
139+
}
140+
List<BaseColInfo> pctInfos = mvPartitionInfo.getPctInfos();
141+
for (BaseColInfo pctInfo : pctInfos) {
142+
MTMVRelatedTableIf pctTable = MTMVUtil.getRelatedTable(pctInfo.getTableInfo());
143+
PartitionType partitionType = pctTable.getPartitionType(MvccUtil.getSnapshotFromContext(pctTable));
144+
if (partitionType == PartitionType.RANGE) {
145+
Type partitionColumnType = MTMVPartitionUtil.getPartitionColumnType(pctTable, pctInfo.getColName());
146+
if (!partitionColumnType.isDateType()) {
147+
throw new AnalysisException(
148+
"partitionColumnType should be date/datetime "
149+
+ "when PartitionType is range and expr is date_trunc + date_add/date_sub");
150+
}
151+
} else {
152+
throw new AnalysisException("date_trunc + date_add/date_sub only support range partition");
153+
}
154+
}
155+
}
156+
157+
@Override
158+
public String toSql(MTMVPartitionInfo mvPartitionInfo) {
159+
String funcName = offsetHours >= 0 ? "date_add" : "date_sub";
160+
return String.format("date_trunc(%s(`%s`, INTERVAL %s HOUR), '%s')",
161+
funcName, mvPartitionInfo.getPartitionCol(), Math.abs(offsetHours), timeUnit);
162+
}
163+
164+
private PartitionValue getUpperValue(PartitionValue upperValue, DateTimeV2Literal beginTruncTime,
165+
Type partitionColumnType) throws AnalysisException {
166+
if (upperValue.isMax()) {
167+
throw new AnalysisException("date_trunc + date_add/date_sub not support MAXVALUE partition");
168+
}
169+
DateTimeV2Literal endTruncTime = dateTruncByOffset(upperValue.getStringValue(), Optional.empty(), true);
170+
if (!Objects.equals(beginTruncTime, endTruncTime)) {
171+
throw new AnalysisException(
172+
String.format("partition values not equal, beginTruncTime: %s, endTruncTime: %s",
173+
beginTruncTime, endTruncTime));
174+
}
175+
DateTimeV2Literal endTime = dateIncrement(beginTruncTime);
176+
return new PartitionValue(dateTimeToStr(endTime, partitionColumnType));
177+
}
178+
179+
private DateTimeV2Literal dateTruncByOffset(String value, Optional<String> dateFormat, boolean isUpper)
180+
throws AnalysisException {
181+
DateTimeV2Literal dateTimeLiteral = strToDate(value, dateFormat);
182+
dateTimeLiteral = dateOffset(dateTimeLiteral);
183+
if (isUpper) {
184+
dateTimeLiteral = (DateTimeV2Literal) DateTimeArithmetic.secondsSub(dateTimeLiteral, new BigIntLiteral(1));
185+
}
186+
Expression expression = DateTimeExtractAndTransform.dateTrunc(dateTimeLiteral, new VarcharLiteral(timeUnit));
187+
if (!(expression instanceof DateTimeV2Literal)) {
188+
throw new AnalysisException("dateTrunc() should return DateLiteral, expression: " + expression);
189+
}
190+
return (DateTimeV2Literal) expression;
191+
}
192+
193+
private DateTimeV2Literal dateOffset(DateTimeV2Literal value) throws AnalysisException {
194+
long offsetSeconds = offsetHours * 3600L;
195+
Expression result = offsetSeconds >= 0
196+
? DateTimeArithmetic.secondsAdd(value, new BigIntLiteral(offsetSeconds))
197+
: DateTimeArithmetic.secondsSub(value, new BigIntLiteral(-offsetSeconds));
198+
if (!(result instanceof DateTimeV2Literal)) {
199+
throw new AnalysisException("date offset should return DateTimeV2Literal, result: " + result);
200+
}
201+
return (DateTimeV2Literal) result;
202+
}
203+
204+
private DateTimeV2Literal strToDate(String value, Optional<String> dateFormat) throws AnalysisException {
205+
try {
206+
return new DateTimeV2Literal(value);
207+
} catch (Exception e) {
208+
if (!dateFormat.isPresent()) {
209+
throw e;
210+
}
211+
Expression strToDate = DateTimeExtractAndTransform.strToDate(new VarcharLiteral(value),
212+
new VarcharLiteral(dateFormat.get()));
213+
if (strToDate instanceof DateV2Literal) {
214+
DateV2Literal dateV2Literal = (DateV2Literal) strToDate;
215+
return new DateTimeV2Literal(dateV2Literal.getYear(), dateV2Literal.getMonth(), dateV2Literal.getDay(),
216+
0, 0, 0);
217+
} else if (strToDate instanceof DateTimeV2Literal) {
218+
return (DateTimeV2Literal) strToDate;
219+
} else {
220+
throw new AnalysisException(
221+
String.format("strToDate failed, stringValue: %s, dateFormat: %s", value, dateFormat));
222+
}
223+
}
224+
}
225+
226+
private DateTimeV2Literal dateIncrement(DateTimeV2Literal value) throws AnalysisException {
227+
Expression result;
228+
switch (timeUnit) {
229+
case "year":
230+
result = value.plusYears(1L);
231+
break;
232+
case "quarter":
233+
result = value.plusMonths(3L);
234+
break;
235+
case "month":
236+
result = value.plusMonths(1L);
237+
break;
238+
case "week":
239+
result = value.plusWeeks(1L);
240+
break;
241+
case "day":
242+
result = value.plusDays(1L);
243+
break;
244+
case "hour":
245+
result = value.plusHours(1L);
246+
break;
247+
default:
248+
throw new AnalysisException(
249+
"async materialized view partition roll up not support timeUnit: " + timeUnit);
250+
}
251+
if (!(result instanceof DateTimeV2Literal)) {
252+
throw new AnalysisException("dateIncrement() should return DateTimeLiteral, result: " + result);
253+
}
254+
return (DateTimeV2Literal) result;
255+
}
256+
257+
private String dateTimeToStr(DateTimeV2Literal literal, Type partitionColumnType) throws AnalysisException {
258+
if (partitionColumnType.isDate() || partitionColumnType.isDateV2()) {
259+
return String.format(PartitionExprUtil.DATE_FORMATTER, literal.getYear(), literal.getMonth(),
260+
literal.getDay());
261+
} else if (partitionColumnType.isDatetime() || partitionColumnType.isDatetimeV2()
262+
|| partitionColumnType.isTimeStampTz()) {
263+
return String.format(PartitionExprUtil.DATETIME_FORMATTER,
264+
literal.getYear(), literal.getMonth(), literal.getDay(),
265+
literal.getHour(), literal.getMinute(), literal.getSecond());
266+
} else {
267+
throw new AnalysisException(
268+
"MTMV not support partition with column type : " + partitionColumnType);
269+
}
270+
}
271+
272+
private Optional<String> getDateFormat(Map<String, String> mvProperties) {
273+
return StringUtils.isEmpty(mvProperties.get(PropertyAnalyzer.PROPERTIES_PARTITION_DATE_FORMAT))
274+
? Optional.empty()
275+
: Optional.of(mvProperties.get(PropertyAnalyzer.PROPERTIES_PARTITION_DATE_FORMAT));
276+
}
277+
278+
private long parseOffsetHours(LiteralExpr offsetExpr) throws AnalysisException {
279+
try {
280+
return Long.parseLong(offsetExpr.getStringValue());
281+
} catch (NumberFormatException e) {
282+
throw new AnalysisException("date_add/date_sub hour offset should be integer");
283+
}
284+
}
285+
286+
@Override
287+
public boolean equals(Object o) {
288+
if (o == null || getClass() != o.getClass()) {
289+
return false;
290+
}
291+
MTMVPartitionExprDateTruncDateAddSub that = (MTMVPartitionExprDateTruncDateAddSub) o;
292+
return offsetHours == that.offsetHours && Objects.equals(timeUnit, that.timeUnit);
293+
}
294+
295+
@Override
296+
public int hashCode() {
297+
return Objects.hash(timeUnit, offsetHours);
298+
}
299+
}

‎fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionExprFactory.java‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,8 +19,11 @@
1919

2020
import org.apache.doris.analysis.Expr;
2121
import org.apache.doris.analysis.FunctionCallExpr;
22+
import org.apache.doris.analysis.TimestampArithmeticExpr;
2223
import org.apache.doris.common.AnalysisException;
2324

25+
import java.util.List;
26+
2427
/**
2528
* MTMV Partition Expr Factory
2629
*/
@@ -32,6 +35,10 @@ public static MTMVPartitionExprService getExprService(Expr expr) throws Analysis
3235
FunctionCallExpr functionCallExpr = (FunctionCallExpr) expr;
3336
String fnName = functionCallExpr.getFnName().getFunction().toLowerCase();
3437
if ("date_trunc".equals(fnName)) {
38+
List<Expr> paramsExprs = functionCallExpr.getParams().exprs();
39+
if (paramsExprs.size() == 2 && paramsExprs.get(0) instanceof TimestampArithmeticExpr) {
40+
return new MTMVPartitionExprDateTruncDateAddSub(functionCallExpr);
41+
}
3542
return new MTMVPartitionExprDateTrunc(functionCallExpr);
3643
}
3744
throw new AnalysisException("async materialized view partition not support function name: " + fnName);

‎fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PartitionIncrementMaintainer.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,8 @@
3535
import org.apache.doris.nereids.trees.expressions.SlotReference;
3636
import org.apache.doris.nereids.trees.expressions.WindowExpression;
3737
import org.apache.doris.nereids.trees.expressions.functions.scalar.DateTrunc;
38+
import org.apache.doris.nereids.trees.expressions.functions.scalar.HoursAdd;
39+
import org.apache.doris.nereids.trees.expressions.functions.scalar.HoursSub;
3840
import org.apache.doris.nereids.trees.expressions.literal.Literal;
3941
import org.apache.doris.nereids.trees.expressions.visitor.DefaultExpressionRewriter;
4042
import org.apache.doris.nereids.trees.plans.JoinType;
@@ -91,7 +93,7 @@ public static class PartitionIncrementChecker extends
9193
DefaultPlanVisitor<Void, PartitionIncrementCheckContext> {
9294
public static final PartitionIncrementChecker INSTANCE = new PartitionIncrementChecker();
9395
public static final Set<Class<? extends Expression>> SUPPORT_EXPRESSION_TYPES =
94-
ImmutableSet.of(DateTrunc.class, SlotReference.class, Literal.class);
96+
ImmutableSet.of(DateTrunc.class, HoursAdd.class, HoursSub.class, SlotReference.class, Literal.class);
9597

9698
@Override
9799
public Void visitLogicalProject(LogicalProject<? extends Plan> project,

‎fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/MTMVPartitionDefinition.java‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
import org.apache.doris.analysis.FunctionCallExpr;
2525
import org.apache.doris.analysis.SlotRef;
2626
import org.apache.doris.analysis.StringLiteral;
27+
import org.apache.doris.analysis.TimestampArithmeticExpr;
2728
import org.apache.doris.catalog.Column;
2829
import org.apache.doris.catalog.PartitionType;
2930
import org.apache.doris.common.Pair;
@@ -47,6 +48,8 @@
4748
import org.apache.doris.nereids.trees.expressions.Expression;
4849
import org.apache.doris.nereids.trees.expressions.Slot;
4950
import org.apache.doris.nereids.trees.expressions.functions.scalar.DateTrunc;
51+
import org.apache.doris.nereids.trees.expressions.functions.scalar.HoursAdd;
52+
import org.apache.doris.nereids.trees.expressions.functions.scalar.HoursSub;
5053
import org.apache.doris.nereids.trees.expressions.literal.Literal;
5154

5255
import com.google.common.collect.Lists;
@@ -180,6 +183,10 @@ private static List<Pair<Integer, Expr>> convertDateTruncToLegacyArguments(List<
180183
private static Pair<Integer, Expr> convertToLegacyRecursion(Expression expression) {
181184
if (expression instanceof Slot) {
182185
return Pair.of(1, new SlotRef(null, ((Slot) expression).getName()));
186+
} else if (expression instanceof HoursAdd) {
187+
return Pair.of(1, convertDateAddSubToLegacy((HoursAdd) expression, "date_add"));
188+
} else if (expression instanceof HoursSub) {
189+
return Pair.of(1, convertDateAddSubToLegacy((HoursSub) expression, "date_sub"));
183190
} else if (expression instanceof Literal) {
184191
return Pair.of(2, new StringLiteral(((Literal) expression).getStringValue()));
185192
} else if (expression instanceof Cast) {
@@ -190,6 +197,18 @@ private static Pair<Integer, Expr> convertToLegacyRecursion(Expression expressio
190197
}
191198
}
192199

200+
private static TimestampArithmeticExpr convertDateAddSubToLegacy(Expression expression, String funcName) {
201+
Pair<Integer, Expr> timeExprPair = convertToLegacyRecursion(expression.child(0));
202+
if (timeExprPair.key() != 1) {
203+
throw new AnalysisException("unsupported date arithmetic argument " + expression.toString());
204+
}
205+
if (!(expression.child(1) instanceof Literal)) {
206+
throw new AnalysisException("date arithmetic offset should be literal " + expression.toString());
207+
}
208+
Expr amountExpr = ((Literal) expression.child(1)).toLegacyLiteral();
209+
return new TimestampArithmeticExpr(funcName, timeExprPair.value(), amountExpr, "HOUR");
210+
}
211+
193212
public MTMVPartitionType getPartitionType() {
194213
return partitionType;
195214
}

0 commit comments

Comments
 (0)