Skip to content

Commit c47c034

Browse files
committed
[feature](mtmv) Support hour-offset MTMV partition expressions with UTC-midnight base partitions
### What problem does this PR solve? Issue Number: close apache#62395 Problem Summary: Implement 1-to-N partition mapping to support hour-offset MTMV partition expressions when base table partitions are aligned to UTC-midnight boundaries. ### Release note Feature: Support hour-offset MTMV partition expressions with UTC-midnight base partitions. ### Check List (For Author) - Test: Regression test + Unit test - Behavior changed: Yes - Does this need documentation: Yes
1 parent 6e655c3 commit c47c034

9 files changed

Lines changed: 1658 additions & 12 deletions
Lines changed: 315 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,315 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
package org.apache.doris.mtmv;
19+
20+
import org.apache.doris.analysis.CastExpr;
21+
import org.apache.doris.analysis.Expr;
22+
import org.apache.doris.analysis.ExprToSqlVisitor;
23+
import org.apache.doris.analysis.FunctionCallExpr;
24+
import org.apache.doris.analysis.LiteralExpr;
25+
import org.apache.doris.analysis.PartitionExprUtil;
26+
import org.apache.doris.analysis.PartitionKeyDesc;
27+
import org.apache.doris.analysis.PartitionValue;
28+
import org.apache.doris.analysis.StringLiteral;
29+
import org.apache.doris.analysis.TimestampArithmeticExpr;
30+
import org.apache.doris.analysis.ToSqlParams;
31+
import org.apache.doris.catalog.PartitionType;
32+
import org.apache.doris.catalog.Type;
33+
import org.apache.doris.common.AnalysisException;
34+
import org.apache.doris.common.util.PropertyAnalyzer;
35+
import org.apache.doris.datasource.mvcc.MvccUtil;
36+
import org.apache.doris.nereids.trees.expressions.Expression;
37+
import org.apache.doris.nereids.trees.expressions.functions.executable.DateTimeArithmetic;
38+
import org.apache.doris.nereids.trees.expressions.functions.executable.DateTimeExtractAndTransform;
39+
import org.apache.doris.nereids.trees.expressions.literal.BigIntLiteral;
40+
import org.apache.doris.nereids.trees.expressions.literal.DateTimeV2Literal;
41+
import org.apache.doris.nereids.trees.expressions.literal.DateV2Literal;
42+
import org.apache.doris.nereids.trees.expressions.literal.VarcharLiteral;
43+
44+
import com.google.common.base.Preconditions;
45+
import com.google.common.collect.ImmutableSet;
46+
import org.apache.commons.lang3.StringUtils;
47+
48+
import java.util.ArrayList;
49+
import java.util.Collections;
50+
import java.util.List;
51+
import java.util.Map;
52+
import java.util.Objects;
53+
import java.util.Optional;
54+
import java.util.Set;
55+
56+
/**
57+
* Rollup service for expressions:
58+
* date_trunc(date_add/date_sub(partition_col, interval N hour), 'unit')
59+
*/
60+
public class MTMVPartitionExprDateTruncDateAddSub implements MTMVPartitionExprService {
61+
private static final Set<String> TIME_UNITS = ImmutableSet.of(
62+
"year", "quarter", "week", "month", "day", "hour");
63+
private final String timeUnit;
64+
private final long offsetHours;
65+
66+
public MTMVPartitionExprDateTruncDateAddSub(FunctionCallExpr functionCallExpr) throws AnalysisException {
67+
List<Expr> paramsExprs = functionCallExpr.getParams().exprs();
68+
if (paramsExprs.size() != 2) {
69+
throw new AnalysisException("date_trunc params exprs size should be 2.");
70+
}
71+
Expr dateTruncUnit = paramsExprs.get(1);
72+
if (!(dateTruncUnit instanceof StringLiteral)) {
73+
throw new AnalysisException("date_trunc param of time unit is not string literal.");
74+
}
75+
this.timeUnit = dateTruncUnit.getStringValue().toLowerCase();
76+
77+
Expr dateTruncArg = paramsExprs.get(0);
78+
while (dateTruncArg instanceof CastExpr) {
79+
dateTruncArg = dateTruncArg.getChild(0);
80+
}
81+
if (!(dateTruncArg instanceof TimestampArithmeticExpr)) {
82+
throw new AnalysisException("date_trunc first argument should be date_add/date_sub in mtmv partition");
83+
}
84+
TimestampArithmeticExpr timestampArithmeticExpr = (TimestampArithmeticExpr) dateTruncArg;
85+
if (timestampArithmeticExpr.getTimeUnit() != TimestampArithmeticExpr.TimeUnit.HOUR) {
86+
throw new AnalysisException("only HOUR unit is supported in date_add/date_sub for mtmv partition");
87+
}
88+
if (!(timestampArithmeticExpr.getChild(1) instanceof LiteralExpr)) {
89+
throw new AnalysisException("date_add/date_sub offset should be a literal");
90+
}
91+
long offset = parseOffsetHours((LiteralExpr) timestampArithmeticExpr.getChild(1));
92+
String functionName = timestampArithmeticExpr.getFuncName().toLowerCase();
93+
if ("date_sub".equals(functionName)) {
94+
this.offsetHours = -offset;
95+
} else if ("date_add".equals(functionName)) {
96+
this.offsetHours = offset;
97+
} else {
98+
throw new AnalysisException("only date_add/date_sub is supported in mtmv partition");
99+
}
100+
}
101+
102+
@Override
103+
public String getRollUpIdentity(PartitionKeyDesc partitionKeyDesc, Map<String, String> mvProperties)
104+
throws AnalysisException {
105+
DateTimeV2Literal first = null;
106+
Optional<String> dateFormat = getDateFormat(mvProperties);
107+
List<List<PartitionValue>> inValues = partitionKeyDesc.getInValues();
108+
for (int i = 0; i < inValues.size(); i++) {
109+
PartitionValue partitionValue = inValues.get(i).get(0);
110+
if (partitionValue.isNullPartition()) {
111+
throw new AnalysisException("date_trunc + date_add/date_sub not support null partition value");
112+
}
113+
DateTimeV2Literal identity = dateTruncByOffset(partitionValue.getStringValue(), dateFormat, false);
114+
if (i == 0) {
115+
first = identity;
116+
} else if (!isSameTime(first, identity)) {
117+
throw new AnalysisException(
118+
String.format("partition values not equal, first: %s, identity: %s", first, identity));
119+
}
120+
}
121+
Preconditions.checkState(first != null, "partition values is empty");
122+
return first.toString();
123+
}
124+
125+
@Override
126+
public PartitionKeyDesc generateRollUpPartitionKeyDesc(PartitionKeyDesc partitionKeyDesc,
127+
MTMVPartitionInfo mvPartitionInfo, MTMVRelatedTableIf pctTable) throws AnalysisException {
128+
List<PartitionKeyDesc> descs = generateRollUpPartitionKeyDescs(partitionKeyDesc, mvPartitionInfo, pctTable);
129+
Preconditions.checkState(descs.size() == 1,
130+
"expected single roll-up partition desc, but got: %s", descs);
131+
return descs.get(0);
132+
}
133+
134+
@Override
135+
public List<PartitionKeyDesc> generateRollUpPartitionKeyDescs(PartitionKeyDesc partitionKeyDesc,
136+
MTMVPartitionInfo mvPartitionInfo, MTMVRelatedTableIf pctTable) throws AnalysisException {
137+
Type partitionColumnType = MTMVPartitionUtil
138+
.getPartitionColumnType(pctTable, mvPartitionInfo.getPartitionColByPctTable(pctTable));
139+
Preconditions.checkState(partitionKeyDesc.getLowerValues().size() == 1,
140+
"only support one partition column");
141+
142+
DateTimeV2Literal beginBucket = dateTruncByOffset(
143+
partitionKeyDesc.getLowerValues().get(0).getStringValue(), Optional.empty(), false);
144+
DateTimeV2Literal endBucket = dateTruncByOffset(
145+
partitionKeyDesc.getUpperValues().get(0).getStringValue(), Optional.empty(), true);
146+
147+
List<PartitionKeyDesc> res = new ArrayList<>();
148+
DateTimeV2Literal currentBucket = beginBucket;
149+
while (!currentBucket.toJavaDateType().isAfter(endBucket.toJavaDateType())) {
150+
PartitionValue lowerValue = new PartitionValue(dateTimeToStr(currentBucket, partitionColumnType));
151+
DateTimeV2Literal nextBucket = dateIncrement(currentBucket);
152+
PartitionValue upperValue = new PartitionValue(dateTimeToStr(nextBucket, partitionColumnType));
153+
res.add(PartitionKeyDesc.createFixed(
154+
Collections.singletonList(lowerValue),
155+
Collections.singletonList(upperValue)));
156+
currentBucket = nextBucket;
157+
}
158+
return res;
159+
}
160+
161+
@Override
162+
public void analyze(MTMVPartitionInfo mvPartitionInfo) throws AnalysisException {
163+
if (!TIME_UNITS.contains(this.timeUnit)) {
164+
throw new AnalysisException(
165+
String.format("timeUnit not support: %s, only support: %s", this.timeUnit, TIME_UNITS));
166+
}
167+
List<BaseColInfo> pctInfos = mvPartitionInfo.getPctInfos();
168+
for (BaseColInfo pctInfo : pctInfos) {
169+
MTMVRelatedTableIf pctTable = MTMVUtil.getRelatedTable(pctInfo.getTableInfo());
170+
PartitionType partitionType = pctTable.getPartitionType(MvccUtil.getSnapshotFromContext(pctTable));
171+
if (partitionType == PartitionType.RANGE) {
172+
Type partitionColumnType = MTMVPartitionUtil.getPartitionColumnType(pctTable, pctInfo.getColName());
173+
if (!(partitionColumnType.isDatetime() || partitionColumnType.isDatetimeV2())) {
174+
throw new AnalysisException(
175+
"partitionColumnType should be datetime/datetimev2 "
176+
+ "when PartitionType is range and expr is date_trunc + date_add/date_sub");
177+
}
178+
} else {
179+
throw new AnalysisException("date_trunc + date_add/date_sub only support range partition");
180+
}
181+
}
182+
}
183+
184+
@Override
185+
public String toSql(MTMVPartitionInfo mvPartitionInfo) {
186+
Preconditions.checkState(mvPartitionInfo.getExpr() != null, "mtmv partition expr is null");
187+
return mvPartitionInfo.getExpr().accept(ExprToSqlVisitor.INSTANCE, ToSqlParams.WITHOUT_TABLE);
188+
}
189+
190+
private DateTimeV2Literal dateTruncByOffset(String value, Optional<String> dateFormat, boolean isUpper)
191+
throws AnalysisException {
192+
DateTimeV2Literal dateTimeLiteral = strToDate(value, dateFormat);
193+
dateTimeLiteral = dateOffset(dateTimeLiteral);
194+
if (isUpper) {
195+
dateTimeLiteral = (DateTimeV2Literal) DateTimeArithmetic.microSecondsSub(
196+
dateTimeLiteral, new BigIntLiteral(1));
197+
}
198+
Expression expression = DateTimeExtractAndTransform.dateTrunc(dateTimeLiteral, new VarcharLiteral(timeUnit));
199+
if (!(expression instanceof DateTimeV2Literal)) {
200+
throw new AnalysisException("dateTrunc() should return DateLiteral, expression: " + expression);
201+
}
202+
return (DateTimeV2Literal) expression;
203+
}
204+
205+
private static boolean isSameTime(DateTimeV2Literal left, DateTimeV2Literal right) {
206+
return left.toJavaDateType().equals(right.toJavaDateType());
207+
}
208+
209+
private DateTimeV2Literal dateOffset(DateTimeV2Literal value) throws AnalysisException {
210+
long offsetSeconds = offsetHours * 3600L;
211+
Expression result = offsetSeconds >= 0
212+
? DateTimeArithmetic.secondsAdd(value, new BigIntLiteral(offsetSeconds))
213+
: DateTimeArithmetic.secondsSub(value, new BigIntLiteral(-offsetSeconds));
214+
if (!(result instanceof DateTimeV2Literal)) {
215+
throw new AnalysisException("date offset should return DateTimeV2Literal, result: " + result);
216+
}
217+
return (DateTimeV2Literal) result;
218+
}
219+
220+
private DateTimeV2Literal strToDate(String value, Optional<String> dateFormat) throws AnalysisException {
221+
try {
222+
return new DateTimeV2Literal(value);
223+
} catch (Exception e) {
224+
if (!dateFormat.isPresent()) {
225+
throw e;
226+
}
227+
Expression strToDate = DateTimeExtractAndTransform.strToDate(new VarcharLiteral(value),
228+
new VarcharLiteral(dateFormat.get()));
229+
if (strToDate instanceof DateV2Literal) {
230+
DateV2Literal dateV2Literal = (DateV2Literal) strToDate;
231+
return new DateTimeV2Literal(dateV2Literal.getYear(), dateV2Literal.getMonth(), dateV2Literal.getDay(),
232+
0, 0, 0);
233+
} else if (strToDate instanceof DateTimeV2Literal) {
234+
return (DateTimeV2Literal) strToDate;
235+
} else {
236+
throw new AnalysisException(
237+
String.format("strToDate failed, stringValue: %s, dateFormat: %s", value, dateFormat));
238+
}
239+
}
240+
}
241+
242+
private DateTimeV2Literal dateIncrement(DateTimeV2Literal value) throws AnalysisException {
243+
Expression result;
244+
switch (timeUnit) {
245+
case "year":
246+
result = value.plusYears(1L);
247+
break;
248+
case "quarter":
249+
result = value.plusMonths(3L);
250+
break;
251+
case "month":
252+
result = value.plusMonths(1L);
253+
break;
254+
case "week":
255+
result = value.plusWeeks(1L);
256+
break;
257+
case "day":
258+
result = value.plusDays(1L);
259+
break;
260+
case "hour":
261+
result = value.plusHours(1L);
262+
break;
263+
default:
264+
throw new AnalysisException(
265+
"async materialized view partition roll up not support timeUnit: " + timeUnit);
266+
}
267+
if (!(result instanceof DateTimeV2Literal)) {
268+
throw new AnalysisException("dateIncrement() should return DateTimeLiteral, result: " + result);
269+
}
270+
return (DateTimeV2Literal) result;
271+
}
272+
273+
private String dateTimeToStr(DateTimeV2Literal literal, Type partitionColumnType) throws AnalysisException {
274+
if (partitionColumnType.isDate() || partitionColumnType.isDateV2()) {
275+
return String.format(PartitionExprUtil.DATE_FORMATTER, literal.getYear(), literal.getMonth(),
276+
literal.getDay());
277+
} else if (partitionColumnType.isDatetime() || partitionColumnType.isDatetimeV2()
278+
|| partitionColumnType.isTimeStampTz()) {
279+
return String.format(PartitionExprUtil.DATETIME_FORMATTER,
280+
literal.getYear(), literal.getMonth(), literal.getDay(),
281+
literal.getHour(), literal.getMinute(), literal.getSecond());
282+
} else {
283+
throw new AnalysisException(
284+
"MTMV not support partition with column type : " + partitionColumnType);
285+
}
286+
}
287+
288+
private Optional<String> getDateFormat(Map<String, String> mvProperties) {
289+
return StringUtils.isEmpty(mvProperties.get(PropertyAnalyzer.PROPERTIES_PARTITION_DATE_FORMAT))
290+
? Optional.empty()
291+
: Optional.of(mvProperties.get(PropertyAnalyzer.PROPERTIES_PARTITION_DATE_FORMAT));
292+
}
293+
294+
private long parseOffsetHours(LiteralExpr offsetExpr) throws AnalysisException {
295+
try {
296+
return Long.parseLong(offsetExpr.getStringValue());
297+
} catch (NumberFormatException e) {
298+
throw new AnalysisException("date_add/date_sub hour offset should be integer");
299+
}
300+
}
301+
302+
@Override
303+
public boolean equals(Object o) {
304+
if (o == null || getClass() != o.getClass()) {
305+
return false;
306+
}
307+
MTMVPartitionExprDateTruncDateAddSub that = (MTMVPartitionExprDateTruncDateAddSub) o;
308+
return offsetHours == that.offsetHours && Objects.equals(timeUnit, that.timeUnit);
309+
}
310+
311+
@Override
312+
public int hashCode() {
313+
return Objects.hash(timeUnit, offsetHours);
314+
}
315+
}

‎fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionExprFactory.java‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,10 +17,14 @@
1717

1818
package org.apache.doris.mtmv;
1919

20+
import org.apache.doris.analysis.CastExpr;
2021
import org.apache.doris.analysis.Expr;
2122
import org.apache.doris.analysis.FunctionCallExpr;
23+
import org.apache.doris.analysis.TimestampArithmeticExpr;
2224
import org.apache.doris.common.AnalysisException;
2325

26+
import java.util.List;
27+
2428
/**
2529
* MTMV Partition Expr Factory
2630
*/
@@ -32,6 +36,14 @@ public static MTMVPartitionExprService getExprService(Expr expr) throws Analysis
3236
FunctionCallExpr functionCallExpr = (FunctionCallExpr) expr;
3337
String fnName = functionCallExpr.getFnName().getFunction().toLowerCase();
3438
if ("date_trunc".equals(fnName)) {
39+
List<Expr> paramsExprs = functionCallExpr.getParams().exprs();
40+
Expr dateTruncArg = paramsExprs.size() >= 1 ? paramsExprs.get(0) : null;
41+
while (dateTruncArg instanceof CastExpr) {
42+
dateTruncArg = dateTruncArg.getChild(0);
43+
}
44+
if (paramsExprs.size() == 2 && dateTruncArg instanceof TimestampArithmeticExpr) {
45+
return new MTMVPartitionExprDateTruncDateAddSub(functionCallExpr);
46+
}
3547
return new MTMVPartitionExprDateTrunc(functionCallExpr);
3648
}
3749
throw new AnalysisException("async materialized view partition not support function name: " + fnName);

‎fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionExprService.java‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,8 @@
2020
import org.apache.doris.analysis.PartitionKeyDesc;
2121
import org.apache.doris.common.AnalysisException;
2222

23+
import java.util.Collections;
24+
import java.util.List;
2325
import java.util.Map;
2426

2527
/**
@@ -50,6 +52,19 @@ PartitionKeyDesc generateRollUpPartitionKeyDesc(
5052
PartitionKeyDesc partitionKeyDesc, MTMVPartitionInfo mvPartitionInfo, MTMVRelatedTableIf pctTable)
5153
throws AnalysisException;
5254

55+
/**
56+
* For range partition, a single base range partition may overlap multiple roll-up buckets
57+
* (e.g. {@code date_trunc(date_add(col, INTERVAL 3 HOUR), 'day')} on UTC-midnight base partitions),
58+
* so return all roll-up PartitionKeyDesc values that should be associated with the input range.
59+
*
60+
* <p>Default implementation keeps the historical 1-to-1 behavior.
61+
*/
62+
default List<PartitionKeyDesc> generateRollUpPartitionKeyDescs(
63+
PartitionKeyDesc partitionKeyDesc, MTMVPartitionInfo mvPartitionInfo, MTMVRelatedTableIf pctTable)
64+
throws AnalysisException {
65+
return Collections.singletonList(generateRollUpPartitionKeyDesc(partitionKeyDesc, mvPartitionInfo, pctTable));
66+
}
67+
5368
/**
5469
* Check if user input is legal
5570
*

‎fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescRollUpGenerator.java‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -143,9 +143,11 @@ public Map<PartitionKeyDesc, Set<String>> rollUpRange(Map<PartitionKeyDesc, Set<
143143
Map<PartitionKeyDesc, Set<String>> result = Maps.newHashMap();
144144
MTMVPartitionExprService exprSerice = MTMVPartitionExprFactory.getExprService(mvPartitionInfo.getExpr());
145145
for (Entry<PartitionKeyDesc, Set<String>> entry : relatedPartitionDescs.entrySet()) {
146-
PartitionKeyDesc rollUpDesc = exprSerice.generateRollUpPartitionKeyDesc(entry.getKey(), mvPartitionInfo,
147-
pctTable);
148-
result.computeIfAbsent(rollUpDesc, k -> Sets.newHashSet()).addAll(entry.getValue());
146+
List<PartitionKeyDesc> rollUpDescs = exprSerice.generateRollUpPartitionKeyDescs(entry.getKey(),
147+
mvPartitionInfo, pctTable);
148+
for (PartitionKeyDesc rollUpDesc : rollUpDescs) {
149+
result.computeIfAbsent(rollUpDesc, k -> Sets.newHashSet()).addAll(entry.getValue());
150+
}
149151
}
150152
return result;
151153
}

0 commit comments

Comments
 (0)