Skip to content

Commit 19dfb87

Browse files
authored
[fix](partition prunning) Correct partition pruning for calendar arithmetic (#69828)
### What problem does this PR solve? Issue Number: None Related PR: None Problem Summary: Calendar month, quarter, year and YEAR_MONTH additions and subtractions clamp dates at month end while preserving the time of day. Consecutive input dates can therefore produce decreasing datetime values. Treating these functions as globally monotonic lets RANGE partition pruning remove partitions containing matching rows, such as January 29 filtered by months_add(datetime_column, 1) < '2023-02-28 10:00:00'. Share a calendar arithmetic monotonicity trait that retains the optimization for DATE and single-day ranges and keeps other datetime ranges conservatively. This also covers DATE_ADD, DATE_SUB, TIMESTAMPADD, INTERVAL expressions and runtime-filter classification. MONTHS_DIFF, QUARTERS_DIFF and YEARS_DIFF also fold some month-end datetime arguments differently from BE because LocalDateTime.until adjusts the end date before counting calendar units. Count date units first, then compare time parts only when the remaining date fields match, preserving BE's truncation semantics for both argument orders and all supported date-like literal types. ### Release note Fix missing query rows caused by calendar arithmetic partition pruning and align month, quarter and year difference constant folding with runtime results. ### Check List (For Author) - Test: FE build with Checkstyle; 94 FE unit tests; regression tests - Unit Test: CalendarAddSubMonotonicTest, DateTimeExtractAndTransformTest, PartitionPrunerTest, RuntimeFilterPruneClassifierTest, TimestampNsMonotonicityTest and InferPredicateFromMonotonicFunctionTest - Regression test: test_calendar_arithmetic_prune (73 generated result checks) and test_date_function_prune; verify the new suite fails on the original FE - Behavior changed: Yes, retain matching rows and correct calendar difference constants - Does this need documentation: No ### What problem does this PR solve? Issue Number: close #xxx Related PR: #xxx Problem Summary: ### Release note None ### Check List (For Author) - Test <!-- At least one of them must be included. --> - [ ] Regression test - [ ] Unit Test - [ ] Manual test (add detailed scripts or steps below) - [ ] No need to test or manual test. Explain why: - [ ] This is a refactor/code format and no logic has been changed. - [ ] Previous test can cover this change. - [ ] No code files have been changed. - [ ] Other reason <!-- Add your reason? --> - Behavior changed: - [ ] No. - [ ] Yes. <!-- Explain the behavior change --> - Does this need documentation? - [ ] No. - [ ] Yes. <!-- Add document PR link here. eg: apache/doris-website#1214 --> ### Check List (For Reviewer who merge this PR) - [ ] Confirm the release note - [ ] Confirm test cases - [ ] Confirm document - [ ] Add branch pick label <!-- Add branch pick label that this PR should merge into -->
1 parent 8e499c0 commit 19dfb87

19 files changed

Lines changed: 1102 additions & 91 deletions

File tree

‎fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/OneRangePartitionEvaluator.java‎

Lines changed: 19 additions & 53 deletions
Original file line numberDiff line numberDiff line change
@@ -40,13 +40,12 @@
4040
import org.apache.doris.nereids.trees.expressions.NullSafeEqual;
4141
import org.apache.doris.nereids.trees.expressions.Or;
4242
import org.apache.doris.nereids.trees.expressions.Slot;
43-
import org.apache.doris.nereids.trees.expressions.functions.Monotonic;
43+
import org.apache.doris.nereids.trees.expressions.functions.RangeInferable;
4444
import org.apache.doris.nereids.trees.expressions.functions.scalar.NonNullable;
4545
import org.apache.doris.nereids.trees.expressions.functions.scalar.Nullable;
4646
import org.apache.doris.nereids.trees.expressions.literal.BooleanLiteral;
4747
import org.apache.doris.nereids.trees.expressions.literal.Literal;
4848
import org.apache.doris.nereids.trees.expressions.literal.MaxLiteral;
49-
import org.apache.doris.nereids.trees.expressions.literal.NullLiteral;
5049
import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
5150
import org.apache.doris.nereids.types.BooleanType;
5251
import org.apache.doris.nereids.util.ExpressionUtils;
@@ -69,6 +68,7 @@
6968
import java.util.Map;
7069
import java.util.Map.Entry;
7170
import java.util.Objects;
71+
import java.util.Optional;
7272
import java.util.Set;
7373
import java.util.function.BiFunction;
7474

@@ -532,8 +532,8 @@ private EvaluateRangeResult evaluateChildrenThenThis(Expression expr, EvaluateRa
532532
for (int i = 0; i < children.size(); i++) {
533533
Expression child = children.get(i);
534534
EvaluateRangeResult childResult;
535-
if (child instanceof Monotonic) {
536-
childResult = visitMonotonic(child, context);
535+
if (child instanceof RangeInferable) {
536+
childResult = visitRangeInferable(child, context);
537537
} else {
538538
childResult = child.accept(this, context);
539539
}
@@ -1005,21 +1005,21 @@ private void diverge() {
10051005
}
10061006
}
10071007

1008-
public EvaluateRangeResult visitMonotonic(Expression monotonic, EvaluateRangeInput context) {
1009-
EvaluateRangeResult rangeResult = evaluateChildrenThenThis(monotonic, context);
1010-
if (!rangeResult.result.getClass().equals(monotonic.getClass())) {
1008+
public EvaluateRangeResult visitRangeInferable(Expression expression, EvaluateRangeInput context) {
1009+
EvaluateRangeResult rangeResult = evaluateChildrenThenThis(expression, context);
1010+
if (!rangeResult.result.getClass().equals(expression.getClass())) {
10111011
return rangeResult;
10121012
}
1013-
return computeMonotonicFunctionRange(rangeResult, context);
1013+
return computeFunctionRange(rangeResult, context);
10141014
}
10151015

1016-
private EvaluateRangeResult computeMonotonicFunctionRange(EvaluateRangeResult result, EvaluateRangeInput context) {
1017-
Monotonic func = (Monotonic) result.result;
1016+
private EvaluateRangeResult computeFunctionRange(EvaluateRangeResult result, EvaluateRangeInput context) {
1017+
RangeInferable func = (RangeInferable) result.result;
10181018
if (context.rangeMap.containsKey(func)) {
10191019
return new EvaluateRangeResult((Expression) func, ImmutableMap.of((Expression) func,
10201020
context.rangeMap.get(func)), result.childrenResult);
10211021
}
1022-
int childIndex = func.getMonotonicFunctionChildIndex();
1022+
int childIndex = func.getRangeInputChildIndex();
10231023
Expression funcChild = func.child(childIndex);
10241024
boolean isNullable = partitionSlotContainsNull.getOrDefault(funcChild, true);
10251025
Expression withNullable = func.withConstantArgs(isNullable ? new Nullable(funcChild)
@@ -1039,51 +1039,17 @@ private EvaluateRangeResult computeMonotonicFunctionRange(EvaluateRangeResult re
10391039
Literal lower = span.hasLowerBound() ? span.lowerEndpoint().getValue() : null;
10401040
Literal upper = span.hasUpperBound() && !(span.upperEndpoint().getValue() instanceof MaxLiteral)
10411041
? span.upperEndpoint().getValue() : null;
1042-
if (!func.isMonotonic(lower, upper)) {
1042+
Optional<ColumnRange> inferredRange = func.inferRange(lower, upper, expressionRewriteContext);
1043+
if (!inferredRange.isPresent()) {
10431044
return result;
10441045
}
1045-
Expression lowerValue = lower != null ? FoldConstantRuleOnFE.evaluate(func.withConstantArgs(lower),
1046-
expressionRewriteContext) : null;
1047-
Expression upperValue = upper != null ? FoldConstantRuleOnFE.evaluate(func.withConstantArgs(upper),
1048-
expressionRewriteContext) : null;
1049-
if (!checkFoldConstantValueIsValid(lowerValue, upperValue)) {
1050-
return result;
1051-
}
1052-
1053-
if (!func.isPositive()) {
1054-
Expression temp = lowerValue;
1055-
lowerValue = upperValue;
1056-
upperValue = temp;
1057-
}
1058-
LinkedHashMap<Expression, ColumnRange> newRanges = Maps.newLinkedHashMap();
1059-
ColumnRange newRange = ColumnRange.all();
1060-
if (lowerValue instanceof Literal && upperValue instanceof Literal && lowerValue.equals(upperValue)) {
1061-
newRange = ColumnRange.singleton((Literal) lowerValue);
1062-
context.rangeMap.put((Expression) func, newRange);
1063-
newRanges.put((Expression) func, newRange);
1064-
return new EvaluateRangeResult(lowerValue, newRanges, result.childrenResult);
1065-
} else {
1066-
if (lowerValue instanceof Literal) {
1067-
newRange = newRange.withLowerBound((Literal) lowerValue);
1068-
}
1069-
if (upperValue instanceof Literal) {
1070-
newRange = newRange.withUpperBound((Literal) upperValue);
1071-
}
1072-
if (newRange.isEmptyRange() || !newRange.span().hasLowerBound() && !newRange.span().hasUpperBound()) {
1073-
return result;
1074-
}
1075-
context.rangeMap.put((Expression) func, newRange);
1076-
newRanges.put((Expression) func, newRange);
1077-
return new EvaluateRangeResult((Expression) func, newRanges, result.childrenResult);
1078-
}
1079-
}
10801046

1081-
// only allow literal(except NullLiteral) and null
1082-
private boolean checkFoldConstantValueIsValid(Expression lowerValue, Expression upperValue) {
1083-
if (lowerValue instanceof NullLiteral || upperValue instanceof NullLiteral) {
1084-
return false;
1047+
ColumnRange newRange = inferredRange.get();
1048+
context.rangeMap.put((Expression) func, newRange);
1049+
Map<Expression, ColumnRange> newRanges = ImmutableMap.of((Expression) func, newRange);
1050+
if (newRange.isSingleton()) {
1051+
return new EvaluateRangeResult(newRange.getLowerBound().getValue(), newRanges, result.childrenResult);
10851052
}
1086-
return (lowerValue == null || lowerValue instanceof Literal)
1087-
&& (upperValue == null || upperValue instanceof Literal);
1053+
return new EvaluateRangeResult((Expression) func, newRanges, result.childrenResult);
10881054
}
10891055
}
Lines changed: 87 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,87 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
package org.apache.doris.nereids.trees.expressions.functions;
19+
20+
import org.apache.doris.nereids.rules.expression.ExpressionRewriteContext;
21+
import org.apache.doris.nereids.rules.expression.rules.ColumnRange;
22+
import org.apache.doris.nereids.trees.expressions.literal.DateLiteral;
23+
import org.apache.doris.nereids.trees.expressions.literal.DateTimeV2Literal;
24+
import org.apache.doris.nereids.trees.expressions.literal.Literal;
25+
import org.apache.doris.nereids.trees.expressions.literal.TimeStampNsLiteral;
26+
import org.apache.doris.nereids.trees.expressions.literal.TimestampTzLiteral;
27+
28+
import java.time.LocalDateTime;
29+
import java.time.LocalTime;
30+
import java.util.Optional;
31+
32+
/** Monotonicity of date arithmetic that can clamp the day to the end of a month. */
33+
public interface CalendarAddSubMonotonic extends DateAddSubMonotonic {
34+
@Override
35+
default boolean isMonotonic(Literal lower, Literal upper) {
36+
// Clamping consecutive dates to the same date resets the time at midnight. DATE has no
37+
// time part, and within one input day the calendar adjustment preserves the time order.
38+
return canEvaluateRange()
39+
&& (child(0).getDataType().isDateV2Type() || MonotonicityUtils.isWithinSameDay(lower, upper));
40+
}
41+
42+
/** The date part remains non-decreasing even when clamping resets the time part. */
43+
default boolean canEvaluateRange() {
44+
return MonotonicityUtils.hasConstantOtherArguments(this, 0);
45+
}
46+
47+
@Override
48+
default Optional<ColumnRange> inferRange(Literal lower, Literal upper, ExpressionRewriteContext context) {
49+
if (!canEvaluateRange()) {
50+
return Optional.empty();
51+
}
52+
boolean monotonic = isMonotonic(lower, upper);
53+
return evaluateRangeEndpoints(lower, upper, context).flatMap(bounds -> RangeInferable.createRange(
54+
monotonic || lower == null ? bounds.first
55+
: getRangeLowerBound((DateLiteral) lower, (DateLiteral) bounds.first),
56+
monotonic || upper == null ? bounds.second
57+
: getRangeUpperBound((DateLiteral) upper, (DateLiteral) bounds.second)));
58+
}
59+
60+
/** Later input days can map to this output date with an earlier time. */
61+
default Literal getRangeLowerBound(DateLiteral input, DateLiteral output) {
62+
if (input.getDay() < input.toJavaDateType().toLocalDate().lengthOfMonth()
63+
&& output.getDay() == output.toJavaDateType().toLocalDate().lengthOfMonth()) {
64+
return withTime(output, LocalTime.MIN);
65+
}
66+
return output;
67+
}
68+
69+
/** Earlier input days can map to a clamped output date with a later time. */
70+
default Literal getRangeUpperBound(DateLiteral input, DateLiteral output) {
71+
if (output.getDay() < input.getDay()) {
72+
return withTime(output, LocalTime.MAX);
73+
}
74+
return output;
75+
}
76+
77+
private static Literal withTime(DateLiteral output, LocalTime time) {
78+
LocalDateTime dateTime = output.toJavaDateType().with(time);
79+
if (output instanceof TimeStampNsLiteral) {
80+
return TimeStampNsLiteral.fromJavaDateType(dateTime);
81+
}
82+
if (output instanceof TimestampTzLiteral) {
83+
return TimestampTzLiteral.fromJavaDateType(dateTime, ((TimestampTzLiteral) output).getScale());
84+
}
85+
return DateTimeV2Literal.fromJavaDateType(dateTime, ((DateTimeV2Literal) output).getDataType().getScale());
86+
}
87+
}

‎fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/Monotonic.java‎

Lines changed: 19 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -17,11 +17,14 @@
1717

1818
package org.apache.doris.nereids.trees.expressions.functions;
1919

20-
import org.apache.doris.nereids.trees.expressions.Expression;
20+
import org.apache.doris.nereids.rules.expression.ExpressionRewriteContext;
21+
import org.apache.doris.nereids.rules.expression.rules.ColumnRange;
2122
import org.apache.doris.nereids.trees.expressions.literal.Literal;
2223

24+
import java.util.Optional;
25+
2326
/** monotonicity of expressions */
24-
public interface Monotonic extends ExpressionTrait {
27+
public interface Monotonic extends RangeInferable {
2528
default boolean isMonotonic(Literal lower, Literal upper) {
2629
return true;
2730
}
@@ -33,8 +36,18 @@ default boolean isMonotonic(Literal lower, Literal upper) {
3336
// e.g. date_trunc(dt,'xxx') return 0
3437
int getMonotonicFunctionChildIndex();
3538

36-
// return the function with the arguments replaced by literal
37-
// e.g. date_trunc(dt, 'day'), dt in range ['2020-01-01 10:00:00', '2020-01-03 10:00:00']
38-
// return date_trunc('2020-01-01 10:00:00', 'day')
39-
Expression withConstantArgs(Expression literal);
39+
@Override
40+
default int getRangeInputChildIndex() {
41+
return getMonotonicFunctionChildIndex();
42+
}
43+
44+
@Override
45+
default Optional<ColumnRange> inferRange(Literal lower, Literal upper, ExpressionRewriteContext context) {
46+
if (!isMonotonic(lower, upper)) {
47+
return Optional.empty();
48+
}
49+
return evaluateRangeEndpoints(lower, upper, context).flatMap(bounds -> isPositive()
50+
? RangeInferable.createRange(bounds.first, bounds.second)
51+
: RangeInferable.createRange(bounds.second, bounds.first));
52+
}
4053
}
Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,72 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
package org.apache.doris.nereids.trees.expressions.functions;
19+
20+
import org.apache.doris.common.Pair;
21+
import org.apache.doris.nereids.rules.expression.ExpressionRewriteContext;
22+
import org.apache.doris.nereids.rules.expression.rules.ColumnRange;
23+
import org.apache.doris.nereids.rules.expression.rules.FoldConstantRuleOnFE;
24+
import org.apache.doris.nereids.trees.expressions.Expression;
25+
import org.apache.doris.nereids.trees.expressions.literal.Literal;
26+
import org.apache.doris.nereids.trees.expressions.literal.NullLiteral;
27+
28+
import java.util.Optional;
29+
30+
/** Infer an enclosing result range without requiring the expression to be monotonic. */
31+
public interface RangeInferable extends ExpressionTrait {
32+
int getRangeInputChildIndex();
33+
34+
// Replace the range input with a constant while preserving the other arguments.
35+
Expression withConstantArgs(Expression literal);
36+
37+
/**
38+
* Return a range containing every possible result, or empty when no safe range can be inferred.
39+
* A null lower or upper endpoint means that the input is unbounded on that side.
40+
*/
41+
Optional<ColumnRange> inferRange(Literal lower, Literal upper, ExpressionRewriteContext context);
42+
43+
/** Fold the input endpoints, retaining null for an unbounded side, and reject SQL NULL or unfolded results. */
44+
default Optional<Pair<Literal, Literal>> evaluateRangeEndpoints(
45+
Literal lower, Literal upper, ExpressionRewriteContext context) {
46+
Expression lowerValue = lower == null ? null
47+
: FoldConstantRuleOnFE.evaluate(withConstantArgs(lower), context);
48+
Expression upperValue = upper == null ? null
49+
: FoldConstantRuleOnFE.evaluate(withConstantArgs(upper), context);
50+
if (lowerValue instanceof NullLiteral || upperValue instanceof NullLiteral
51+
|| lowerValue != null && !(lowerValue instanceof Literal)
52+
|| upperValue != null && !(upperValue instanceof Literal)) {
53+
return Optional.empty();
54+
}
55+
return Optional.of(Pair.of((Literal) lowerValue, (Literal) upperValue));
56+
}
57+
58+
/** Build inclusive result bounds. Input endpoint exclusions need not hold after applying a function. */
59+
static Optional<ColumnRange> createRange(Literal lower, Literal upper) {
60+
ColumnRange range = ColumnRange.all();
61+
if (lower != null) {
62+
range = range.withLowerBound(lower);
63+
}
64+
if (upper != null) {
65+
range = range.withUpperBound(upper);
66+
}
67+
if (range.isEmptyRange() || !range.span().hasLowerBound() && !range.span().hasUpperBound()) {
68+
return Optional.empty();
69+
}
70+
return Optional.of(range);
71+
}
72+
}

0 commit comments

Comments
 (0)