Skip to content

Commit 59a46a7

Browse files
authored
[opt](Nereids) partition predicate large or to min-max (apache#19)
* rewrite large In to min/max for partition prune
1 parent 818f9d7 commit 59a46a7

5 files changed

Lines changed: 180 additions & 0 deletions

File tree

‎fe/fe-core/src/main/java/org/apache/doris/analysis/DateLiteral.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1827,4 +1827,8 @@ private static int getTimeZoneSplitPos(String arg) {
18271827
}
18281828
return split + 1;
18291829
}
1830+
1831+
public double getDouble() {
1832+
return (year * 10000 + month * 100 + day) * 1000000L + hour * 10000 + minute * 100 + second;
1833+
}
18301834
}

‎fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/PartitionPruner.java‎

Lines changed: 88 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,9 +17,11 @@
1717

1818
package org.apache.doris.nereids.rules.expression.rules;
1919

20+
import org.apache.doris.analysis.LiteralExpr;
2021
import org.apache.doris.catalog.ListPartitionItem;
2122
import org.apache.doris.catalog.PartitionInfo;
2223
import org.apache.doris.catalog.PartitionItem;
24+
import org.apache.doris.catalog.PartitionKey;
2325
import org.apache.doris.catalog.RangePartitionItem;
2426
import org.apache.doris.nereids.CascadesContext;
2527
import org.apache.doris.nereids.trees.expressions.Cast;
@@ -43,6 +45,7 @@
4345
/**
4446
* PartitionPruner
4547
*/
48+
@Data
4649
public class PartitionPruner extends DefaultExpressionRewriter<Void> {
4750
private final List<OnePartitionEvaluator> partitions;
4851
private final Expression partitionPredicate;
@@ -115,6 +118,12 @@ public static List<Long> prune(List<Slot> partitionSlots, Expression partitionPr
115118
PartitionTableType partitionTableType) {
116119
partitionPredicate = TryEliminateUninterestedPredicates.rewrite(
117120
partitionPredicate, ImmutableSet.copyOf(partitionSlots), cascadesContext);
121+
if (cascadesContext.getConnectContext().getSessionVariable().inToMinmaxParitionRewriteThreshold > 0
122+
&& canRewriteToMinMax(idToPartitions)) {
123+
double rangeLength = totalRangeLength(idToPartitions);
124+
partitionPredicate = rewritePartitionPredicateForRangePartition(
125+
partitionPredicate, rangeLength, cascadesContext);
126+
}
118127

119128
List<OnePartitionEvaluator> evaluators = idToPartitions.entrySet()
120129
.stream()
@@ -127,6 +136,85 @@ public static List<Long> prune(List<Slot> partitionSlots, Expression partitionPr
127136
return partitionPruner.prune();
128137
}
129138

139+
private static boolean isRangePartition(Map<Long, PartitionItem> idToPartitions) {
140+
if (idToPartitions.isEmpty()) {
141+
return false;
142+
}
143+
PartitionItem item = idToPartitions.values().iterator().next();
144+
return item instanceof RangePartitionItem;
145+
}
146+
147+
private static boolean canRewriteToMinMax(Map<Long, PartitionItem> idToPartitions) {
148+
if (isRangePartition(idToPartitions)) {
149+
RangePartitionItem item = (RangePartitionItem) idToPartitions.values().iterator().next();
150+
if (item == null) {
151+
return false;
152+
}
153+
PartitionKey k1 = item.getItems().upperEndpoint();
154+
if (k1.getKeys().size() != 1) {
155+
return false; // multi keys or hive keys
156+
}
157+
LiteralExpr bound = k1.getKeys().get(0);
158+
if (bound instanceof org.apache.doris.analysis.DateLiteral) {
159+
return true;
160+
}
161+
}
162+
return false;
163+
}
164+
165+
private static double getUpperBoundFromRangePartitionItem(RangePartitionItem item) {
166+
org.apache.doris.analysis.DateLiteral d1 =
167+
(org.apache.doris.analysis.DateLiteral) item.getItems().upperEndpoint().getKeys().get(0);
168+
return d1.getDouble();
169+
}
170+
171+
private static double getLowerBoundFromRangePartitionItem(RangePartitionItem item) {
172+
org.apache.doris.analysis.DateLiteral d1 =
173+
(org.apache.doris.analysis.DateLiteral) item.getItems().lowerEndpoint().getKeys().get(0);
174+
return d1.getDouble();
175+
}
176+
177+
// for range partition, total range is from the end of the first partition to the beginning of the last partition
178+
private static double totalRangeLength(Map<Long, PartitionItem> idToPartitions) {
179+
if (idToPartitions.size() == 1) {
180+
Long id = idToPartitions.keySet().iterator().next();
181+
RangePartitionItem item = (RangePartitionItem) idToPartitions.get(id);
182+
double d1 = getUpperBoundFromRangePartitionItem(item);
183+
double d2 = getLowerBoundFromRangePartitionItem(item);
184+
return Math.max(1, d1 - d2);
185+
}
186+
187+
Long id = idToPartitions.keySet().iterator().next();
188+
RangePartitionItem firstItem = (RangePartitionItem) idToPartitions.get(id);
189+
RangePartitionItem lastItem = firstItem;
190+
double min = getLowerBoundFromRangePartitionItem(firstItem);
191+
double max = getUpperBoundFromRangePartitionItem(lastItem);
192+
for (Long pid : idToPartitions.keySet()) {
193+
RangePartitionItem item = (RangePartitionItem) idToPartitions.get(pid);
194+
double lower = getLowerBoundFromRangePartitionItem(item);
195+
if (lower < min) {
196+
min = lower;
197+
firstItem = item;
198+
} else {
199+
double upper = getUpperBoundFromRangePartitionItem(item);
200+
if (upper > max) {
201+
max = upper;
202+
lastItem = item;
203+
}
204+
}
205+
}
206+
double endOfFirstItem = getUpperBoundFromRangePartitionItem(firstItem);
207+
double beginOfLastItem = getLowerBoundFromRangePartitionItem(lastItem);
208+
return Math.max(1, beginOfLastItem - endOfFirstItem);
209+
}
210+
211+
private static Expression rewritePartitionPredicateForRangePartition(Expression partitionPredicate, double range,
212+
CascadesContext ctx) {
213+
partitionPredicate = OrToIn.INSTANCE.rewrite(partitionPredicate, null);
214+
RewriteRangePartitionPredicate rewriter = new RewriteRangePartitionPredicate(range);
215+
return partitionPredicate.accept(rewriter, ctx);
216+
}
217+
130218
/**
131219
* convert partition item to partition evaluator
132220
*/
Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
package org.apache.doris.nereids.rules.expression.rules;
19+
20+
import org.apache.doris.nereids.CascadesContext;
21+
import org.apache.doris.nereids.trees.expressions.And;
22+
import org.apache.doris.nereids.trees.expressions.Expression;
23+
import org.apache.doris.nereids.trees.expressions.GreaterThanEqual;
24+
import org.apache.doris.nereids.trees.expressions.InPredicate;
25+
import org.apache.doris.nereids.trees.expressions.LessThanEqual;
26+
import org.apache.doris.nereids.trees.expressions.literal.DateLiteral;
27+
import org.apache.doris.nereids.trees.expressions.literal.DateTimeLiteral;
28+
import org.apache.doris.nereids.trees.expressions.literal.Literal;
29+
import org.apache.doris.nereids.trees.expressions.visitor.DefaultExpressionRewriter;
30+
31+
/**
32+
* F = '2022-01-01 00:01:01' or F = 'date2' or ...
33+
* =>
34+
* min_date <= F <= max_date
35+
*/
36+
public class RewriteRangePartitionPredicate extends
37+
DefaultExpressionRewriter<CascadesContext> {
38+
private final double rangeLength;
39+
40+
public RewriteRangePartitionPredicate(double rangeLength) {
41+
this.rangeLength = rangeLength;
42+
}
43+
44+
public static Expression rewrite(Expression expression, double rangeLength, CascadesContext ctx) {
45+
return expression.accept(new RewriteRangePartitionPredicate(rangeLength), ctx);
46+
}
47+
48+
@Override
49+
public Expression visitInPredicate(InPredicate inPredicate, CascadesContext ctx) {
50+
if (inPredicate.getOptions().size() > 10) {
51+
Expression opt0 = inPredicate.getOptions().get(0);
52+
if (opt0 instanceof DateLiteral || opt0 instanceof DateTimeLiteral) {
53+
Literal minOpt = (Literal) inPredicate.getOptions().get(0);
54+
Double minVal = minOpt.getDouble();
55+
Literal maxOpt = (Literal) inPredicate.getOptions().get(0);
56+
Double maxVal = maxOpt.getDouble();
57+
for (int i = 1; i < inPredicate.getOptions().size(); i++) {
58+
Expression opt = inPredicate.getOptions().get(i);
59+
if (!(opt instanceof Literal)) {
60+
return inPredicate;
61+
}
62+
double optValue = ((Literal) opt).getDouble();
63+
if (optValue < minVal) {
64+
minVal = optValue;
65+
minOpt = (Literal) opt;
66+
} else if (optValue > maxVal) {
67+
maxVal = optValue;
68+
maxOpt = (Literal) opt;
69+
}
70+
}
71+
double inToMinMaxThreshold = ctx.getConnectContext()
72+
.getSessionVariable().inToMinmaxParitionRewriteThreshold;
73+
if (((maxVal - minVal) / rangeLength) < inToMinMaxThreshold) {
74+
return new And(
75+
new GreaterThanEqual(inPredicate.getCompareExpr(), minOpt),
76+
new LessThanEqual(inPredicate.getCompareExpr(), maxOpt));
77+
}
78+
}
79+
}
80+
return inPredicate;
81+
}
82+
}

‎fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/TryEliminateUninterestedPredicates.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,9 @@ private TryEliminateUninterestedPredicates(Set<Slot> interestedSlots, CascadesCo
5050
this.expressionRewriteContext = new ExpressionRewriteContext(cascadesContext);
5151
}
5252

53+
/**
54+
* rewrite
55+
*/
5356
public static Expression rewrite(Expression expression, Set<Slot> interestedSlots,
5457
CascadesContext cascadesContext) {
5558
// before eliminate uninterested predicate, we must push down `Not` under CompoundPredicate

‎fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1142,6 +1142,9 @@ public void setMaxJoinNumberOfReorder(int maxJoinNumberOfReorder) {
11421142
"Query largest scan bytes in single BE, Unit bytes"})
11431143
private long queryMaxScanBytes = -1;
11441144

1145+
@VariableMgr.VarAttr(name = "in_to_minmax_parition_rewrite_threshold", needForward = true)
1146+
public double inToMinmaxParitionRewriteThreshold = 0.2;
1147+
11451148
// If this fe is in fuzzy mode, then will use initFuzzyModeVariables to generate some variables,
11461149
// not the default value set in the code.
11471150
public void initFuzzyModeVariables() {

0 commit comments

Comments
 (0)