Skip to content

Commit 17a1ce5

Browse files
authored
[fix](nereids) add a project node above sort node to eliminate unused order by keys (#17913)
if the order by keys are not simple slot in sort node, the order by exprs have to been added to sort node's output tuple. In that case, we need add a project node above sort node to eliminate the unused order by exprs. for example: ```sql WITH t0 AS (SELECT DATE_FORMAT(date, '%Y%m%d') AS date FROM cir_1756_t1 ), t3 AS (SELECT date_format(date, '%Y%m%d') AS `date` FROM `cir_1756_t2` GROUP BY date_format(date, '%Y%m%d') **ORDER BY date_format(date, '%Y%m%d')** ) SELECT t0.date FROM t0 LEFT JOIN t3 ON t0.date = t3.date; ``` before: ``` +--------------------------------------------------------------------------------------------------------------------------------------------------+ | Explain String | +--------------------------------------------------------------------------------------------------------------------------------------------------+ | LogicalProject[159] ( distinct=false, projects=[date#1], excepts=[], canEliminate=true ) | | +--LogicalJoin[158] ( type=LEFT_OUTER_JOIN, markJoinSlotReference=Optional.empty, hashJoinConjuncts=[(date#1 = date#3)], otherJoinConjuncts=[] ) | | |--LogicalProject[151] ( distinct=false, projects=[date_format(date#0, '%Y%m%d') AS `date`#1], excepts=[], canEliminate=true ) | | | +--LogicalOlapScan ( qualified=default_cluster:bugfix.cir_1756_t1, indexName=cir_1756_t1, selectedIndexId=412339, preAgg=ON ) | | +--LogicalSort[157] ( orderKeys=[date_format(cast(date#3 as DATETIME), '%Y%m%d') asc null first] ) | | +--LogicalAggregate[156] ( groupByExpr=[date#3], outputExpr=[date#3], hasRepeat=false ) | | +--LogicalProject[155] ( distinct=false, projects=[date_format(date#2, '%Y%m%d') AS `date`#3], excepts=[], canEliminate=true ) | | +--LogicalOlapScan ( qualified=default_cluster:bugfix.cir_1756_t2, indexName=cir_1756_t2, selectedIndexId=412352, preAgg=ON ) | +--------------------------------------------------------------------------------------------------------------------------------------------------+ ``` after: ``` +--------------------------------------------------------------------------------------------------------------------------------------------------+ | Explain String | +--------------------------------------------------------------------------------------------------------------------------------------------------+ | LogicalProject[171] ( distinct=false, projects=[date#2], excepts=[], canEliminate=true ) | | +--LogicalJoin[170] ( type=LEFT_OUTER_JOIN, markJoinSlotReference=Optional.empty, hashJoinConjuncts=[(date#2 = date#4)], otherJoinConjuncts=[] ) | | |--LogicalProject[162] ( distinct=false, projects=[date_format(date#0, '%Y%m%d') AS `date`#2], excepts=[], canEliminate=true ) | | | +--LogicalOlapScan ( qualified=default_cluster:bugfix.cir_1756_t1, indexName=cir_1756_t1, selectedIndexId=1049812, preAgg=ON ) | | +--LogicalProject[169] ( distinct=false, projects=[date#4], excepts=[], canEliminate=false ) | | +--LogicalSort[168] ( orderKeys=[date_format(cast(date#4 as DATETIME), '%Y%m%d') asc null first] ) | | +--LogicalAggregate[167] ( groupByExpr=[date#4], outputExpr=[date#4], hasRepeat=false ) | | +--LogicalProject[166] ( distinct=false, projects=[date_format(date#3, '%Y%m%d') AS `date`#4], excepts=[], canEliminate=true ) | | +--LogicalOlapScan ( qualified=default_cluster:bugfix.cir_1756_t2, indexName=cir_1756_t2, selectedIndexId=1049825, preAgg=ON ) | +--------------------------------------------------------------------------------------------------------------------------------------------------+ ```
1 parent 6cbf393 commit 17a1ce5

16 files changed

Lines changed: 182 additions & 22 deletions

‎fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -133,6 +133,7 @@ public enum RuleType {
133133
COLUMN_PRUNE_FILTER_CHILD(RuleTypeClass.REWRITE),
134134
PRUNE_ONE_ROW_RELATION_COLUMN(RuleTypeClass.REWRITE),
135135
COLUMN_PRUNE_SORT_CHILD(RuleTypeClass.REWRITE),
136+
COLUMN_PRUNE_SORT(RuleTypeClass.REWRITE),
136137
COLUMN_PRUNE_JOIN_CHILD(RuleTypeClass.REWRITE),
137138
COLUMN_PRUNE_REPEAT_CHILD(RuleTypeClass.REWRITE),
138139
// expression of plan rewrite

‎fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rewrite/ExpressionRewrite.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -130,7 +130,7 @@ public Rule build() {
130130
if (projects.equals(newProjects)) {
131131
return project;
132132
}
133-
return new LogicalProject<>(newProjects, project.child());
133+
return project.withProjectsAndChild(newProjects, project.child());
134134
}).toRule(RuleType.REWRITE_PROJECT_EXPRESSION);
135135
}
136136
}

‎fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/ColumnPruning.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@ public List<Rule> buildRules() {
3535
new PruneFilterChildColumns().build(),
3636
new PruneAggChildColumns().build(),
3737
new PruneJoinChildrenColumns().build(),
38+
new PruneSortColumns().build(),
3839
new PruneSortChildColumns().build(),
3940
new MergeProjects().build(),
4041
new PruneRepeatChildColumns().build()

‎fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/ExtractAndNormalizeWindowExpression.java‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,6 @@
2828
import org.apache.doris.nereids.trees.expressions.functions.agg.AggregateFunction;
2929
import org.apache.doris.nereids.trees.plans.Plan;
3030
import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate;
31-
import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
3231
import org.apache.doris.nereids.trees.plans.logical.LogicalWindow;
3332
import org.apache.doris.nereids.util.ExpressionUtils;
3433

@@ -65,10 +64,11 @@ public Rule build() {
6564
boolean needAggregate = bottomProjects.stream().anyMatch(expr ->
6665
expr.anyMatch(AggregateFunction.class::isInstance));
6766
if (needAggregate) {
68-
normalizedChild = new LogicalAggregate<>(
69-
ImmutableList.of(), ImmutableList.copyOf(bottomProjects), project.child());
67+
normalizedChild = new LogicalAggregate<>(ImmutableList.of(),
68+
ImmutableList.copyOf(bottomProjects), project.child());
7069
} else {
71-
normalizedChild = new LogicalProject<>(ImmutableList.copyOf(bottomProjects), project.child());
70+
normalizedChild = project.withProjectsAndChild(
71+
ImmutableList.copyOf(bottomProjects), project.child());
7272
}
7373
}
7474

@@ -89,7 +89,7 @@ public Rule build() {
8989

9090
// 3. handle top projects
9191
List<NamedExpression> topProjects = ctxForWindows.normalizeToUseSlotRef(normalizedOutputs1);
92-
return new LogicalProject<>(topProjects, normalizedLogicalWindow);
92+
return project.withProjectsAndChild(topProjects, normalizedLogicalWindow);
9393
}).toRule(RuleType.EXTRACT_AND_NORMALIZE_WINDOW_EXPRESSIONS);
9494
}
9595

‎fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/MergeProjects.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,6 @@
2121
import org.apache.doris.nereids.rules.RuleType;
2222
import org.apache.doris.nereids.rules.rewrite.OneRewriteRuleFactory;
2323
import org.apache.doris.nereids.trees.expressions.NamedExpression;
24-
import org.apache.doris.nereids.trees.plans.Plan;
2524
import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
2625

2726
import java.util.List;
@@ -47,9 +46,10 @@ public class MergeProjects extends OneRewriteRuleFactory {
4746
@Override
4847
public Rule build() {
4948
return logicalProject(logicalProject()).then(project -> {
50-
LogicalProject<Plan> childProject = project.child();
49+
LogicalProject childProject = project.child();
5150
List<NamedExpression> projectExpressions = project.mergeProjections(childProject);
52-
return new LogicalProject<>(projectExpressions, childProject.child(0));
51+
LogicalProject newProject = childProject.canEliminate() ? project : childProject;
52+
return newProject.withProjectsAndChild(projectExpressions, childProject.child(0));
5353
}).toRule(RuleType.MERGE_PROJECTS);
5454
}
5555
}
Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
package org.apache.doris.nereids.rules.rewrite.logical;
19+
20+
import org.apache.doris.nereids.rules.Rule;
21+
import org.apache.doris.nereids.rules.RuleType;
22+
import org.apache.doris.nereids.rules.rewrite.OneRewriteRuleFactory;
23+
import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
24+
25+
import com.google.common.collect.ImmutableList;
26+
27+
import java.util.stream.Collectors;
28+
29+
/**
30+
the sort node will create new slots for order by keys if the order by keys is not in the output
31+
so need create a project above sort node to prune the unnecessary order by keys
32+
*/
33+
public class PruneSortColumns extends OneRewriteRuleFactory {
34+
@Override
35+
public Rule build() {
36+
return logicalSort()
37+
.when(sort -> !sort.isOrderKeysPruned() && !sort.getOutputSet()
38+
.containsAll(sort.getOrderKeys().stream()
39+
.map(orderKey -> orderKey.getExpr()).collect(Collectors.toSet())))
40+
.then(sort -> {
41+
return new LogicalProject(sort.getOutput(), ImmutableList.of(), false,
42+
sort.withOrderKeysPruned(true));
43+
}).toRule(RuleType.COLUMN_PRUNE_SORT);
44+
}
45+
}

‎fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PullUpCorrelatedFilterUnderApplyAggregateProject.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -75,7 +75,7 @@ public Rule build() {
7575
}
7676
});
7777

78-
LogicalProject newProject = new LogicalProject<>(newProjects, filter.child());
78+
LogicalProject newProject = project.withProjectsAndChild(newProjects, filter.child());
7979
LogicalFilter newFilter = new LogicalFilter<>(filter.getConjuncts(), newProject);
8080
LogicalAggregate newAgg = agg.withChildren(ImmutableList.of(newFilter));
8181
return new LogicalApply<>(apply.getCorrelationSlot(), apply.getSubqueryExpr(),

‎fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PullUpProjectUnderApply.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -64,7 +64,7 @@ public Rule build() {
6464
if (apply.getSubqueryExpr() instanceof ScalarSubquery) {
6565
newProjects.add(project.getProjects().get(0));
6666
}
67-
return new LogicalProject(newProjects, newCorrelate);
67+
return project.withProjectsAndChild(newProjects, newCorrelate);
6868
}).toRule(RuleType.PULL_UP_PROJECT_UNDER_APPLY);
6969
}
7070
}

‎fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownAliasThroughJoin.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -90,12 +90,12 @@ public Rule build() {
9090
if (leftOutput.equals(leftProjects)) {
9191
left = join.left();
9292
} else {
93-
left = new LogicalProject<>(leftProjects, join.left());
93+
left = project.withProjectsAndChild(leftProjects, join.left());
9494
}
9595
if (rightOutput.equals(rightProjects)) {
9696
right = join.right();
9797
} else {
98-
right = new LogicalProject<>(rightProjects, join.right());
98+
right = project.withProjectsAndChild(rightProjects, join.right());
9999
}
100100

101101
// If condition use alias slot, we should replace condition
@@ -109,7 +109,7 @@ public Rule build() {
109109
List<Expression> newOther = replaceJoinConjuncts(join.getOtherJoinConjuncts(), replaceMap);
110110

111111
Plan newJoin = join.withConjunctsChildren(newHash, newOther, left, right);
112-
return new LogicalProject<>(newProjects, newJoin);
112+
return project.withProjectsAndChild(newProjects, newJoin);
113113
}).toRule(RuleType.PUSHDOWN_ALIAS_THROUGH_JOIN);
114114
}
115115

‎fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownFilterThroughProject.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@ public List<Rule> buildRules() {
4444
RuleType.PUSHDOWN_FILTER_THROUGH_PROJECT.build(
4545
logicalFilter(logicalProject()).then(filter -> {
4646
LogicalProject<Plan> project = filter.child();
47-
return new LogicalProject<>(
47+
return project.withProjectsAndChild(
4848
project.getProjects(),
4949
new LogicalFilter<>(
5050
ExpressionUtils.replace(filter.getConjuncts(), project.getAliasToProducer()),
@@ -60,7 +60,7 @@ public List<Rule> buildRules() {
6060
LogicalLimit<LogicalProject<Plan>> limit = filter.child();
6161
LogicalProject<Plan> project = limit.child();
6262

63-
return new LogicalProject<>(
63+
return project.withProjectsAndChild(
6464
project.getProjects(),
6565
new LogicalFilter<>(
6666
ExpressionUtils.replace(filter.getConjuncts(), project.getAliasToProducer()),

0 commit comments

Comments
 (0)