Skip to content

Commit 5970f98

Browse files
authored
[feature](Nereids): add ColumnPruningPostProcessor. (#32800)
1 parent 49875c3 commit 5970f98

126 files changed

Lines changed: 2329 additions & 2012 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.
Lines changed: 102 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,102 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
package org.apache.doris.nereids.processor.post;
19+
20+
import org.apache.doris.nereids.CascadesContext;
21+
import org.apache.doris.nereids.annotation.DependsRules;
22+
import org.apache.doris.nereids.trees.expressions.NamedExpression;
23+
import org.apache.doris.nereids.trees.expressions.Slot;
24+
import org.apache.doris.nereids.trees.plans.Plan;
25+
import org.apache.doris.nereids.trees.plans.physical.AbstractPhysicalJoin;
26+
import org.apache.doris.nereids.trees.plans.physical.AbstractPhysicalPlan;
27+
import org.apache.doris.nereids.trees.plans.physical.PhysicalDistribute;
28+
import org.apache.doris.nereids.trees.plans.physical.PhysicalProject;
29+
30+
import java.util.ArrayList;
31+
import java.util.List;
32+
import java.util.Set;
33+
import java.util.stream.Collectors;
34+
35+
/**
36+
* Prune column for Join-Cluster
37+
*/
38+
@DependsRules({
39+
MergeProjectPostProcessor.class
40+
})
41+
public class ColumnPruningPostProcessor extends PlanPostProcessor {
42+
@Override
43+
public PhysicalProject visitPhysicalProject(PhysicalProject<? extends Plan> project, CascadesContext ctx) {
44+
Plan child = project.child();
45+
Plan newChild = child.accept(this, ctx);
46+
if (newChild instanceof AbstractPhysicalJoin) {
47+
AbstractPhysicalJoin<? extends Plan, ? extends Plan> join = (AbstractPhysicalJoin) newChild;
48+
Plan left = join.left();
49+
Plan right = join.right();
50+
Set<Slot> leftOutput = left.getOutputSet();
51+
Set<Slot> rightOutput = right.getOutputSet();
52+
53+
Set<Slot> usedSlots = project.getProjects().stream().flatMap(ne -> ne.getInputSlots().stream())
54+
.collect(Collectors.toSet());
55+
usedSlots.addAll(join.getConditionSlot());
56+
57+
List<NamedExpression> leftNewProjections = new ArrayList<>();
58+
List<NamedExpression> rightNewProjections = new ArrayList<>();
59+
60+
for (Slot usedSlot : usedSlots) {
61+
if (leftOutput.contains(usedSlot)) {
62+
leftNewProjections.add(usedSlot);
63+
} else if (rightOutput.contains(usedSlot)) {
64+
rightNewProjections.add(usedSlot);
65+
}
66+
}
67+
68+
Plan newLeft;
69+
if (left instanceof PhysicalDistribute) {
70+
newLeft = leftNewProjections.size() != leftOutput.size() && !leftNewProjections.isEmpty()
71+
? left.withChildren(new PhysicalProject<>(leftNewProjections,
72+
left.getLogicalProperties(), left.child(0)))
73+
: left;
74+
} else {
75+
newLeft = leftNewProjections.size() != leftOutput.size() && !leftNewProjections.isEmpty()
76+
? new PhysicalProject<>(leftNewProjections, left.getLogicalProperties(),
77+
left).copyStatsAndGroupIdFrom((AbstractPhysicalPlan) left)
78+
: left;
79+
}
80+
Plan newRight;
81+
if (right instanceof PhysicalDistribute) {
82+
newRight = rightNewProjections.size() != rightOutput.size() && !rightNewProjections.isEmpty()
83+
? right.withChildren(new PhysicalProject<>(rightNewProjections,
84+
right.getLogicalProperties(), right.child(0)))
85+
: right;
86+
} else {
87+
newRight = rightNewProjections.size() != rightOutput.size() && !rightNewProjections.isEmpty()
88+
? new PhysicalProject<>(rightNewProjections, right.getLogicalProperties(),
89+
right).copyStatsAndGroupIdFrom((AbstractPhysicalPlan) right)
90+
: right;
91+
}
92+
93+
if (newLeft != left || newRight != right) {
94+
return (PhysicalProject) project.withChildren(join.withChildren(newLeft, newRight));
95+
} else {
96+
return project;
97+
}
98+
}
99+
return project;
100+
}
101+
}
102+

‎fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/PlanPostProcessors.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,7 @@ public List<PlanPostProcessor> getProcessors() {
5959
// add processor if we need
6060
Builder<PlanPostProcessor> builder = ImmutableList.builder();
6161
builder.add(new PushDownFilterThroughProject());
62+
builder.add(new ColumnPruningPostProcessor());
6263
builder.add(new MergeProjectPostProcessor());
6364
builder.add(new RecomputeLogicalPropertiesProcessor());
6465
builder.add(new AddOffsetIntoDistribute());

‎fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/AbstractPhysicalPlan.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -149,7 +149,7 @@ public Plan getExplainPlan(ConnectContext ctx) {
149149
return this;
150150
}
151151

152-
public <T extends AbstractPhysicalPlan> T copyStatsAndGroupIdFrom(T from) {
152+
public <T extends AbstractPhysicalPlan> AbstractPhysicalPlan copyStatsAndGroupIdFrom(T from) {
153153
T newPlan = (T) withPhysicalPropertiesAndStats(
154154
from.getPhysicalProperties(), from.getStats());
155155
newPlan.setMutableState(MutableState.KEY_GROUP, from.getGroupIdAsString());
Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
package org.apache.doris.nereids.postprocess;
19+
20+
import org.apache.doris.nereids.processor.post.ColumnPruningPostProcessor;
21+
import org.apache.doris.nereids.rules.rewrite.InferFilterNotNull;
22+
import org.apache.doris.nereids.trees.plans.JoinType;
23+
import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan;
24+
import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
25+
import org.apache.doris.nereids.trees.plans.physical.PhysicalNestedLoopJoin;
26+
import org.apache.doris.nereids.trees.plans.physical.PhysicalPlan;
27+
import org.apache.doris.nereids.trees.plans.physical.PhysicalProject;
28+
import org.apache.doris.nereids.util.LogicalPlanBuilder;
29+
import org.apache.doris.nereids.util.MemoPatternMatchSupported;
30+
import org.apache.doris.nereids.util.MemoTestUtils;
31+
import org.apache.doris.nereids.util.PlanChecker;
32+
import org.apache.doris.nereids.util.PlanConstructor;
33+
34+
import com.google.common.collect.ImmutableList;
35+
import org.junit.jupiter.api.Assertions;
36+
import org.junit.jupiter.api.Test;
37+
38+
class ColumnPruningPostProcessorTest implements MemoPatternMatchSupported {
39+
private final LogicalOlapScan scan1 = PlanConstructor.newLogicalOlapScan(0, "t1", 0);
40+
private final LogicalOlapScan scan2 = PlanConstructor.newLogicalOlapScan(1, "t2", 0);
41+
42+
@Test
43+
void test() {
44+
LogicalPlan plan = new LogicalPlanBuilder(scan1)
45+
.join(scan2, JoinType.INNER_JOIN, ImmutableList.of())
46+
.project(ImmutableList.of(0, 2))
47+
.build();
48+
49+
PhysicalPlan physicalPlan = PlanChecker.from(MemoTestUtils.createConnectContext(), plan)
50+
.applyTopDown(new InferFilterNotNull())
51+
.implement()
52+
.getPhysicalPlan();
53+
54+
ColumnPruningPostProcessor processor = new ColumnPruningPostProcessor();
55+
PhysicalPlan newPlan = (PhysicalPlan) physicalPlan.accept(processor, null);
56+
57+
Assertions.assertTrue(newPlan instanceof PhysicalProject);
58+
Assertions.assertTrue(newPlan.child(0) instanceof PhysicalNestedLoopJoin);
59+
Assertions.assertTrue(newPlan.child(0).child(0) instanceof PhysicalProject);
60+
Assertions.assertTrue(newPlan.child(0).child(1) instanceof PhysicalProject);
61+
}
62+
}

‎regression-test/data/nereids_hint_tpch_p0/shape/q15.out‎

Lines changed: 20 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -9,25 +9,26 @@ PhysicalResultSink
99
------------PhysicalProject
1010
--------------PhysicalOlapScan[supplier]
1111
------------PhysicalDistribute[DistributionSpecHash]
12-
--------------hashJoin[INNER_JOIN] hashCondition=((revenue0.total_revenue = max(total_revenue))) otherCondition=()
13-
----------------hashAgg[GLOBAL]
14-
------------------PhysicalDistribute[DistributionSpecGather]
15-
--------------------hashAgg[LOCAL]
16-
----------------------PhysicalProject
17-
------------------------hashAgg[GLOBAL]
18-
--------------------------PhysicalDistribute[DistributionSpecHash]
19-
----------------------------hashAgg[LOCAL]
20-
------------------------------PhysicalProject
21-
--------------------------------filter((lineitem.l_shipdate < '1996-04-01') and (lineitem.l_shipdate >= '1996-01-01'))
22-
----------------------------------PhysicalOlapScan[lineitem]
23-
----------------PhysicalDistribute[DistributionSpecReplicated]
24-
------------------PhysicalProject
25-
--------------------hashAgg[GLOBAL]
26-
----------------------PhysicalDistribute[DistributionSpecHash]
27-
------------------------hashAgg[LOCAL]
28-
--------------------------PhysicalProject
29-
----------------------------filter((lineitem.l_shipdate < '1996-04-01') and (lineitem.l_shipdate >= '1996-01-01'))
30-
------------------------------PhysicalOlapScan[lineitem]
12+
--------------PhysicalProject
13+
----------------hashJoin[INNER_JOIN] hashCondition=((revenue0.total_revenue = max(total_revenue))) otherCondition=()
14+
------------------hashAgg[GLOBAL]
15+
--------------------PhysicalDistribute[DistributionSpecGather]
16+
----------------------hashAgg[LOCAL]
17+
------------------------PhysicalProject
18+
--------------------------hashAgg[GLOBAL]
19+
----------------------------PhysicalDistribute[DistributionSpecHash]
20+
------------------------------hashAgg[LOCAL]
21+
--------------------------------PhysicalProject
22+
----------------------------------filter((lineitem.l_shipdate < '1996-04-01') and (lineitem.l_shipdate >= '1996-01-01'))
23+
------------------------------------PhysicalOlapScan[lineitem]
24+
------------------PhysicalDistribute[DistributionSpecReplicated]
25+
--------------------PhysicalProject
26+
----------------------hashAgg[GLOBAL]
27+
------------------------PhysicalDistribute[DistributionSpecHash]
28+
--------------------------hashAgg[LOCAL]
29+
----------------------------PhysicalProject
30+
------------------------------filter((lineitem.l_shipdate < '1996-04-01') and (lineitem.l_shipdate >= '1996-01-01'))
31+
--------------------------------PhysicalOlapScan[lineitem]
3132

3233
Hint log:
3334
Used: leading(revenue0 supplier )

0 commit comments

Comments
 (0)