Skip to content

Commit b58e421

Browse files
feiniaofeiafeifoxtail463yangtao555
authored
branch-4.2:[fix](cost model) scale partitioned scalar aggregate cost by BE number apache#66354 (apache#68368)
picked from apache#66354 --------- Co-authored-by: foxtail463 <[email protected]> Co-authored-by: yangtao555 <[email protected]>
1 parent 4369b35 commit b58e421

28 files changed

Lines changed: 566 additions & 563 deletions

File tree

‎fe/fe-core/src/main/java/org/apache/doris/nereids/cost/CostModel.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -359,7 +359,9 @@ public Cost visitPhysicalHashAggregate(
359359
exprCost / 100 + inputStatistics.getRowCount() / beNumber,
360360
inputStatistics.getRowCount() / beNumber, 0);
361361
} else {
362-
int factor = aggregate.getGroupByExpressions().isEmpty() ? 1 : beNumber;
362+
boolean isPartitioned = !aggregate.getGroupByExpressions().isEmpty()
363+
|| aggregate.getPartitionExpressions().filter(expressions -> !expressions.isEmpty()).isPresent();
364+
int factor = isPartitioned ? beNumber : 1;
363365
// global
364366
return Cost.of(context.getCostWeight(),
365367
exprCost / 100 + inputStatistics.getRowCount() / factor,

‎fe/fe-core/src/test/java/org/apache/doris/nereids/cost/CostModelV1Test.java‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,15 +17,28 @@
1717

1818
package org.apache.doris.nereids.cost;
1919

20+
import org.apache.doris.nereids.PlanContext;
2021
import org.apache.doris.nereids.StatementContext;
2122
import org.apache.doris.nereids.sqltest.SqlTestBase;
23+
import org.apache.doris.nereids.trees.expressions.Slot;
24+
import org.apache.doris.nereids.trees.expressions.functions.agg.AggregateParam;
25+
import org.apache.doris.nereids.trees.plans.AggMode;
26+
import org.apache.doris.nereids.trees.plans.AggPhase;
2227
import org.apache.doris.nereids.trees.plans.Plan;
28+
import org.apache.doris.nereids.trees.plans.physical.PhysicalHashAggregate;
2329
import org.apache.doris.nereids.trees.plans.physical.PhysicalHashJoin;
2430
import org.apache.doris.nereids.util.PlanChecker;
31+
import org.apache.doris.nereids.util.PlanConstructor;
2532
import org.apache.doris.qe.SessionVariable;
33+
import org.apache.doris.statistics.Statistics;
34+
import org.apache.doris.statistics.StatisticsBuilder;
2635

36+
import com.google.common.collect.ImmutableList;
2737
import org.junit.jupiter.api.Assertions;
2838
import org.junit.jupiter.api.Test;
39+
import org.mockito.Mockito;
40+
41+
import java.util.Optional;
2942

3043
class CostModelV1Test extends SqlTestBase {
3144

@@ -87,4 +100,41 @@ void testMaterializingCost() {
87100
.getBestPlanTree();
88101
p.anyMatch(j -> j instanceof PhysicalHashJoin && ((PhysicalHashJoin<?, ?>) j).getJoinType().isRightJoin());
89102
}
103+
104+
@Test
105+
void testPartitionedScalarAggregateCostUsesClusterScale() {
106+
int originBeNumberForTest = connectContext.getSessionVariable().getBeNumberForTest();
107+
connectContext.getSessionVariable().setBeNumberForTest(4);
108+
try {
109+
Plan child = PlanConstructor.newLogicalOlapScan(101, "partitioned_scalar_agg_t", 0);
110+
Slot partitionKey = child.getOutput().get(0);
111+
PhysicalHashAggregate<Plan> aggregate = new PhysicalHashAggregate<Plan>(
112+
ImmutableList.of(), ImmutableList.of(partitionKey), Optional.of(ImmutableList.of(partitionKey)),
113+
new AggregateParam(AggPhase.GLOBAL, AggMode.INPUT_TO_RESULT), false, null, false, child);
114+
PhysicalHashAggregate<Plan> singlePointAggregate = new PhysicalHashAggregate<Plan>(
115+
ImmutableList.of(), ImmutableList.of(partitionKey), Optional.empty(),
116+
new AggregateParam(AggPhase.GLOBAL, AggMode.INPUT_TO_RESULT), false, null, false, child);
117+
PhysicalHashAggregate<Plan> groupByAggregate = new PhysicalHashAggregate<Plan>(
118+
ImmutableList.of(partitionKey), ImmutableList.of(partitionKey), Optional.empty(),
119+
new AggregateParam(AggPhase.GLOBAL, AggMode.INPUT_TO_RESULT), false, null, false, child);
120+
Statistics childStats = new StatisticsBuilder().setRowCount(1000).build();
121+
PlanContext context = Mockito.mock(PlanContext.class);
122+
Mockito.when(context.getChildStatistics(0)).thenReturn(childStats);
123+
Mockito.when(context.getCostWeight()).thenReturn(new CostWeight(1, 1, 1.5, 1));
124+
125+
Cost cost = new CostModel(connectContext).visitPhysicalHashAggregate(aggregate, context);
126+
Cost singlePointCost = new CostModel(connectContext).visitPhysicalHashAggregate(singlePointAggregate,
127+
context);
128+
Cost groupByCost = new CostModel(connectContext).visitPhysicalHashAggregate(groupByAggregate, context);
129+
130+
Assertions.assertEquals(250, cost.getCpuCost(), 1e-9);
131+
Assertions.assertEquals(250, cost.getMemoryCost(), 1e-9);
132+
Assertions.assertEquals(1000, singlePointCost.getCpuCost(), 1e-9);
133+
Assertions.assertEquals(1000, singlePointCost.getMemoryCost(), 1e-9);
134+
Assertions.assertEquals(250, groupByCost.getCpuCost(), 1e-9);
135+
Assertions.assertEquals(250, groupByCost.getMemoryCost(), 1e-9);
136+
} finally {
137+
connectContext.getSessionVariable().setBeNumberForTest(originBeNumberForTest);
138+
}
139+
}
90140
}

‎regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query16.out‎

Lines changed: 20 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -3,33 +3,31 @@
33
PhysicalResultSink
44
--PhysicalLimit[GLOBAL]
55
----PhysicalLimit[LOCAL]
6-
------hashAgg[DISTINCT_GLOBAL]
6+
------hashAgg[GLOBAL]
77
--------PhysicalDistribute[DistributionSpecGather]
8-
----------hashAgg[DISTINCT_LOCAL]
9-
------------hashAgg[GLOBAL]
10-
--------------hashAgg[LOCAL]
8+
----------hashAgg[GLOBAL]
9+
------------PhysicalProject
10+
--------------hashJoin[INNER_JOIN broadcast] hashCondition=((cs1.cs_call_center_sk = call_center.cc_call_center_sk)) otherCondition=() build RFs:RF3 cc_call_center_sk->[cs_call_center_sk]
1111
----------------PhysicalProject
12-
------------------hashJoin[INNER_JOIN broadcast] hashCondition=((cs1.cs_call_center_sk = call_center.cc_call_center_sk)) otherCondition=() build RFs:RF3 cc_call_center_sk->[cs_call_center_sk]
12+
------------------hashJoin[INNER_JOIN broadcast] hashCondition=((cs1.cs_ship_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF2 ca_address_sk->[cs_ship_addr_sk]
1313
--------------------PhysicalProject
14-
----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((cs1.cs_ship_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF2 ca_address_sk->[cs_ship_addr_sk]
15-
------------------------PhysicalProject
16-
--------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((cs1.cs_ship_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->[cs_ship_date_sk]
17-
----------------------------hashJoin[LEFT_ANTI_JOIN bucketShuffle] hashCondition=((cs1.cs_order_number = cr1.cr_order_number)) otherCondition=()
14+
----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((cs1.cs_ship_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->[cs_ship_date_sk]
15+
------------------------hashJoin[LEFT_ANTI_JOIN bucketShuffle] hashCondition=((cs1.cs_order_number = cr1.cr_order_number)) otherCondition=()
16+
--------------------------PhysicalProject
17+
----------------------------hashJoin[RIGHT_SEMI_JOIN shuffle] hashCondition=((cs1.cs_order_number = cs2.cs_order_number)) otherCondition=(( not (cs_warehouse_sk = cs_warehouse_sk))) build RFs:RF0 cs_order_number->[cs_order_number]
1818
------------------------------PhysicalProject
19-
--------------------------------hashJoin[RIGHT_SEMI_JOIN shuffle] hashCondition=((cs1.cs_order_number = cs2.cs_order_number)) otherCondition=(( not (cs_warehouse_sk = cs_warehouse_sk))) build RFs:RF0 cs_order_number->[cs_order_number]
20-
----------------------------------PhysicalProject
21-
------------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF0
22-
----------------------------------PhysicalProject
23-
------------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF1 RF2 RF3
19+
--------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF0
2420
------------------------------PhysicalProject
25-
--------------------------------PhysicalOlapScan[catalog_returns]
26-
----------------------------PhysicalProject
27-
------------------------------filter((date_dim.d_date <= '2002-05-31') and (date_dim.d_date >= '2002-04-01'))
28-
--------------------------------PhysicalOlapScan[date_dim]
21+
--------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF1 RF2 RF3
22+
--------------------------PhysicalProject
23+
----------------------------PhysicalOlapScan[catalog_returns]
2924
------------------------PhysicalProject
30-
--------------------------filter((customer_address.ca_state = 'WV'))
31-
----------------------------PhysicalOlapScan[customer_address]
25+
--------------------------filter((date_dim.d_date <= '2002-05-31') and (date_dim.d_date >= '2002-04-01'))
26+
----------------------------PhysicalOlapScan[date_dim]
3227
--------------------PhysicalProject
33-
----------------------filter(cc_county IN ('Barrow County', 'Daviess County', 'Luce County', 'Richland County', 'Ziebach County'))
34-
------------------------PhysicalOlapScan[call_center]
28+
----------------------filter((customer_address.ca_state = 'WV'))
29+
------------------------PhysicalOlapScan[customer_address]
30+
----------------PhysicalProject
31+
------------------filter(cc_county IN ('Barrow County', 'Daviess County', 'Luce County', 'Richland County', 'Ziebach County'))
32+
--------------------PhysicalOlapScan[call_center]
3533

‎regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query94.out‎

Lines changed: 20 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -3,33 +3,31 @@
33
PhysicalResultSink
44
--PhysicalLimit[GLOBAL]
55
----PhysicalLimit[LOCAL]
6-
------hashAgg[DISTINCT_GLOBAL]
6+
------hashAgg[GLOBAL]
77
--------PhysicalDistribute[DistributionSpecGather]
8-
----------hashAgg[DISTINCT_LOCAL]
9-
------------hashAgg[GLOBAL]
10-
--------------hashAgg[LOCAL]
8+
----------hashAgg[GLOBAL]
9+
------------PhysicalProject
10+
--------------hashJoin[INNER_JOIN broadcast] hashCondition=((ws1.ws_web_site_sk = web_site.web_site_sk)) otherCondition=() build RFs:RF3 web_site_sk->[ws_web_site_sk]
1111
----------------PhysicalProject
12-
------------------hashJoin[INNER_JOIN broadcast] hashCondition=((ws1.ws_web_site_sk = web_site.web_site_sk)) otherCondition=() build RFs:RF3 web_site_sk->[ws_web_site_sk]
12+
------------------hashJoin[INNER_JOIN broadcast] hashCondition=((ws1.ws_ship_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF2 ca_address_sk->[ws_ship_addr_sk]
1313
--------------------PhysicalProject
14-
----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((ws1.ws_ship_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF2 ca_address_sk->[ws_ship_addr_sk]
15-
------------------------PhysicalProject
16-
--------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((ws1.ws_ship_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->[ws_ship_date_sk]
17-
----------------------------hashJoin[LEFT_ANTI_JOIN bucketShuffle] hashCondition=((ws1.ws_order_number = wr1.wr_order_number)) otherCondition=()
14+
----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((ws1.ws_ship_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->[ws_ship_date_sk]
15+
------------------------hashJoin[LEFT_ANTI_JOIN bucketShuffle] hashCondition=((ws1.ws_order_number = wr1.wr_order_number)) otherCondition=()
16+
--------------------------PhysicalProject
17+
----------------------------hashJoin[RIGHT_SEMI_JOIN shuffle] hashCondition=((ws1.ws_order_number = ws2.ws_order_number)) otherCondition=(( not (ws_warehouse_sk = ws_warehouse_sk))) build RFs:RF0 ws_order_number->[ws_order_number]
1818
------------------------------PhysicalProject
19-
--------------------------------hashJoin[RIGHT_SEMI_JOIN shuffle] hashCondition=((ws1.ws_order_number = ws2.ws_order_number)) otherCondition=(( not (ws_warehouse_sk = ws_warehouse_sk))) build RFs:RF0 ws_order_number->[ws_order_number]
20-
----------------------------------PhysicalProject
21-
------------------------------------PhysicalOlapScan[web_sales] apply RFs: RF0
22-
----------------------------------PhysicalProject
23-
------------------------------------PhysicalOlapScan[web_sales] apply RFs: RF1 RF2 RF3
19+
--------------------------------PhysicalOlapScan[web_sales] apply RFs: RF0
2420
------------------------------PhysicalProject
25-
--------------------------------PhysicalOlapScan[web_returns]
26-
----------------------------PhysicalProject
27-
------------------------------filter((date_dim.d_date <= '2000-04-01') and (date_dim.d_date >= '2000-02-01'))
28-
--------------------------------PhysicalOlapScan[date_dim]
21+
--------------------------------PhysicalOlapScan[web_sales] apply RFs: RF1 RF2 RF3
22+
--------------------------PhysicalProject
23+
----------------------------PhysicalOlapScan[web_returns]
2924
------------------------PhysicalProject
30-
--------------------------filter((customer_address.ca_state = 'OK'))
31-
----------------------------PhysicalOlapScan[customer_address]
25+
--------------------------filter((date_dim.d_date <= '2000-04-01') and (date_dim.d_date >= '2000-02-01'))
26+
----------------------------PhysicalOlapScan[date_dim]
3227
--------------------PhysicalProject
33-
----------------------filter((web_site.web_company_name = 'pri'))
34-
------------------------PhysicalOlapScan[web_site]
28+
----------------------filter((customer_address.ca_state = 'OK'))
29+
------------------------PhysicalOlapScan[customer_address]
30+
----------------PhysicalProject
31+
------------------filter((web_site.web_company_name = 'pri'))
32+
--------------------PhysicalOlapScan[web_site]
3533

‎regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query95.out‎

Lines changed: 21 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -11,34 +11,32 @@ PhysicalCteAnchor ( cteId=CTEId#0 )
1111
--PhysicalResultSink
1212
----PhysicalLimit[GLOBAL]
1313
------PhysicalLimit[LOCAL]
14-
--------hashAgg[DISTINCT_GLOBAL]
14+
--------hashAgg[GLOBAL]
1515
----------PhysicalDistribute[DistributionSpecGather]
16-
------------hashAgg[DISTINCT_LOCAL]
17-
--------------hashAgg[GLOBAL]
18-
----------------hashAgg[LOCAL]
16+
------------hashAgg[GLOBAL]
17+
--------------PhysicalProject
18+
----------------hashJoin[INNER_JOIN broadcast] hashCondition=((ws1.ws_web_site_sk = web_site.web_site_sk)) otherCondition=() build RFs:RF6 web_site_sk->[ws_web_site_sk]
1919
------------------PhysicalProject
20-
--------------------hashJoin[INNER_JOIN broadcast] hashCondition=((ws1.ws_web_site_sk = web_site.web_site_sk)) otherCondition=() build RFs:RF6 web_site_sk->[ws_web_site_sk]
20+
--------------------hashJoin[INNER_JOIN broadcast] hashCondition=((ws1.ws_ship_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF5 ca_address_sk->[ws_ship_addr_sk]
2121
----------------------PhysicalProject
22-
------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((ws1.ws_ship_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF5 ca_address_sk->[ws_ship_addr_sk]
23-
--------------------------PhysicalProject
24-
----------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((ws1.ws_ship_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF4 d_date_sk->[ws_ship_date_sk]
25-
------------------------------hashJoin[RIGHT_SEMI_JOIN shuffleBucket] hashCondition=((ws1.ws_order_number = ws_wh.ws_order_number)) otherCondition=() build RFs:RF3 ws_order_number->[ws_order_number]
26-
--------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF3
27-
--------------------------------hashJoin[RIGHT_SEMI_JOIN bucketShuffle] hashCondition=((ws1.ws_order_number = web_returns.wr_order_number)) otherCondition=() build RFs:RF2 ws_order_number->[wr_order_number];RF7 ws_order_number->[ws_order_number,ws_order_number]
28-
----------------------------------PhysicalProject
29-
------------------------------------hashJoin[INNER_JOIN shuffle] hashCondition=((web_returns.wr_order_number = ws_wh.ws_order_number)) otherCondition=()
30-
--------------------------------------PhysicalCteConsumer ( cteId=CTEId#0 )
31-
--------------------------------------PhysicalProject
32-
----------------------------------------PhysicalOlapScan[web_returns] apply RFs: RF2
22+
------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((ws1.ws_ship_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF4 d_date_sk->[ws_ship_date_sk]
23+
--------------------------hashJoin[RIGHT_SEMI_JOIN shuffleBucket] hashCondition=((ws1.ws_order_number = ws_wh.ws_order_number)) otherCondition=() build RFs:RF3 ws_order_number->[ws_order_number]
24+
----------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF3
25+
----------------------------hashJoin[RIGHT_SEMI_JOIN bucketShuffle] hashCondition=((ws1.ws_order_number = web_returns.wr_order_number)) otherCondition=() build RFs:RF2 ws_order_number->[wr_order_number];RF7 ws_order_number->[ws_order_number,ws_order_number]
26+
------------------------------PhysicalProject
27+
--------------------------------hashJoin[INNER_JOIN shuffle] hashCondition=((web_returns.wr_order_number = ws_wh.ws_order_number)) otherCondition=()
28+
----------------------------------PhysicalCteConsumer ( cteId=CTEId#0 )
3329
----------------------------------PhysicalProject
34-
------------------------------------PhysicalOlapScan[web_sales] apply RFs: RF4 RF5 RF6
30+
------------------------------------PhysicalOlapScan[web_returns] apply RFs: RF2
3531
------------------------------PhysicalProject
36-
--------------------------------filter((date_dim.d_date <= '1999-04-02') and (date_dim.d_date >= '1999-02-01'))
37-
----------------------------------PhysicalOlapScan[date_dim]
32+
--------------------------------PhysicalOlapScan[web_sales] apply RFs: RF4 RF5 RF6
3833
--------------------------PhysicalProject
39-
----------------------------filter((customer_address.ca_state = 'NC'))
40-
------------------------------PhysicalOlapScan[customer_address]
34+
----------------------------filter((date_dim.d_date <= '1999-04-02') and (date_dim.d_date >= '1999-02-01'))
35+
------------------------------PhysicalOlapScan[date_dim]
4136
----------------------PhysicalProject
42-
------------------------filter((web_site.web_company_name = 'pri'))
43-
--------------------------PhysicalOlapScan[web_site]
37+
------------------------filter((customer_address.ca_state = 'NC'))
38+
--------------------------PhysicalOlapScan[customer_address]
39+
------------------PhysicalProject
40+
--------------------filter((web_site.web_company_name = 'pri'))
41+
----------------------PhysicalOlapScan[web_site]
4442

0 commit comments

Comments
 (0)