Skip to content

Commit 051b9ac

Browse files
foxtail463yangtao555
andauthored
[fix](cost model) scale group-by-less global agg cost by BE number when partitioned (apache#66354)
Problem Summary: A scalar count(distinct) runs its global aggregate hash-shuffled by the distinct key (non-empty partitionExpressions), i.e. in parallel across BEs. But the cost model set the parallelism factor from group-by keys alone, so with no group by it used factor = 1, costing the aggregate as single-point. This overestimated the parallel plan and made CBO reject it. Solution: Treat a global aggregate as partitioned when it has group by keys or non-empty partitionExpressions, and scale its cost by beNumber accordingly. --------- Co-authored-by: yangtao555 <[email protected]>
1 parent 86c78b9 commit 051b9ac

27 files changed

Lines changed: 565 additions & 562 deletions

File tree

‎fe/fe-core/src/main/java/org/apache/doris/nereids/cost/CostModel.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -360,7 +360,9 @@ public Cost visitPhysicalHashAggregate(
360360
exprCost / 100 + inputStatistics.getRowCount() / beNumber,
361361
inputStatistics.getRowCount() / beNumber, 0);
362362
} else {
363-
int factor = aggregate.getGroupByExpressions().isEmpty() ? 1 : beNumber;
363+
boolean isPartitioned = !aggregate.getGroupByExpressions().isEmpty()
364+
|| aggregate.getPartitionExpressions().filter(expressions -> !expressions.isEmpty()).isPresent();
365+
int factor = isPartitioned ? beNumber : 1;
364366
// global
365367
return Cost.of(context.getSessionVariable(),
366368
exprCost / 100 + inputStatistics.getRowCount() / factor,

‎fe/fe-core/src/test/java/org/apache/doris/nereids/cost/CostModelV1Test.java‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,12 +17,26 @@
1717

1818
package org.apache.doris.nereids.cost;
1919

20+
import org.apache.doris.nereids.PlanContext;
2021
import org.apache.doris.nereids.sqltest.SqlTestBase;
22+
import org.apache.doris.nereids.trees.expressions.Slot;
23+
import org.apache.doris.nereids.trees.expressions.functions.agg.AggregateParam;
24+
import org.apache.doris.nereids.trees.plans.AggMode;
25+
import org.apache.doris.nereids.trees.plans.AggPhase;
2126
import org.apache.doris.nereids.trees.plans.Plan;
27+
import org.apache.doris.nereids.trees.plans.physical.PhysicalHashAggregate;
2228
import org.apache.doris.nereids.trees.plans.physical.PhysicalHashJoin;
2329
import org.apache.doris.nereids.util.PlanChecker;
30+
import org.apache.doris.nereids.util.PlanConstructor;
31+
import org.apache.doris.statistics.Statistics;
32+
import org.apache.doris.statistics.StatisticsBuilder;
2433

34+
import com.google.common.collect.ImmutableList;
35+
import org.junit.jupiter.api.Assertions;
2536
import org.junit.jupiter.api.Test;
37+
import org.mockito.Mockito;
38+
39+
import java.util.Optional;
2640

2741
class CostModelV1Test extends SqlTestBase {
2842

@@ -39,4 +53,41 @@ void testMaterializingCost() {
3953
.getBestPlanTree();
4054
p.anyMatch(j -> j instanceof PhysicalHashJoin && ((PhysicalHashJoin<?, ?>) j).getJoinType().isRightJoin());
4155
}
56+
57+
@Test
58+
void testPartitionedScalarAggregateCostUsesClusterScale() {
59+
int originBeNumberForTest = connectContext.getSessionVariable().getBeNumberForTest();
60+
connectContext.getSessionVariable().setBeNumberForTest(4);
61+
try {
62+
Plan child = PlanConstructor.newLogicalOlapScan(101, "partitioned_scalar_agg_t", 0);
63+
Slot partitionKey = child.getOutput().get(0);
64+
PhysicalHashAggregate<Plan> aggregate = new PhysicalHashAggregate<Plan>(
65+
ImmutableList.of(), ImmutableList.of(partitionKey), Optional.of(ImmutableList.of(partitionKey)),
66+
new AggregateParam(AggPhase.GLOBAL, AggMode.INPUT_TO_RESULT), false, null, false, child);
67+
PhysicalHashAggregate<Plan> singlePointAggregate = new PhysicalHashAggregate<Plan>(
68+
ImmutableList.of(), ImmutableList.of(partitionKey), Optional.empty(),
69+
new AggregateParam(AggPhase.GLOBAL, AggMode.INPUT_TO_RESULT), false, null, false, child);
70+
PhysicalHashAggregate<Plan> groupByAggregate = new PhysicalHashAggregate<Plan>(
71+
ImmutableList.of(partitionKey), ImmutableList.of(partitionKey), Optional.empty(),
72+
new AggregateParam(AggPhase.GLOBAL, AggMode.INPUT_TO_RESULT), false, null, false, child);
73+
Statistics childStats = new StatisticsBuilder().setRowCount(1000).build();
74+
PlanContext context = Mockito.mock(PlanContext.class);
75+
Mockito.when(context.getChildStatistics(0)).thenReturn(childStats);
76+
Mockito.when(context.getSessionVariable()).thenReturn(connectContext.getSessionVariable());
77+
78+
Cost cost = new CostModel(connectContext).visitPhysicalHashAggregate(aggregate, context);
79+
Cost singlePointCost = new CostModel(connectContext).visitPhysicalHashAggregate(singlePointAggregate,
80+
context);
81+
Cost groupByCost = new CostModel(connectContext).visitPhysicalHashAggregate(groupByAggregate, context);
82+
83+
Assertions.assertEquals(250, cost.getCpuCost(), 1e-9);
84+
Assertions.assertEquals(250, cost.getMemoryCost(), 1e-9);
85+
Assertions.assertEquals(1000, singlePointCost.getCpuCost(), 1e-9);
86+
Assertions.assertEquals(1000, singlePointCost.getMemoryCost(), 1e-9);
87+
Assertions.assertEquals(250, groupByCost.getCpuCost(), 1e-9);
88+
Assertions.assertEquals(250, groupByCost.getMemoryCost(), 1e-9);
89+
} finally {
90+
connectContext.getSessionVariable().setBeNumberForTest(originBeNumberForTest);
91+
}
92+
}
4293
}

‎regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query16.out‎

Lines changed: 20 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -3,33 +3,31 @@
33
PhysicalResultSink
44
--PhysicalLimit[GLOBAL]
55
----PhysicalLimit[LOCAL]
6-
------hashAgg[DISTINCT_GLOBAL]
6+
------hashAgg[GLOBAL]
77
--------PhysicalDistribute[DistributionSpecGather]
8-
----------hashAgg[DISTINCT_LOCAL]
9-
------------hashAgg[GLOBAL]
10-
--------------hashAgg[LOCAL]
8+
----------hashAgg[GLOBAL]
9+
------------PhysicalProject
10+
--------------hashJoin[INNER_JOIN broadcast] hashCondition=((cs1.cs_call_center_sk = call_center.cc_call_center_sk)) otherCondition=() build RFs:RF3 cc_call_center_sk->cs_call_center_sk
1111
----------------PhysicalProject
12-
------------------hashJoin[INNER_JOIN broadcast] hashCondition=((cs1.cs_call_center_sk = call_center.cc_call_center_sk)) otherCondition=() build RFs:RF3 cc_call_center_sk->cs_call_center_sk
12+
------------------hashJoin[INNER_JOIN broadcast] hashCondition=((cs1.cs_ship_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF2 ca_address_sk->cs_ship_addr_sk
1313
--------------------PhysicalProject
14-
----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((cs1.cs_ship_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF2 ca_address_sk->cs_ship_addr_sk
15-
------------------------PhysicalProject
16-
--------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((cs1.cs_ship_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->cs_ship_date_sk
17-
----------------------------hashJoin[LEFT_ANTI_JOIN bucketShuffle] hashCondition=((cs1.cs_order_number = cr1.cr_order_number)) otherCondition=()
14+
----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((cs1.cs_ship_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->cs_ship_date_sk
15+
------------------------hashJoin[LEFT_ANTI_JOIN bucketShuffle] hashCondition=((cs1.cs_order_number = cr1.cr_order_number)) otherCondition=()
16+
--------------------------PhysicalProject
17+
----------------------------hashJoin[RIGHT_SEMI_JOIN shuffle] hashCondition=((cs1.cs_order_number = cs2.cs_order_number)) otherCondition=(( not (cs1.cs_warehouse_sk = cs2.cs_warehouse_sk))) build RFs:RF0 cs_order_number->cs_order_number
1818
------------------------------PhysicalProject
19-
--------------------------------hashJoin[RIGHT_SEMI_JOIN shuffle] hashCondition=((cs1.cs_order_number = cs2.cs_order_number)) otherCondition=(( not (cs1.cs_warehouse_sk = cs2.cs_warehouse_sk))) build RFs:RF0 cs_order_number->cs_order_number
20-
----------------------------------PhysicalProject
21-
------------------------------------PhysicalOlapScan[catalog_sales(cs2)] apply RFs: RF0
22-
----------------------------------PhysicalProject
23-
------------------------------------PhysicalOlapScan[catalog_sales(cs1)] apply RFs: RF1 RF2 RF3
19+
--------------------------------PhysicalOlapScan[catalog_sales(cs2)] apply RFs: RF0
2420
------------------------------PhysicalProject
25-
--------------------------------PhysicalOlapScan[catalog_returns(cr1)]
26-
----------------------------PhysicalProject
27-
------------------------------filter((date_dim.d_date <= '2002-05-31') and (date_dim.d_date >= '2002-04-01'))
28-
--------------------------------PhysicalOlapScan[date_dim]
21+
--------------------------------PhysicalOlapScan[catalog_sales(cs1)] apply RFs: RF1 RF2 RF3
22+
--------------------------PhysicalProject
23+
----------------------------PhysicalOlapScan[catalog_returns(cr1)]
2924
------------------------PhysicalProject
30-
--------------------------filter((customer_address.ca_state = 'WV'))
31-
----------------------------PhysicalOlapScan[customer_address]
25+
--------------------------filter((date_dim.d_date <= '2002-05-31') and (date_dim.d_date >= '2002-04-01'))
26+
----------------------------PhysicalOlapScan[date_dim]
3227
--------------------PhysicalProject
33-
----------------------filter(call_center.cc_county IN ('Barrow County', 'Daviess County', 'Luce County', 'Richland County', 'Ziebach County'))
34-
------------------------PhysicalOlapScan[call_center]
28+
----------------------filter((customer_address.ca_state = 'WV'))
29+
------------------------PhysicalOlapScan[customer_address]
30+
----------------PhysicalProject
31+
------------------filter(call_center.cc_county IN ('Barrow County', 'Daviess County', 'Luce County', 'Richland County', 'Ziebach County'))
32+
--------------------PhysicalOlapScan[call_center]
3533

‎regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query94.out‎

Lines changed: 20 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -3,33 +3,31 @@
33
PhysicalResultSink
44
--PhysicalLimit[GLOBAL]
55
----PhysicalLimit[LOCAL]
6-
------hashAgg[DISTINCT_GLOBAL]
6+
------hashAgg[GLOBAL]
77
--------PhysicalDistribute[DistributionSpecGather]
8-
----------hashAgg[DISTINCT_LOCAL]
9-
------------hashAgg[GLOBAL]
10-
--------------hashAgg[LOCAL]
8+
----------hashAgg[GLOBAL]
9+
------------PhysicalProject
10+
--------------hashJoin[INNER_JOIN broadcast] hashCondition=((ws1.ws_web_site_sk = web_site.web_site_sk)) otherCondition=() build RFs:RF3 web_site_sk->ws_web_site_sk
1111
----------------PhysicalProject
12-
------------------hashJoin[INNER_JOIN broadcast] hashCondition=((ws1.ws_web_site_sk = web_site.web_site_sk)) otherCondition=() build RFs:RF3 web_site_sk->ws_web_site_sk
12+
------------------hashJoin[INNER_JOIN broadcast] hashCondition=((ws1.ws_ship_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF2 ca_address_sk->ws_ship_addr_sk
1313
--------------------PhysicalProject
14-
----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((ws1.ws_ship_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF2 ca_address_sk->ws_ship_addr_sk
15-
------------------------PhysicalProject
16-
--------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((ws1.ws_ship_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->ws_ship_date_sk
17-
----------------------------hashJoin[LEFT_ANTI_JOIN bucketShuffle] hashCondition=((ws1.ws_order_number = wr1.wr_order_number)) otherCondition=()
14+
----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((ws1.ws_ship_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->ws_ship_date_sk
15+
------------------------hashJoin[LEFT_ANTI_JOIN bucketShuffle] hashCondition=((ws1.ws_order_number = wr1.wr_order_number)) otherCondition=()
16+
--------------------------PhysicalProject
17+
----------------------------hashJoin[RIGHT_SEMI_JOIN shuffle] hashCondition=((ws1.ws_order_number = ws2.ws_order_number)) otherCondition=(( not (ws1.ws_warehouse_sk = ws2.ws_warehouse_sk))) build RFs:RF0 ws_order_number->ws_order_number
1818
------------------------------PhysicalProject
19-
--------------------------------hashJoin[RIGHT_SEMI_JOIN shuffle] hashCondition=((ws1.ws_order_number = ws2.ws_order_number)) otherCondition=(( not (ws1.ws_warehouse_sk = ws2.ws_warehouse_sk))) build RFs:RF0 ws_order_number->ws_order_number
20-
----------------------------------PhysicalProject
21-
------------------------------------PhysicalOlapScan[web_sales(ws2)] apply RFs: RF0
22-
----------------------------------PhysicalProject
23-
------------------------------------PhysicalOlapScan[web_sales(ws1)] apply RFs: RF1 RF2 RF3
19+
--------------------------------PhysicalOlapScan[web_sales(ws2)] apply RFs: RF0
2420
------------------------------PhysicalProject
25-
--------------------------------PhysicalOlapScan[web_returns(wr1)]
26-
----------------------------PhysicalProject
27-
------------------------------filter((date_dim.d_date <= '2000-04-01') and (date_dim.d_date >= '2000-02-01'))
28-
--------------------------------PhysicalOlapScan[date_dim]
21+
--------------------------------PhysicalOlapScan[web_sales(ws1)] apply RFs: RF1 RF2 RF3
22+
--------------------------PhysicalProject
23+
----------------------------PhysicalOlapScan[web_returns(wr1)]
2924
------------------------PhysicalProject
30-
--------------------------filter((customer_address.ca_state = 'OK'))
31-
----------------------------PhysicalOlapScan[customer_address]
25+
--------------------------filter((date_dim.d_date <= '2000-04-01') and (date_dim.d_date >= '2000-02-01'))
26+
----------------------------PhysicalOlapScan[date_dim]
3227
--------------------PhysicalProject
33-
----------------------filter((web_site.web_company_name = 'pri'))
34-
------------------------PhysicalOlapScan[web_site]
28+
----------------------filter((customer_address.ca_state = 'OK'))
29+
------------------------PhysicalOlapScan[customer_address]
30+
----------------PhysicalProject
31+
------------------filter((web_site.web_company_name = 'pri'))
32+
--------------------PhysicalOlapScan[web_site]
3533

‎regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query95.out‎

Lines changed: 21 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -11,34 +11,32 @@ PhysicalCteAnchor ( cteId=CTEId#0 )
1111
--PhysicalResultSink
1212
----PhysicalLimit[GLOBAL]
1313
------PhysicalLimit[LOCAL]
14-
--------hashAgg[DISTINCT_GLOBAL]
14+
--------hashAgg[GLOBAL]
1515
----------PhysicalDistribute[DistributionSpecGather]
16-
------------hashAgg[DISTINCT_LOCAL]
17-
--------------hashAgg[GLOBAL]
18-
----------------hashAgg[LOCAL]
16+
------------hashAgg[GLOBAL]
17+
--------------PhysicalProject
18+
----------------hashJoin[INNER_JOIN broadcast] hashCondition=((ws1.ws_web_site_sk = web_site.web_site_sk)) otherCondition=() build RFs:RF7 web_site_sk->ws_web_site_sk
1919
------------------PhysicalProject
20-
--------------------hashJoin[INNER_JOIN broadcast] hashCondition=((ws1.ws_web_site_sk = web_site.web_site_sk)) otherCondition=() build RFs:RF7 web_site_sk->ws_web_site_sk
20+
--------------------hashJoin[INNER_JOIN broadcast] hashCondition=((ws1.ws_ship_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF6 ca_address_sk->ws_ship_addr_sk
2121
----------------------PhysicalProject
22-
------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((ws1.ws_ship_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF6 ca_address_sk->ws_ship_addr_sk
23-
--------------------------PhysicalProject
24-
----------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((ws1.ws_ship_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF5 d_date_sk->ws_ship_date_sk
25-
------------------------------hashJoin[RIGHT_SEMI_JOIN shuffleBucket] hashCondition=((ws1.ws_order_number = ws_wh.ws_order_number)) otherCondition=() build RFs:RF4 ws_order_number->ws_order_number
26-
--------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF4
27-
--------------------------------hashJoin[RIGHT_SEMI_JOIN bucketShuffle] hashCondition=((ws1.ws_order_number = web_returns.wr_order_number)) otherCondition=() build RFs:RF2 ws_order_number->wr_order_number;RF8 ws_order_number->ws_order_number;RF9 ws_order_number->ws_order_number
28-
----------------------------------PhysicalProject
29-
------------------------------------hashJoin[INNER_JOIN shuffle] hashCondition=((web_returns.wr_order_number = ws_wh.ws_order_number)) otherCondition=() build RFs:RF1 wr_order_number->ws_order_number
30-
--------------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF1
31-
--------------------------------------PhysicalProject
32-
----------------------------------------PhysicalOlapScan[web_returns] apply RFs: RF2
22+
------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((ws1.ws_ship_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF5 d_date_sk->ws_ship_date_sk
23+
--------------------------hashJoin[RIGHT_SEMI_JOIN shuffleBucket] hashCondition=((ws1.ws_order_number = ws_wh.ws_order_number)) otherCondition=() build RFs:RF4 ws_order_number->ws_order_number
24+
----------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF4
25+
----------------------------hashJoin[RIGHT_SEMI_JOIN bucketShuffle] hashCondition=((ws1.ws_order_number = web_returns.wr_order_number)) otherCondition=() build RFs:RF2 ws_order_number->wr_order_number;RF8 ws_order_number->ws_order_number;RF9 ws_order_number->ws_order_number
26+
------------------------------PhysicalProject
27+
--------------------------------hashJoin[INNER_JOIN shuffle] hashCondition=((web_returns.wr_order_number = ws_wh.ws_order_number)) otherCondition=() build RFs:RF1 wr_order_number->ws_order_number
28+
----------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF1
3329
----------------------------------PhysicalProject
34-
------------------------------------PhysicalOlapScan[web_sales(ws1)] apply RFs: RF5 RF6 RF7
30+
------------------------------------PhysicalOlapScan[web_returns] apply RFs: RF2
3531
------------------------------PhysicalProject
36-
--------------------------------filter((date_dim.d_date <= '1999-04-02') and (date_dim.d_date >= '1999-02-01'))
37-
----------------------------------PhysicalOlapScan[date_dim]
32+
--------------------------------PhysicalOlapScan[web_sales(ws1)] apply RFs: RF5 RF6 RF7
3833
--------------------------PhysicalProject
39-
----------------------------filter((customer_address.ca_state = 'NC'))
40-
------------------------------PhysicalOlapScan[customer_address]
34+
----------------------------filter((date_dim.d_date <= '1999-04-02') and (date_dim.d_date >= '1999-02-01'))
35+
------------------------------PhysicalOlapScan[date_dim]
4136
----------------------PhysicalProject
42-
------------------------filter((web_site.web_company_name = 'pri'))
43-
--------------------------PhysicalOlapScan[web_site]
37+
------------------------filter((customer_address.ca_state = 'NC'))
38+
--------------------------PhysicalOlapScan[customer_address]
39+
------------------PhysicalProject
40+
--------------------filter((web_site.web_company_name = 'pri'))
41+
----------------------PhysicalOlapScan[web_site]
4442

0 commit comments

Comments
 (0)