Skip to content

Commit 302ba2e

Browse files
branch-4.0: [opt](nereids) optimize bs downgrade alg. apache#56279 (apache#56889)
Cherry-picked from apache#56279 Co-authored-by: minghong <[email protected]>
1 parent 87d569e commit 302ba2e

87 files changed

Lines changed: 282 additions & 286 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎fe/fe-core/src/main/java/org/apache/doris/nereids/properties/ChildrenPropertiesRegulator.java‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -303,8 +303,7 @@ private boolean isBucketShuffleDownGrade(Plan oneSidePlan, DistributionSpecHash
303303
int backEndNum = Math.max(1, ConnectContext.get().getEnv().getClusterInfo()
304304
.getBackendsNumber(true));
305305
int paraNum = Math.max(1, ConnectContext.get().getSessionVariable().getParallelExecInstanceNum());
306-
int totalParaNum = Math.min(10, backEndNum * paraNum);
307-
return totalBucketNum < totalParaNum;
306+
return totalBucketNum < backEndNum * paraNum * 0.8;
308307
}
309308
}
310309
}

‎regression-test/data/shape_check/ssb_sf100/shape/q4.3.out‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ PhysicalResultSink
88
----------PhysicalDistribute[DistributionSpecHash]
99
------------hashAgg[LOCAL]
1010
--------------PhysicalProject
11-
----------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((lineorder.lo_custkey = customer.c_custkey)) otherCondition=() build RFs:RF3 lo_custkey->[c_custkey]
11+
----------------hashJoin[INNER_JOIN shuffle] hashCondition=((lineorder.lo_custkey = customer.c_custkey)) otherCondition=() build RFs:RF3 lo_custkey->[c_custkey]
1212
------------------PhysicalProject
1313
--------------------PhysicalOlapScan[customer] apply RFs: RF3
1414
------------------PhysicalProject

‎regression-test/data/shape_check/tpcds_sf100/rf_prune/query15.out‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -8,18 +8,18 @@ PhysicalResultSink
88
----------PhysicalDistribute[DistributionSpecHash]
99
------------hashAgg[LOCAL]
1010
--------------PhysicalProject
11-
----------------hashJoin[INNER_JOIN shuffle] hashCondition=((catalog_sales.cs_bill_customer_sk = customer.c_customer_sk)) otherCondition=(OR[substring(ca_zip, 1, 5) IN ('80348', '81792', '83405', '85392', '85460', '85669', '86197', '86475', '88274'),ca_state IN ('CA', 'GA', 'WA'),(catalog_sales.cs_sales_price > 500.00)]) build RFs:RF2 c_customer_sk->[cs_bill_customer_sk]
11+
----------------hashJoin[INNER_JOIN shuffle] hashCondition=((catalog_sales.cs_bill_customer_sk = customer.c_customer_sk)) otherCondition=(OR[substring(ca_zip, 1, 5) IN ('80348', '81792', '83405', '85392', '85460', '85669', '86197', '86475', '88274'),ca_state IN ('CA', 'GA', 'WA'),(catalog_sales.cs_sales_price > 500.00)])
1212
------------------PhysicalProject
1313
--------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->[cs_sold_date_sk]
1414
----------------------PhysicalProject
15-
------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF1 RF2
15+
------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF1
1616
----------------------PhysicalProject
1717
------------------------filter((date_dim.d_qoy = 1) and (date_dim.d_year = 2001))
1818
--------------------------PhysicalOlapScan[date_dim]
1919
------------------PhysicalProject
20-
--------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((customer.c_current_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF0 c_current_addr_sk->[ca_address_sk]
21-
----------------------PhysicalProject
22-
------------------------PhysicalOlapScan[customer_address] apply RFs: RF0
20+
--------------------hashJoin[INNER_JOIN shuffle] hashCondition=((customer.c_current_addr_sk = customer_address.ca_address_sk)) otherCondition=()
2321
----------------------PhysicalProject
2422
------------------------PhysicalOlapScan[customer]
23+
----------------------PhysicalProject
24+
------------------------PhysicalOlapScan[customer_address]
2525

‎regression-test/data/shape_check/tpcds_sf100/rf_prune/query18.out‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ PhysicalResultSink
1010
--------------hashAgg[LOCAL]
1111
----------------PhysicalRepeat
1212
------------------PhysicalProject
13-
--------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((catalog_sales.cs_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF5 cs_item_sk->[i_item_sk]
13+
--------------------hashJoin[INNER_JOIN shuffle] hashCondition=((catalog_sales.cs_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF5 cs_item_sk->[i_item_sk]
1414
----------------------PhysicalProject
1515
------------------------PhysicalOlapScan[item] apply RFs: RF5
1616
----------------------PhysicalProject

‎regression-test/data/shape_check/tpcds_sf100/rf_prune/query19.out‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ PhysicalResultSink
1111
----------------PhysicalProject
1212
------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_store_sk = store.s_store_sk)) otherCondition=(( not (substring(ca_zip, 1, 5) = substring(s_zip, 1, 5))))
1313
--------------------PhysicalProject
14-
----------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((customer.c_current_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF3 c_current_addr_sk->[ca_address_sk]
14+
----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer.c_current_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF3 c_current_addr_sk->[ca_address_sk]
1515
------------------------PhysicalProject
1616
--------------------------PhysicalOlapScan[customer_address] apply RFs: RF3
1717
------------------------PhysicalProject

‎regression-test/data/shape_check/tpcds_sf100/rf_prune/query24.out‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -11,20 +11,20 @@ PhysicalCteAnchor ( cteId=CTEId#0 )
1111
----------------PhysicalProject
1212
------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_item_sk = item.i_item_sk)) otherCondition=()
1313
--------------------PhysicalProject
14-
----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store.s_zip = customer_address.ca_zip) and (store_sales.ss_customer_sk = customer.c_customer_sk)) otherCondition=() build RFs:RF2 ca_zip->[s_zip];RF3 c_customer_sk->[ss_customer_sk]
14+
----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store.s_zip = customer_address.ca_zip) and (store_sales.ss_customer_sk = customer.c_customer_sk)) otherCondition=()
1515
------------------------PhysicalProject
1616
--------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_store_sk = store.s_store_sk)) otherCondition=() build RFs:RF1 s_store_sk->[ss_store_sk]
1717
----------------------------PhysicalProject
18-
------------------------------PhysicalOlapScan[store_sales] apply RFs: RF1 RF3
18+
------------------------------PhysicalOlapScan[store_sales] apply RFs: RF1
1919
----------------------------PhysicalProject
2020
------------------------------filter((store.s_market_id = 8))
21-
--------------------------------PhysicalOlapScan[store] apply RFs: RF2
21+
--------------------------------PhysicalOlapScan[store]
2222
------------------------PhysicalProject
23-
--------------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((customer.c_current_addr_sk = customer_address.ca_address_sk)) otherCondition=(( not (c_birth_country = upper(ca_country)))) build RFs:RF0 c_current_addr_sk->[ca_address_sk]
24-
----------------------------PhysicalProject
25-
------------------------------PhysicalOlapScan[customer_address] apply RFs: RF0
23+
--------------------------hashJoin[INNER_JOIN shuffle] hashCondition=((customer.c_current_addr_sk = customer_address.ca_address_sk)) otherCondition=(( not (c_birth_country = upper(ca_country))))
2624
----------------------------PhysicalProject
2725
------------------------------PhysicalOlapScan[customer]
26+
----------------------------PhysicalProject
27+
------------------------------PhysicalOlapScan[customer_address]
2828
--------------------PhysicalProject
2929
----------------------PhysicalOlapScan[item]
3030
----------------PhysicalProject

‎regression-test/data/shape_check/tpcds_sf100/rf_prune/query26.out‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -8,9 +8,7 @@ PhysicalResultSink
88
----------PhysicalDistribute[DistributionSpecHash]
99
------------hashAgg[LOCAL]
1010
--------------PhysicalProject
11-
----------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((catalog_sales.cs_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF3 cs_item_sk->[i_item_sk]
12-
------------------PhysicalProject
13-
--------------------PhysicalOlapScan[item] apply RFs: RF3
11+
----------------hashJoin[INNER_JOIN shuffle] hashCondition=((catalog_sales.cs_item_sk = item.i_item_sk)) otherCondition=()
1412
------------------PhysicalProject
1513
--------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_promo_sk = promotion.p_promo_sk)) otherCondition=() build RFs:RF2 p_promo_sk->[cs_promo_sk]
1614
----------------------PhysicalProject
@@ -28,4 +26,6 @@ PhysicalResultSink
2826
----------------------PhysicalProject
2927
------------------------filter(OR[(promotion.p_channel_email = 'N'),(promotion.p_channel_event = 'N')])
3028
--------------------------PhysicalOlapScan[promotion]
29+
------------------PhysicalProject
30+
--------------------PhysicalOlapScan[item]
3131

‎regression-test/data/shape_check/tpcds_sf100/rf_prune/query29.out‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -16,9 +16,7 @@ PhysicalResultSink
1616
----------------------PhysicalProject
1717
------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store.s_store_sk = store_sales.ss_store_sk)) otherCondition=()
1818
--------------------------PhysicalProject
19-
----------------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((item.i_item_sk = store_sales.ss_item_sk)) otherCondition=() build RFs:RF5 ss_item_sk->[i_item_sk]
20-
------------------------------PhysicalProject
21-
--------------------------------PhysicalOlapScan[item] apply RFs: RF5
19+
----------------------------hashJoin[INNER_JOIN shuffle] hashCondition=((item.i_item_sk = store_sales.ss_item_sk)) otherCondition=()
2220
------------------------------PhysicalProject
2321
--------------------------------hashJoin[INNER_JOIN colocated] hashCondition=((store_sales.ss_customer_sk = store_returns.sr_customer_sk) and (store_sales.ss_item_sk = store_returns.sr_item_sk) and (store_sales.ss_ticket_number = store_returns.sr_ticket_number)) otherCondition=() build RFs:RF2 sr_customer_sk->[ss_customer_sk];RF3 sr_item_sk->[ss_item_sk];RF4 sr_ticket_number->[ss_ticket_number]
2422
----------------------------------PhysicalProject
@@ -35,6 +33,8 @@ PhysicalResultSink
3533
--------------------------------------PhysicalProject
3634
----------------------------------------filter((d2.d_moy <= 7) and (d2.d_moy >= 4) and (d2.d_year = 1999))
3735
------------------------------------------PhysicalOlapScan[date_dim]
36+
------------------------------PhysicalProject
37+
--------------------------------PhysicalOlapScan[item]
3838
--------------------------PhysicalProject
3939
----------------------------PhysicalOlapScan[store]
4040
------------------PhysicalProject

‎regression-test/data/shape_check/tpcds_sf100/rf_prune/query30.out‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -7,16 +7,16 @@ PhysicalCteAnchor ( cteId=CTEId#0 )
77
--------PhysicalDistribute[DistributionSpecHash]
88
----------hashAgg[LOCAL]
99
------------PhysicalProject
10-
--------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((web_returns.wr_returning_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF1 wr_returning_addr_sk->[ca_address_sk]
11-
----------------PhysicalProject
12-
------------------PhysicalOlapScan[customer_address] apply RFs: RF1
10+
--------------hashJoin[INNER_JOIN shuffle] hashCondition=((web_returns.wr_returning_addr_sk = customer_address.ca_address_sk)) otherCondition=()
1311
----------------PhysicalProject
1412
------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_returns.wr_returned_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->[wr_returned_date_sk]
1513
--------------------PhysicalProject
1614
----------------------PhysicalOlapScan[web_returns] apply RFs: RF0
1715
--------------------PhysicalProject
1816
----------------------filter((date_dim.d_year = 2002))
1917
------------------------PhysicalOlapScan[date_dim]
18+
----------------PhysicalProject
19+
------------------PhysicalOlapScan[customer_address]
2020
--PhysicalResultSink
2121
----PhysicalProject
2222
------PhysicalLazyMaterialize[materializedSlots:(customer.c_customer_id,ctr1.ctr_total_return) lazySlots:(customer.c_birth_country,customer.c_birth_day,customer.c_birth_month,customer.c_birth_year,customer.c_email_address,customer.c_first_name,customer.c_last_name,customer.c_last_review_date_sk,customer.c_login,customer.c_preferred_cust_flag,customer.c_salutation)]

‎regression-test/data/shape_check/tpcds_sf100/rf_prune/query33.out‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -62,7 +62,7 @@ PhysicalResultSink
6262
----------------------PhysicalDistribute[DistributionSpecHash]
6363
------------------------hashAgg[LOCAL]
6464
--------------------------PhysicalProject
65-
----------------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((web_sales.ws_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF10 ws_item_sk->[i_item_sk]
65+
----------------------------hashJoin[INNER_JOIN shuffle] hashCondition=((web_sales.ws_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF10 ws_item_sk->[i_item_sk]
6666
------------------------------PhysicalProject
6767
--------------------------------PhysicalOlapScan[item] apply RFs: RF10 RF11
6868
------------------------------PhysicalProject

0 commit comments

Comments
 (0)