Skip to content

Commit f80495d

Browse files
authored
[fix](Nereids) ban right outer, right anti, full outer with bucket shuffle (apache#26529)
if left bucket has no data, we do not generate left bucket instance. These join should reserve all right side data. But because left instance is not exists. So right data will be discard since no dest be set. We ban these join temporarily until we could generate all instance for left side in Coordinator.
1 parent 5d45579 commit f80495d

5 files changed

Lines changed: 56 additions & 10 deletions

File tree

‎fe/fe-core/src/main/java/org/apache/doris/nereids/properties/ChildrenPropertiesRegulator.java‎

Lines changed: 22 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@
3030
import org.apache.doris.nereids.trees.expressions.SlotReference;
3131
import org.apache.doris.nereids.trees.expressions.functions.agg.MultiDistinction;
3232
import org.apache.doris.nereids.trees.plans.AggMode;
33+
import org.apache.doris.nereids.trees.plans.JoinType;
3334
import org.apache.doris.nereids.trees.plans.Plan;
3435
import org.apache.doris.nereids.trees.plans.SortPhase;
3536
import org.apache.doris.nereids.trees.plans.physical.AbstractPhysicalSort;
@@ -178,6 +179,12 @@ public Boolean visitPhysicalFilter(PhysicalFilter<? extends Plan> filter, Void c
178179
return true;
179180
}
180181

182+
private boolean couldNotRightBucketShuffleJoin(JoinType joinType) {
183+
return joinType == JoinType.RIGHT_ANTI_JOIN
184+
|| joinType == JoinType.RIGHT_OUTER_JOIN
185+
|| joinType == JoinType.FULL_OUTER_JOIN;
186+
}
187+
181188
@Override
182189
public Boolean visitPhysicalHashJoin(PhysicalHashJoin<? extends Plan, ? extends Plan> hashJoin,
183190
Void context) {
@@ -207,12 +214,22 @@ public Boolean visitPhysicalHashJoin(PhysicalHashJoin<? extends Plan, ? extends
207214
Optional<PhysicalProperties> updatedForLeft = Optional.empty();
208215
Optional<PhysicalProperties> updatedForRight = Optional.empty();
209216

210-
if ((leftHashSpec.getShuffleType() == ShuffleType.NATURAL
211-
&& rightHashSpec.getShuffleType() == ShuffleType.NATURAL)) {
217+
if (JoinUtils.couldColocateJoin(leftHashSpec, rightHashSpec)) {
212218
// check colocate join with scan
213-
if (JoinUtils.couldColocateJoin(leftHashSpec, rightHashSpec)) {
214-
return true;
215-
}
219+
return true;
220+
} else if (couldNotRightBucketShuffleJoin(hashJoin.getJoinType())) {
221+
// right anti, right outer, full outer join could not do bucket shuffle join
222+
// TODO remove this after we refactor coordinator
223+
updatedForLeft = Optional.of(calAnotherSideRequired(
224+
ShuffleType.EXECUTION_BUCKETED, leftHashSpec, leftHashSpec,
225+
(DistributionSpecHash) requiredProperties.get(0).getDistributionSpec(),
226+
(DistributionSpecHash) requiredProperties.get(0).getDistributionSpec()));
227+
updatedForRight = Optional.of(calAnotherSideRequired(
228+
ShuffleType.EXECUTION_BUCKETED, leftHashSpec, rightHashSpec,
229+
(DistributionSpecHash) requiredProperties.get(0).getDistributionSpec(),
230+
(DistributionSpecHash) requiredProperties.get(1).getDistributionSpec()));
231+
} else if ((leftHashSpec.getShuffleType() == ShuffleType.NATURAL
232+
&& rightHashSpec.getShuffleType() == ShuffleType.NATURAL)) {
216233
updatedForRight = Optional.of(calAnotherSideRequired(
217234
ShuffleType.STORAGE_BUCKETED, leftHashSpec, rightHashSpec,
218235
(DistributionSpecHash) requiredProperties.get(0).getDistributionSpec(),

‎regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query72.out‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,8 +17,9 @@ PhysicalResultSink
1717
------------------------PhysicalProject
1818
--------------------------hashJoin[INNER_JOIN] hashCondition=((d1.d_week_seq = d2.d_week_seq))otherCondition=()
1919
----------------------------hashJoin[RIGHT_OUTER_JOIN] hashCondition=((catalog_returns.cr_item_sk = catalog_sales.cs_item_sk) and (catalog_returns.cr_order_number = catalog_sales.cs_order_number))otherCondition=()
20-
------------------------------PhysicalProject
21-
--------------------------------PhysicalOlapScan[catalog_returns]
20+
------------------------------PhysicalDistribute
21+
--------------------------------PhysicalProject
22+
----------------------------------PhysicalOlapScan[catalog_returns]
2223
------------------------------PhysicalDistribute
2324
--------------------------------PhysicalProject
2425
----------------------------------hashJoin[INNER_JOIN] hashCondition=((item.i_item_sk = catalog_sales.cs_item_sk))otherCondition=()

‎regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query72.out‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,8 +18,9 @@ PhysicalResultSink
1818
--------------------------hashJoin[INNER_JOIN] hashCondition=((d1.d_week_seq = d2.d_week_seq))otherCondition=()
1919
----------------------------PhysicalDistribute
2020
------------------------------hashJoin[RIGHT_OUTER_JOIN] hashCondition=((catalog_returns.cr_item_sk = catalog_sales.cs_item_sk) and (catalog_returns.cr_order_number = catalog_sales.cs_order_number))otherCondition=()
21-
--------------------------------PhysicalProject
22-
----------------------------------PhysicalOlapScan[catalog_returns]
21+
--------------------------------PhysicalDistribute
22+
----------------------------------PhysicalProject
23+
------------------------------------PhysicalOlapScan[catalog_returns]
2324
--------------------------------PhysicalDistribute
2425
----------------------------------PhysicalProject
2526
------------------------------------hashJoin[LEFT_OUTER_JOIN] hashCondition=((catalog_sales.cs_promo_sk = promotion.p_promo_sk))otherCondition=()

‎regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query80.out‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -62,7 +62,8 @@ PhysicalResultSink
6262
--------------------------hashAgg[LOCAL]
6363
----------------------------PhysicalProject
6464
------------------------------hashJoin[RIGHT_OUTER_JOIN] hashCondition=((web_sales.ws_item_sk = web_returns.wr_item_sk) and (web_sales.ws_order_number = web_returns.wr_order_number))otherCondition=()
65-
--------------------------------PhysicalOlapScan[web_returns]
65+
--------------------------------PhysicalDistribute
66+
----------------------------------PhysicalOlapScan[web_returns]
6667
--------------------------------PhysicalDistribute
6768
----------------------------------hashJoin[INNER_JOIN] hashCondition=((web_sales.ws_web_site_sk = web_site.web_site_sk))otherCondition=()
6869
------------------------------------hashJoin[INNER_JOIN] hashCondition=((web_sales.ws_item_sk = item.i_item_sk))otherCondition=()

‎regression-test/suites/nereids_p0/join/test_outer_join.groovy‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ suite("test_outer_join", "nereids_p0") {
2020
sql "SET enable_fallback_to_original_planner=false"
2121
def tbl1 = "test_outer_join1"
2222
def tbl2 = "test_outer_join2"
23+
def tbl3 = "test_outer_join3"
2324

2425
sql "DROP TABLE IF EXISTS ${tbl1}"
2526
sql """
@@ -37,6 +38,15 @@ suite("test_outer_join", "nereids_p0") {
3738
DISTRIBUTED BY RANDOM BUCKETS 30
3839
PROPERTIES ("replication_num" = "1");
3940
"""
41+
42+
sql "DROP TABLE IF EXISTS ${tbl3}"
43+
sql """
44+
CREATE TABLE IF NOT EXISTS ${tbl3} (
45+
c0 DECIMALV3(8,3)
46+
)
47+
DISTRIBUTED BY HASH (c0) BUCKETS 1 PROPERTIES ("replication_num" = "1");
48+
"""
49+
4050
sql """INSERT INTO ${tbl2} (c0) VALUES ('dr'), ('x7Tq'), ('');"""
4151
sql """INSERT INTO ${tbl1} (c0) VALUES (0.47683432698249817), (0.8864791393280029);"""
4252
sql """INSERT INTO ${tbl1} (c0) VALUES (0.11287713050842285);"""
@@ -56,6 +66,22 @@ suite("test_outer_join", "nereids_p0") {
5666
qt_join """
5767
SELECT * FROM ${tbl2} LEFT OUTER JOIN ${tbl1} ON (('') like ('15DScmSM')) WHERE ('abc' NOT LIKE 'abc');
5868
"""
69+
70+
sql "set disable_join_reorder=true"
71+
explain {
72+
sql "SELECT * FROM ${tbl1} RIGHT OUTER JOIN ${tbl3} ON ${tbl1}.c0 = ${tbl3}.c0"
73+
contains "RIGHT OUTER JOIN(PARTITIONED)"
74+
}
75+
explain {
76+
sql "SELECT * FROM ${tbl1} RIGHT ANTI JOIN ${tbl3} ON ${tbl1}.c0 = ${tbl3}.c0"
77+
contains "RIGHT ANTI JOIN(PARTITIONED)"
78+
}
79+
explain {
80+
sql "SELECT * FROM ${tbl1} FULL OUTER JOIN ${tbl3} ON ${tbl1}.c0 = ${tbl3}.c0"
81+
contains "FULL OUTER JOIN(PARTITIONED)"
82+
}
83+
5984
sql "DROP TABLE IF EXISTS ${tbl1}"
6085
sql "DROP TABLE IF EXISTS ${tbl2}"
86+
sql "DROP TABLE IF EXISTS ${tbl3}"
6187
}

0 commit comments

Comments
 (0)