Skip to content

Commit 41cd2b4

Browse files
[fix](eager-agg) Disable bilateral eager aggregation for decimal SUM (apache#69093)
### What problem does this PR solve? Related PR: apache#63690 Problem Summary: Bilateral eager aggregation can change the result type and scale of a decimal SUM because its compensation uses ordinary decimal multiplication. This can cause an output type mismatch or silently lose precision; for example, a valid result of `0.0000000004` can become `0.000000`. This PR disables bilateral eager aggregation when any aggregate is a DecimalV3 SUM. Single-sided decimal SUM pushdown remains supported. This causes either: - An output type mismatch when an upper projection still expects the original SUM type. - Silent precision loss when the reduced-scale result reaches the query output. For example, a valid SUM of `0.0000000004` could become `0.000000`. This Pr fix this problem: - Introduce an internal decimal multiplication expression that preserves the SUM result type. - Cast the compensation input to the original SUM type when necessary. - Preserve multiplication statistics estimation and prevent arithmetic simplification or FE constant folding from reverting to ordinary decimal multiplication semantics. - Reuse the existing BE multiplication and overflow checks. No BE changes are required.
1 parent f1bc8b3 commit 41cd2b4

5 files changed

Lines changed: 289 additions & 18 deletions

File tree

‎fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/eageraggregation/EagerAggRewriter.java‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@
3535
import org.apache.doris.nereids.trees.expressions.functions.agg.Count;
3636
import org.apache.doris.nereids.trees.expressions.functions.agg.Max;
3737
import org.apache.doris.nereids.trees.expressions.functions.agg.Min;
38+
import org.apache.doris.nereids.trees.expressions.functions.agg.Sum;
3839
import org.apache.doris.nereids.trees.expressions.functions.scalar.If;
3940
import org.apache.doris.nereids.trees.expressions.functions.scalar.Nvl;
4041
import org.apache.doris.nereids.trees.expressions.literal.BigIntLiteral;
@@ -261,6 +262,12 @@ private Pair<Boolean, Boolean> decideJoinPushSide(
261262
return adjustPushSideForNullToNonNull(join, context, toLeft, toRight);
262263
}
263264
if (toLeft && toRight) {
265+
// Decimal SUM compensation can lose scale with ordinary multiplication, while preserving
266+
// its type can reject intermediate products that would cancel in the final SUM.
267+
if (context.getAggFunctions().stream()
268+
.anyMatch(func -> func instanceof Sum && func.getDataType().isDecimalV3Type())) {
269+
return Pair.of(false, false);
270+
}
264271
return join.getJoinType().isInnerOrCrossJoin()
265272
? Pair.of(true, true)
266273
: Pair.of(false, false);

‎fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/eageraggregation/EagerAggRewriterTest.java‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@
3232
import org.apache.doris.nereids.trees.plans.logical.LogicalJoin;
3333
import org.apache.doris.nereids.trees.plans.logical.LogicalRelation;
3434
import org.apache.doris.nereids.types.BigIntType;
35+
import org.apache.doris.nereids.types.DecimalV3Type;
3536
import org.apache.doris.nereids.util.MemoPatternMatchSupported;
3637
import org.apache.doris.nereids.util.PlanChecker;
3738
import org.apache.doris.utframe.TestWithFeService;
@@ -74,6 +75,62 @@ protected void runBeforeAll() throws Exception {
7475
connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION");
7576
}
7677

78+
@Test
79+
void testDecimalSumDisablesBilateralPushDown() throws Exception {
80+
createTable("CREATE TABLE decimal_sum_l (k VARCHAR(20), v DECIMAL(38,18))"
81+
+ " DUPLICATE KEY(k) DISTRIBUTED BY HASH(k) BUCKETS 1"
82+
+ " PROPERTIES ('replication_num'='1')");
83+
createTable("CREATE TABLE decimal_sum_r (k VARCHAR(20), w BIGINT)"
84+
+ " DUPLICATE KEY(k) DISTRIBUTED BY HASH(k) BUCKETS 1"
85+
+ " PROPERTIES ('replication_num'='1')");
86+
int oldMode = connectContext.getSessionVariable().getEagerAggregationMode();
87+
boolean oldReorder = connectContext.getSessionVariable().isDisableJoinReorder();
88+
boolean oldDecimal256 = connectContext.getSessionVariable().enableDecimal256;
89+
try {
90+
connectContext.getSessionVariable().setEagerAggregationMode(1);
91+
connectContext.getSessionVariable().setDisableJoinReorder(true);
92+
for (boolean decimal256 : new boolean[] {false, true}) {
93+
connectContext.getSessionVariable().enableDecimal256 = decimal256;
94+
DecimalV3Type expected = DecimalV3Type.createDecimalV3Type(decimal256 ? 76 : 38, 18);
95+
for (String join : new String[] {"JOIN decimal_sum_r r ON l.k=r.k",
96+
"CROSS JOIN decimal_sum_r r"}) {
97+
for (String sums : new String[] {"SUM(l.v) AS sv, SUM(r.w) AS sw",
98+
"SUM(r.w) AS sw, SUM(l.v) AS sv"}) {
99+
Plan plan = PlanChecker.from(connectContext)
100+
.analyze("SELECT l.k AS g, " + sums + " FROM decimal_sum_l l "
101+
+ join + " GROUP BY l.k")
102+
.rewrite().getPlan();
103+
int decimalOutputIndex = sums.startsWith("SUM(l.v)") ? 1 : 2;
104+
Assertions.assertEquals(expected, plan.getOutput().get(decimalOutputIndex).getDataType());
105+
LogicalJoin<?, ?> logicalJoin = findFirstPlan(plan, LogicalJoin.class);
106+
Assertions.assertNotNull(logicalJoin, plan.treeString());
107+
Assertions.assertFalse(containsPlan(logicalJoin, LogicalAggregate.class), plan.treeString());
108+
}
109+
}
110+
assertNoAggregateUnderJoin("SELECT SUBSTRING(l.k, 1, 6), SUM(l.v), SUM(r.w)"
111+
+ " FROM decimal_sum_l l JOIN decimal_sum_r r ON l.k=r.k"
112+
+ " GROUP BY SUBSTRING(l.k, 1, 6)");
113+
assertNoAggregateUnderJoin("SELECT r.k, SUM(l.v), COUNT(r.w)"
114+
+ " FROM decimal_sum_r r JOIN decimal_sum_l l ON l.k=r.k GROUP BY r.k");
115+
// A decimal SUM on just one side still supports eager aggregation.
116+
for (String from : new String[] {"decimal_sum_l l JOIN decimal_sum_r r ON l.k=r.k",
117+
"decimal_sum_r r JOIN decimal_sum_l l ON l.k=r.k"}) {
118+
Plan plan = PlanChecker.from(connectContext)
119+
.analyze("SELECT l.k, SUM(l.v) FROM " + from + " GROUP BY l.k")
120+
.rewrite().getPlan();
121+
Assertions.assertEquals(expected, plan.getOutput().get(1).getDataType());
122+
LogicalJoin<?, ?> logicalJoin = findFirstPlan(plan, LogicalJoin.class);
123+
Assertions.assertNotNull(logicalJoin, plan.treeString());
124+
Assertions.assertTrue(containsPlan(logicalJoin, LogicalAggregate.class), plan.treeString());
125+
}
126+
}
127+
} finally {
128+
connectContext.getSessionVariable().setEagerAggregationMode(oldMode);
129+
connectContext.getSessionVariable().setDisableJoinReorder(oldReorder);
130+
connectContext.getSessionVariable().enableDecimal256 = oldDecimal256;
131+
}
132+
}
133+
77134
@Test
78135
void testNotPushAggCaseWhenToNullableSideOfOuterJoin() {
79136
connectContext.getSessionVariable().setEagerAggregationMode(1);
Lines changed: 133 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,133 @@
1+
-- This file is automatically generated. You should know what you did if you want to edit this
2+
-- !alias_false_false_off --
3+
negative -2.246913578024691356 6
4+
null \N 6
5+
x 4.00000000E-10 4
6+
7+
-- !no_alias_false_false_off --
8+
negative -2.246913578024691356 6
9+
null \N 6
10+
x 4.00000000E-10 4
11+
12+
-- !insert_false_false_off --
13+
negative -2.246913578024691356 6
14+
null \N 6
15+
x 4.00000000E-10 4
16+
17+
-- !alias_false_false_on --
18+
negative -2.246913578024691356 6
19+
null \N 6
20+
x 4.00000000E-10 4
21+
22+
-- !no_alias_false_false_on --
23+
negative -2.246913578024691356 6
24+
null \N 6
25+
x 4.00000000E-10 4
26+
27+
-- !insert_false_false_on --
28+
negative -2.246913578024691356 6
29+
null \N 6
30+
x 4.00000000E-10 4
31+
32+
-- !alias_false_true_off --
33+
negative -2.246913578024691356 6
34+
null \N 6
35+
x 4.00000000E-10 4
36+
37+
-- !no_alias_false_true_off --
38+
negative -2.246913578024691356 6
39+
null \N 6
40+
x 4.00000000E-10 4
41+
42+
-- !insert_false_true_off --
43+
negative -2.246913578024691356 6
44+
null \N 6
45+
x 4.00000000E-10 4
46+
47+
-- !alias_false_true_on --
48+
negative -2.246913578024691356 6
49+
null \N 6
50+
x 4.00000000E-10 4
51+
52+
-- !no_alias_false_true_on --
53+
negative -2.246913578024691356 6
54+
null \N 6
55+
x 4.00000000E-10 4
56+
57+
-- !insert_false_true_on --
58+
negative -2.246913578024691356 6
59+
null \N 6
60+
x 4.00000000E-10 4
61+
62+
-- !alias_true_false_off --
63+
negative -2.246913578024691356 6
64+
null \N 6
65+
x 4.00000000E-10 4
66+
67+
-- !no_alias_true_false_off --
68+
negative -2.246913578024691356 6
69+
null \N 6
70+
x 4.00000000E-10 4
71+
72+
-- !insert_true_false_off --
73+
negative -2.246913578024691356 6
74+
null \N 6
75+
x 4.00000000E-10 4
76+
77+
-- !alias_true_false_on --
78+
negative -2.246913578024691356 6
79+
null \N 6
80+
x 4.00000000E-10 4
81+
82+
-- !no_alias_true_false_on --
83+
negative -2.246913578024691356 6
84+
null \N 6
85+
x 4.00000000E-10 4
86+
87+
-- !insert_true_false_on --
88+
negative -2.246913578024691356 6
89+
null \N 6
90+
x 4.00000000E-10 4
91+
92+
-- !alias_true_true_off --
93+
negative -2.246913578024691356 6
94+
null \N 6
95+
x 4.00000000E-10 4
96+
97+
-- !no_alias_true_true_off --
98+
negative -2.246913578024691356 6
99+
null \N 6
100+
x 4.00000000E-10 4
101+
102+
-- !insert_true_true_off --
103+
negative -2.246913578024691356 6
104+
null \N 6
105+
x 4.00000000E-10 4
106+
107+
-- !alias_true_true_on --
108+
negative -2.246913578024691356 6
109+
null \N 6
110+
x 4.00000000E-10 4
111+
112+
-- !no_alias_true_true_on --
113+
negative -2.246913578024691356 6
114+
null \N 6
115+
x 4.00000000E-10 4
116+
117+
-- !insert_true_true_on --
118+
negative -2.246913578024691356 6
119+
null \N 6
120+
x 4.00000000E-10 4
121+
122+
-- !cancellation_false_off --
123+
cancel 0E-18 4
124+
125+
-- !cancellation_false_on --
126+
cancel 0E-18 4
127+
128+
-- !cancellation_true_off --
129+
cancel 0E-18 4
130+
131+
-- !cancellation_true_on --
132+
cancel 0E-18 4
133+

‎regression-test/data/query_p0/eager_agg/eager_agg.out‎

Lines changed: 6 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -5,12 +5,8 @@ PhysicalResultSink
55
----hashAgg[LOCAL]
66
------hashJoin[INNER_JOIN] hashCondition=((dt.d_date_sk = ss.ss_sold_date_sk)) otherCondition=()
77
--------hashJoin[INNER_JOIN] hashCondition=((ss.ss_item_sk = ws.ws_item_sk)) otherCondition=()
8-
----------hashAgg[GLOBAL]
9-
------------hashAgg[LOCAL]
10-
--------------PhysicalOlapScan[store_sales(ss)]
11-
----------hashAgg[GLOBAL]
12-
------------hashAgg[LOCAL]
13-
--------------PhysicalOlapScan[web_sales(ws)]
8+
----------PhysicalOlapScan[store_sales(ss)]
9+
----------PhysicalOlapScan[web_sales(ws)]
1410
--------PhysicalOlapScan[date_dim(dt)]
1511

1612
Hint log:
@@ -47,12 +43,8 @@ PhysicalResultSink
4743
----hashAgg[LOCAL]
4844
------hashJoin[INNER_JOIN] hashCondition=((dt.d_date_sk = ss.ss_sold_date_sk)) otherCondition=()
4945
--------hashJoin[INNER_JOIN] hashCondition=((ss.ss_item_sk = ws.ws_item_sk)) otherCondition=()
50-
----------hashAgg[GLOBAL]
51-
------------hashAgg[LOCAL]
52-
--------------PhysicalOlapScan[store_sales(ss)]
53-
----------hashAgg[GLOBAL]
54-
------------hashAgg[LOCAL]
55-
--------------PhysicalOlapScan[web_sales(ws)]
46+
----------PhysicalOlapScan[store_sales(ss)]
47+
----------PhysicalOlapScan[web_sales(ws)]
5648
--------PhysicalOlapScan[date_dim(dt)]
5749

5850
Hint log:
@@ -131,12 +123,8 @@ PhysicalResultSink
131123
----hashAgg[LOCAL]
132124
------hashJoin[INNER_JOIN] hashCondition=((dt.d_date_sk = ss.ss_sold_date_sk)) otherCondition=()
133125
--------hashJoin[INNER_JOIN] hashCondition=((ss.ss_item_sk = ws.ws_item_sk)) otherCondition=()
134-
----------hashAgg[GLOBAL]
135-
------------hashAgg[LOCAL]
136-
--------------PhysicalOlapScan[store_sales(ss)]
137-
----------hashAgg[GLOBAL]
138-
------------hashAgg[LOCAL]
139-
--------------PhysicalOlapScan[web_sales(ws)]
126+
----------PhysicalOlapScan[store_sales(ss)]
127+
----------PhysicalOlapScan[web_sales(ws)]
140128
--------PhysicalOlapScan[date_dim(dt)]
141129

142130
Hint log:
Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
suite("decimal_sum_multiplicity") {
19+
sql """
20+
DROP TABLE IF EXISTS decimal_sum_l;
21+
DROP TABLE IF EXISTS decimal_sum_r;
22+
DROP TABLE IF EXISTS decimal_sum_out;
23+
CREATE TABLE decimal_sum_l (k VARCHAR(20) NOT NULL, v DECIMAL(38,18) NULL)
24+
DUPLICATE KEY(k) DISTRIBUTED BY HASH(k) BUCKETS 1
25+
PROPERTIES ("replication_num"="1");
26+
CREATE TABLE decimal_sum_r (k VARCHAR(20) NOT NULL, w BIGINT NULL)
27+
DUPLICATE KEY(k) DISTRIBUTED BY HASH(k) BUCKETS 1
28+
PROPERTIES ("replication_num"="1");
29+
CREATE TABLE decimal_sum_out (k VARCHAR(20), sv DECIMAL(38,18), sw BIGINT)
30+
DUPLICATE KEY(k) DISTRIBUTED BY HASH(k) BUCKETS 1
31+
PROPERTIES ("replication_num"="1");
32+
INSERT INTO decimal_sum_l VALUES
33+
('x', 0.000000000100000000), ('x', 0.000000000100000000),
34+
('negative', -1.123456789012345678), ('negative', NULL),
35+
('null', NULL), ('null', NULL);
36+
INSERT INTO decimal_sum_r VALUES
37+
('x', 1), ('x', 1), ('negative', 1), ('negative', 2), ('null', 3);
38+
"""
39+
sql "SET disable_join_reorder=true"
40+
sql "SET decimal_overflow_scale=6"
41+
for (def decimal256 : [false, true]) {
42+
sql "SET enable_decimal256=${decimal256}"
43+
for (def checkOverflow : [false, true]) {
44+
sql "SET check_overflow_for_decimal=${checkOverflow}"
45+
for (def mode : [-1, 1]) {
46+
sql "SET eager_aggregation_mode=${mode}"
47+
def suffix = "${decimal256}_${checkOverflow}_${mode == -1 ? 'off' : 'on'}"
48+
// The alias used to expose the stale output type; without it, scale could be lost silently.
49+
"order_qt_alias_${suffix}" """
50+
SELECT l.k AS g, SUM(l.v) AS sv, SUM(r.w) AS sw
51+
FROM decimal_sum_l l JOIN [shuffle] decimal_sum_r r ON l.k=r.k GROUP BY l.k
52+
"""
53+
"order_qt_no_alias_${suffix}" """
54+
SELECT l.k, SUM(l.v), SUM(r.w)
55+
FROM decimal_sum_l l JOIN [shuffle] decimal_sum_r r ON l.k=r.k GROUP BY l.k
56+
"""
57+
sql """
58+
INSERT OVERWRITE TABLE decimal_sum_out
59+
SELECT l.k AS g, CAST(SUM(l.v) AS DECIMAL(38,18)), SUM(r.w)
60+
FROM decimal_sum_l l JOIN [shuffle] decimal_sum_r r ON l.k=r.k GROUP BY l.k
61+
"""
62+
"order_qt_insert_${suffix}" "SELECT * FROM decimal_sum_out"
63+
}
64+
}
65+
}
66+
// The compensation products exceed DECIMAL(38,18), but fit Int128 and cancel in the final SUM.
67+
sql """INSERT INTO decimal_sum_l VALUES
68+
('cancel_positive', 60000000000000000000.000000000000000000),
69+
('cancel_negative', -60000000000000000000.000000000000000000)"""
70+
sql """INSERT INTO decimal_sum_r VALUES
71+
('cancel_positive', 1), ('cancel_positive', 1),
72+
('cancel_negative', 1), ('cancel_negative', 1)"""
73+
for (def decimal256 : [false, true]) {
74+
sql "SET enable_decimal256=${decimal256}"
75+
sql "SET check_overflow_for_decimal=true"
76+
for (def mode : [-1, 1]) {
77+
sql "SET eager_aggregation_mode=${mode}"
78+
"order_qt_cancellation_${decimal256}_${mode == -1 ? 'off' : 'on'}" """
79+
SELECT SUBSTRING(l.k, 1, 6), SUM(l.v), SUM(r.w)
80+
FROM decimal_sum_l l JOIN [shuffle] decimal_sum_r r ON l.k=r.k
81+
WHERE l.k IN ('cancel_positive', 'cancel_negative')
82+
GROUP BY SUBSTRING(l.k, 1, 6)
83+
"""
84+
}
85+
}
86+
}

0 commit comments

Comments
 (0)