Skip to content

Commit d921beb

Browse files
[fix](agg) Normalize equivalent multi-column distinct counts (apache#65206)
### What problem does this PR solve? Related PR: apache#54079 Problem Summary: NormalizeAggregate treated reordered equivalent multi-column COUNT DISTINCT expressions as different aggregate functions, which could make the multi-distinct strategy construct a join from only one aggregate and fail with an index-out-of-bounds exception. It also retained duplicate arguments even though downstream COUNT DISTINCT processing treats them as a set, and the CountIf conversion indexed the original argument list using the deduplicated size. Canonicalize multi-column distinct arguments as an ordered set during normalization and make CountIf consume that deduplicated list directly. ### Release note Fix planning and execution of equivalent multi-column COUNT DISTINCT expressions with reordered or duplicate arguments.
1 parent f2f2a57 commit d921beb

5 files changed

Lines changed: 162 additions & 16 deletions

File tree

‎fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/NormalizeAggregate.java‎

Lines changed: 25 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@
4040
import org.apache.doris.nereids.trees.expressions.WindowExpression;
4141
import org.apache.doris.nereids.trees.expressions.functions.agg.AggregateFunction;
4242
import org.apache.doris.nereids.trees.expressions.functions.agg.AnyValue;
43+
import org.apache.doris.nereids.trees.expressions.functions.agg.Count;
4344
import org.apache.doris.nereids.trees.expressions.functions.generator.Unnest;
4445
import org.apache.doris.nereids.trees.expressions.literal.Literal;
4546
import org.apache.doris.nereids.trees.expressions.literal.TinyIntLiteral;
@@ -163,7 +164,8 @@ private LogicalPlan normalizeAgg(LogicalAggregate<Plan> aggregate, Optional<Logi
163164
Set<Expression> groupingByExprs = Utils.fastToImmutableSet(aggregate.getGroupByExpressions());
164165

165166
// collect all trivial-agg
166-
List<NamedExpression> aggregateOutput = aggregate.getOutputExpressions();
167+
List<NamedExpression> aggregateOutput = normalizeMultiColumnDistinctCount(
168+
aggregate.getOutputExpressions());
167169
Map<AggregateFunction, Map<String, String>> aggFuncs =
168170
CollectNonWindowedAggFuncsWithSessionVar.collect(aggregateOutput);
169171

@@ -448,6 +450,28 @@ private LogicalPlan normalizeAgg(LogicalAggregate<Plan> aggregate, Optional<Logi
448450
having.get().withChildren(new LogicalProject<>(bottomProjectsBuilder.build(), newAggregate)));
449451
}
450452

453+
private List<NamedExpression> normalizeMultiColumnDistinctCount(List<NamedExpression> aggregateOutput) {
454+
// Multi-column distinct counts treat arguments as a set. Remove duplicates and canonicalize equivalent
455+
// counts to the first argument order so the structural equality used below can share one aggregate result.
456+
Map<ImmutableSet<Expression>, Count> distinctArgumentsToCount = new HashMap<>();
457+
return ExpressionUtils.rewriteDownShortCircuit(aggregateOutput, expression -> {
458+
if (!(expression instanceof Count)) {
459+
return expression;
460+
}
461+
Count count = (Count) expression;
462+
if (!count.isDistinct() || count.arity() <= 1) {
463+
return count;
464+
}
465+
ImmutableSet<Expression> distinctArguments = ImmutableSet.copyOf(count.getDistinctArguments());
466+
Count normalizedCount = distinctArgumentsToCount.get(distinctArguments);
467+
if (normalizedCount == null) {
468+
normalizedCount = count.withDistinctAndChildren(true, ImmutableList.copyOf(distinctArguments));
469+
distinctArgumentsToCount.put(distinctArguments, normalizedCount);
470+
}
471+
return count.withDistinctAndChildren(true, normalizedCount.children());
472+
});
473+
}
474+
451475
private List<NamedExpression> normalizeOutput(List<NamedExpression> aggregateOutput,
452476
NormalizeToSlotContext groupByToSlotContext, NormalizeToSlotContext argsOfAggFuncNeedPushDownContext,
453477
NormalizeToSlotContext normalizedAggFuncsToSlotContext) {

‎fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownAggWithDistinctThroughJoinOneSide.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -64,7 +64,7 @@ public List<Rule> buildRules() {
6464
} else {
6565
return funcs.stream()
6666
.allMatch(f -> (f instanceof Min || f instanceof Max || f instanceof Sum
67-
|| f instanceof Count) && f.isDistinct()
67+
|| f instanceof Count) && f.isDistinct() && f.arity() == 1
6868
&& f.child(0) instanceof Slot);
6969
}
7070
})

‎fe/fe-core/src/main/java/org/apache/doris/nereids/util/AggregateUtils.java‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@
4040
import com.google.common.collect.Lists;
4141

4242
import java.util.Collection;
43+
import java.util.Iterator;
4344
import java.util.List;
4445
import java.util.Set;
4546

@@ -62,10 +63,11 @@ public static AggregateFunction tryConvertToMultiDistinct(AggregateFunction func
6263

6364
/**countDistinctMultiExprToCountIf*/
6465
public static Expression countDistinctMultiExprToCountIf(Count count) {
65-
Set<Expression> arguments = ImmutableSet.copyOf(count.getArguments());
66-
Expression countExpr = count.getArgument(arguments.size() - 1);
67-
for (int i = arguments.size() - 2; i >= 0; --i) {
68-
Expression argument = count.getArgument(i);
66+
Iterator<Expression> arguments = ImmutableSet.copyOf(count.getArguments())
67+
.asList().reverse().iterator();
68+
Expression countExpr = arguments.next();
69+
while (arguments.hasNext()) {
70+
Expression argument = arguments.next();
6971
If ifNull = new If(new IsNull(argument), NullLiteral.INSTANCE, countExpr);
7072
countExpr = assignNullType(ifNull);
7173
}
Lines changed: 125 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,125 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
package org.apache.doris.nereids.rules.analysis;
19+
20+
import org.apache.doris.nereids.trees.expressions.Alias;
21+
import org.apache.doris.nereids.trees.expressions.Slot;
22+
import org.apache.doris.nereids.trees.expressions.StatementScopeIdGenerator;
23+
import org.apache.doris.nereids.trees.expressions.functions.agg.Count;
24+
import org.apache.doris.nereids.trees.plans.Plan;
25+
import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate;
26+
import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan;
27+
import org.apache.doris.nereids.util.AggregateUtils;
28+
import org.apache.doris.nereids.util.MemoPatternMatchSupported;
29+
import org.apache.doris.nereids.util.MemoTestUtils;
30+
import org.apache.doris.nereids.util.PlanChecker;
31+
import org.apache.doris.nereids.util.PlanConstructor;
32+
33+
import com.google.common.collect.ImmutableList;
34+
import com.google.common.collect.ImmutableSet;
35+
import org.junit.jupiter.api.Assertions;
36+
import org.junit.jupiter.api.Test;
37+
38+
public class NormalizeMultiColumnDistinctCountTest implements MemoPatternMatchSupported {
39+
@Test
40+
public void testNormalizeDifferentArgumentOrder() {
41+
Plan student = new LogicalOlapScan(StatementScopeIdGenerator.newRelationId(), PlanConstructor.student,
42+
ImmutableList.of());
43+
Slot id = student.getOutput().get(0);
44+
Slot name = student.getOutput().get(2);
45+
Alias countIdName = new Alias(new Count(true, id, name), "count_id_name");
46+
Alias countNameId = new Alias(new Count(true, name, id), "count_name_id");
47+
LogicalAggregate<Plan> root = new LogicalAggregate<>(ImmutableList.of(),
48+
ImmutableList.of(countIdName, countNameId), student);
49+
50+
PlanChecker.from(MemoTestUtils.createConnectContext(), root)
51+
.applyTopDown(new NormalizeAggregate())
52+
.matchesFromRoot(
53+
logicalProject(
54+
logicalAggregate(
55+
logicalProject(
56+
logicalOlapScan()
57+
)
58+
).when(aggregate -> aggregate.getAggregateFunctions().size() == 1)
59+
.when(aggregate -> aggregate.getOutputExpressions().size() == 1)
60+
).when(project -> project.getProjects().size() == 2)
61+
.when(project -> project.getProjects().get(0).getExprId()
62+
.equals(countIdName.getExprId()))
63+
.when(project -> project.getProjects().get(1).getExprId()
64+
.equals(countNameId.getExprId()))
65+
.when(project -> project.getProjects().get(0).getInputSlots()
66+
.equals(project.getProjects().get(1).getInputSlots()))
67+
);
68+
}
69+
70+
@Test
71+
public void testKeepDifferentDistinctArguments() {
72+
Plan student = new LogicalOlapScan(StatementScopeIdGenerator.newRelationId(), PlanConstructor.student,
73+
ImmutableList.of());
74+
Slot id = student.getOutput().get(0);
75+
Slot gender = student.getOutput().get(1);
76+
Slot name = student.getOutput().get(2);
77+
LogicalAggregate<Plan> root = new LogicalAggregate<>(ImmutableList.of(),
78+
ImmutableList.of(
79+
new Alias(new Count(true, id, name), "count_id_name"),
80+
new Alias(new Count(true, id, gender), "count_id_gender")),
81+
student);
82+
83+
PlanChecker.from(MemoTestUtils.createConnectContext(), root)
84+
.applyTopDown(new NormalizeAggregate())
85+
.matches(
86+
logicalJoin(
87+
logicalAggregate()
88+
.when(aggregate -> aggregate.getAggregateFunctions().size() == 1),
89+
logicalAggregate()
90+
.when(aggregate -> aggregate.getAggregateFunctions().size() == 1)
91+
)
92+
);
93+
}
94+
95+
@Test
96+
public void testRemoveDuplicateDistinctArguments() {
97+
Plan student = new LogicalOlapScan(StatementScopeIdGenerator.newRelationId(), PlanConstructor.student,
98+
ImmutableList.of());
99+
Slot id = student.getOutput().get(0);
100+
Slot name = student.getOutput().get(2);
101+
Count count = new Count(true, id, id, name);
102+
103+
Count countIf = (Count) AggregateUtils.countDistinctMultiExprToCountIf(count);
104+
Assertions.assertEquals(ImmutableSet.of(id, name), countIf.getInputSlots());
105+
106+
LogicalAggregate<Plan> root = new LogicalAggregate<>(ImmutableList.of(),
107+
ImmutableList.of(
108+
new Alias(count, "count_id_id_name"),
109+
new Alias(new Count(true, name, id), "count_name_id")),
110+
student);
111+
PlanChecker.from(MemoTestUtils.createConnectContext(), root)
112+
.applyTopDown(new NormalizeAggregate())
113+
.matchesFromRoot(
114+
logicalProject(
115+
logicalAggregate(
116+
logicalProject(
117+
logicalOlapScan()
118+
)
119+
).when(aggregate -> aggregate.getAggregateFunctions().size() == 1)
120+
.when(aggregate -> aggregate.getAggregateFunctions().iterator().next()
121+
.arity() == 2)
122+
)
123+
);
124+
}
125+
}

‎regression-test/suites/nereids_rules_p0/agg_strategy/distinct_agg_rewriter.groovy‎

Lines changed: 5 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -38,17 +38,12 @@ suite("distinct_agg_rewriter") {
3838

3939
sql "set agg_phase=4"
4040
sql "select group_concat(distinct dst_key1 order by dst_key2) from t_gbykey_10_dstkey_10_1000_id group by gby_key"
41-
test {
42-
sql """select number % 2, count(distinct cast(number as varchar), cast(number as varchar)),
43-
group_concat(distinct cast(number as varchar) order by number + 1) from numbers('number'='10') group by number % 2"""
44-
exception "Unsupported query"
45-
}
4641

47-
test {
48-
sql """select count(distinct cast(number as varchar), cast(number as varchar)),
49-
group_concat(distinct cast(number as varchar) order by number + 1) from numbers('number'='10')"""
50-
exception "Unsupported query"
51-
}
42+
sql """select number % 2, count(distinct cast(number as varchar), cast(number as varchar)),
43+
group_concat(distinct cast(number as varchar) order by number + 1) from numbers('number'='10') group by number % 2"""
44+
45+
sql """select count(distinct cast(number as varchar), cast(number as varchar)),
46+
group_concat(distinct cast(number as varchar) order by number + 1) from numbers('number'='10')"""
5247

5348
// expect to have 3 HashAgg instead of 4 HashAgg
5449
sql "set agg_phase=3"

0 commit comments

Comments
 (0)