Skip to content

Commit 2d6942e

Browse files
committed
Improve single distinct
1 parent d025869 commit 2d6942e

3 files changed

Lines changed: 57 additions & 54 deletions

File tree

‎datafusion/optimizer/src/single_distinct_to_groupby.rs‎

Lines changed: 35 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,32 @@ impl SingleDistinctToGroupBy {
6161
}
6262
}
6363

64+
/// Check if all aggregate expressions are only COUNT(DISTINCT ...) with no
65+
/// GROUP BY. In this case, the single_distinct_to_groupby rewrite is
66+
/// counterproductive because the direct COUNT DISTINCT accumulator is more
67+
/// efficient than materializing all distinct values via GROUP BY just to
68+
/// count them.
69+
fn is_count_distinct_only_no_groupby(group_expr: &[Expr], aggr_expr: &[Expr]) -> bool {
70+
if !group_expr.is_empty() || aggr_expr.is_empty() {
71+
return false;
72+
}
73+
74+
aggr_expr.iter().all(|expr| {
75+
matches!(
76+
expr,
77+
Expr::AggregateFunction(AggregateFunction {
78+
func,
79+
params: AggregateFunctionParams {
80+
distinct: true,
81+
filter: None,
82+
order_by,
83+
..
84+
},
85+
}) if func.name() == "count" && order_by.is_empty()
86+
)
87+
})
88+
}
89+
6490
/// Check whether all aggregate exprs are distinct on a single field.
6591
fn is_single_distinct_agg(aggr_expr: &[Expr]) -> Result<bool> {
6692
let mut fields_set = HashSet::new();
@@ -130,7 +156,8 @@ impl OptimizerRule for SingleDistinctToGroupBy {
130156
group_expr,
131157
..
132158
}) if is_single_distinct_agg(&aggr_expr)?
133-
&& !contains_grouping_set(&group_expr) =>
159+
&& !contains_grouping_set(&group_expr)
160+
&& !is_count_distinct_only_no_groupby(&group_expr, &aggr_expr) =>
134161
{
135162
let group_size = group_expr.len();
136163
// alias all original group_by exprs
@@ -350,14 +377,13 @@ mod tests {
350377
.aggregate(Vec::<Expr>::new(), vec![count_distinct(col("b"))])?
351378
.build()?;
352379

353-
// Should work
380+
// Should not be rewritten: COUNT(DISTINCT) with no GROUP BY is more
381+
// efficient using the direct distinct accumulator
354382
assert_optimized_plan_equal!(
355383
plan,
356384
@r"
357-
Projection: count(alias1) AS count(DISTINCT test.b) [count(DISTINCT test.b):Int64]
358-
Aggregate: groupBy=[[]], aggr=[[count(alias1)]] [count(alias1):Int64]
359-
Aggregate: groupBy=[[test.b AS alias1]], aggr=[[]] [alias1:UInt32]
360-
TableScan: test [a:UInt32, b:UInt32, c:UInt32]
385+
Aggregate: groupBy=[[]], aggr=[[count(DISTINCT test.b)]] [count(DISTINCT test.b):Int64]
386+
TableScan: test [a:UInt32, b:UInt32, c:UInt32]
361387
"
362388
)
363389
}
@@ -437,13 +463,12 @@ mod tests {
437463
.aggregate(Vec::<Expr>::new(), vec![count_distinct(lit(2) * col("b"))])?
438464
.build()?;
439465

466+
// Should not be rewritten: COUNT(DISTINCT) with no GROUP BY
440467
assert_optimized_plan_equal!(
441468
plan,
442469
@r"
443-
Projection: count(alias1) AS count(DISTINCT Int32(2) * test.b) [count(DISTINCT Int32(2) * test.b):Int64]
444-
Aggregate: groupBy=[[]], aggr=[[count(alias1)]] [count(alias1):Int64]
445-
Aggregate: groupBy=[[Int32(2) * test.b AS alias1]], aggr=[[]] [alias1:Int64]
446-
TableScan: test [a:UInt32, b:UInt32, c:UInt32]
470+
Aggregate: groupBy=[[]], aggr=[[count(DISTINCT Int32(2) * test.b)]] [count(DISTINCT Int32(2) * test.b):Int64]
471+
TableScan: test [a:UInt32, b:UInt32, c:UInt32]
447472
"
448473
)
449474
}

‎datafusion/sqllogictest/test_files/clickbench.slt‎

Lines changed: 10 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -137,20 +137,12 @@ query TT
137137
EXPLAIN SELECT COUNT(DISTINCT "UserID") FROM hits;
138138
----
139139
logical_plan
140-
01)Projection: count(alias1) AS count(DISTINCT hits.UserID)
141-
02)--Aggregate: groupBy=[[]], aggr=[[count(alias1)]]
142-
03)----Aggregate: groupBy=[[hits.UserID AS alias1]], aggr=[[]]
143-
04)------SubqueryAlias: hits
144-
05)--------TableScan: hits_raw projection=[UserID]
140+
01)Aggregate: groupBy=[[]], aggr=[[count(DISTINCT hits.UserID)]]
141+
02)--SubqueryAlias: hits
142+
03)----TableScan: hits_raw projection=[UserID]
145143
physical_plan
146-
01)ProjectionExec: expr=[count(alias1)@0 as count(DISTINCT hits.UserID)]
147-
02)--AggregateExec: mode=Final, gby=[], aggr=[count(alias1)]
148-
03)----CoalescePartitionsExec
149-
04)------AggregateExec: mode=Partial, gby=[], aggr=[count(alias1)]
150-
05)--------AggregateExec: mode=FinalPartitioned, gby=[alias1@0 as alias1], aggr=[]
151-
06)----------RepartitionExec: partitioning=Hash([alias1@0], 4), input_partitions=1
152-
07)------------AggregateExec: mode=Partial, gby=[UserID@0 as alias1], aggr=[]
153-
08)--------------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/core/tests/data/clickbench_hits_10.parquet]]}, projection=[UserID], file_type=parquet
144+
01)AggregateExec: mode=Single, gby=[], aggr=[count(DISTINCT hits.UserID)]
145+
02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/core/tests/data/clickbench_hits_10.parquet]]}, projection=[UserID], file_type=parquet
154146

155147
query I
156148
SELECT COUNT(DISTINCT "UserID") FROM hits;
@@ -162,20 +154,12 @@ query TT
162154
EXPLAIN SELECT COUNT(DISTINCT "SearchPhrase") FROM hits;
163155
----
164156
logical_plan
165-
01)Projection: count(alias1) AS count(DISTINCT hits.SearchPhrase)
166-
02)--Aggregate: groupBy=[[]], aggr=[[count(alias1)]]
167-
03)----Aggregate: groupBy=[[hits.SearchPhrase AS alias1]], aggr=[[]]
168-
04)------SubqueryAlias: hits
169-
05)--------TableScan: hits_raw projection=[SearchPhrase]
157+
01)Aggregate: groupBy=[[]], aggr=[[count(DISTINCT hits.SearchPhrase)]]
158+
02)--SubqueryAlias: hits
159+
03)----TableScan: hits_raw projection=[SearchPhrase]
170160
physical_plan
171-
01)ProjectionExec: expr=[count(alias1)@0 as count(DISTINCT hits.SearchPhrase)]
172-
02)--AggregateExec: mode=Final, gby=[], aggr=[count(alias1)]
173-
03)----CoalescePartitionsExec
174-
04)------AggregateExec: mode=Partial, gby=[], aggr=[count(alias1)]
175-
05)--------AggregateExec: mode=FinalPartitioned, gby=[alias1@0 as alias1], aggr=[]
176-
06)----------RepartitionExec: partitioning=Hash([alias1@0], 4), input_partitions=1
177-
07)------------AggregateExec: mode=Partial, gby=[SearchPhrase@0 as alias1], aggr=[]
178-
08)--------------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/core/tests/data/clickbench_hits_10.parquet]]}, projection=[SearchPhrase], file_type=parquet
161+
01)AggregateExec: mode=Single, gby=[], aggr=[count(DISTINCT hits.SearchPhrase)]
162+
02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/core/tests/data/clickbench_hits_10.parquet]]}, projection=[SearchPhrase], file_type=parquet
179163

180164
query I
181165
SELECT COUNT(DISTINCT "SearchPhrase") FROM hits;

‎datafusion/sqllogictest/test_files/joins.slt‎

Lines changed: 12 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -1408,25 +1408,19 @@ from join_t1
14081408
inner join join_t2 on join_t1.t1_id = join_t2.t2_id
14091409
----
14101410
logical_plan
1411-
01)Projection: count(alias1) AS count(DISTINCT join_t1.t1_id)
1412-
02)--Aggregate: groupBy=[[]], aggr=[[count(alias1)]]
1413-
03)----Aggregate: groupBy=[[join_t1.t1_id AS alias1]], aggr=[[]]
1414-
04)------Projection: join_t1.t1_id
1415-
05)--------Inner Join: join_t1.t1_id = join_t2.t2_id
1416-
06)----------TableScan: join_t1 projection=[t1_id]
1417-
07)----------TableScan: join_t2 projection=[t2_id]
1411+
01)Aggregate: groupBy=[[]], aggr=[[count(DISTINCT join_t1.t1_id)]]
1412+
02)--Projection: join_t1.t1_id
1413+
03)----Inner Join: join_t1.t1_id = join_t2.t2_id
1414+
04)------TableScan: join_t1 projection=[t1_id]
1415+
05)------TableScan: join_t2 projection=[t2_id]
14181416
physical_plan
1419-
01)ProjectionExec: expr=[count(alias1)@0 as count(DISTINCT join_t1.t1_id)]
1420-
02)--AggregateExec: mode=Final, gby=[], aggr=[count(alias1)]
1421-
03)----CoalescePartitionsExec
1422-
04)------AggregateExec: mode=Partial, gby=[], aggr=[count(alias1)]
1423-
05)--------AggregateExec: mode=FinalPartitioned, gby=[alias1@0 as alias1], aggr=[]
1424-
06)----------RepartitionExec: partitioning=Hash([alias1@0], 2), input_partitions=2
1425-
07)------------AggregateExec: mode=Partial, gby=[t1_id@0 as alias1], aggr=[]
1426-
08)--------------HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(t1_id@0, t2_id@0)], projection=[t1_id@0]
1427-
09)----------------DataSourceExec: partitions=1, partition_sizes=[1]
1428-
10)----------------RepartitionExec: partitioning=RoundRobinBatch(2), input_partitions=1
1429-
11)------------------DataSourceExec: partitions=1, partition_sizes=[1]
1417+
01)AggregateExec: mode=Final, gby=[], aggr=[count(DISTINCT join_t1.t1_id)]
1418+
02)--CoalescePartitionsExec
1419+
03)----AggregateExec: mode=Partial, gby=[], aggr=[count(DISTINCT join_t1.t1_id)]
1420+
04)------HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(t1_id@0, t2_id@0)], projection=[t1_id@0]
1421+
05)--------DataSourceExec: partitions=1, partition_sizes=[1]
1422+
06)--------RepartitionExec: partitioning=RoundRobinBatch(2), input_partitions=1
1423+
07)----------DataSourceExec: partitions=1, partition_sizes=[1]
14301424

14311425
statement ok
14321426
set datafusion.explain.logical_plan_only = true;

0 commit comments

Comments
 (0)