Skip to content

Commit 5673f3e

Browse files
committed
fix: route grouping sets through unordered aggregation
1 parent 80cb8ee commit 5673f3e

2 files changed

Lines changed: 39 additions & 13 deletions

File tree

‎datafusion/core/tests/physical_optimizer/filter_pushdown.rs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2235,7 +2235,7 @@ fn test_pushdown_grouping_sets_filter_on_common_column() {
22352235
- DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true
22362236
output:
22372237
Ok:
2238-
- AggregateExec: mode=Final, gby=[(a@0 as a, b@1 as b), (NULL as a, b@1 as b)], aggr=[cnt], ordering_mode=PartiallySorted([1])
2238+
- AggregateExec: mode=Final, gby=[(a@0 as a, b@1 as b), (NULL as a, b@1 as b)], aggr=[cnt]
22392239
- DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true, predicate=b@1 = bar
22402240
"
22412241
);
@@ -2521,7 +2521,7 @@ fn test_pushdown_through_aggregate_grouping_sets_with_reordered_input() {
25212521
- DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true
25222522
output:
25232523
Ok:
2524-
- AggregateExec: mode=Final, gby=[(a@1 as a, b@2 as b), (NULL as a, b@2 as b)], aggr=[cnt], ordering_mode=PartiallySorted([1])
2524+
- AggregateExec: mode=Final, gby=[(a@1 as a, b@2 as b), (NULL as a, b@2 as b)], aggr=[cnt]
25252525
- ProjectionExec: expr=[c@2 as c, a@0 as a, b@1 as b]
25262526
- DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true, predicate=b@1 = bar
25272527
"

‎datafusion/physical-plan/src/aggregates/mod.rs‎

Lines changed: 37 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -170,7 +170,7 @@ use crate::statistics::{ChildStats, StatisticsArgs};
170170
use crate::{ChildrenPropertiesMode, ReplaceChildrenOptions, validate_child_count};
171171
use crate::{
172172
DisplayFormatType, Distribution, ExecutionPlan, InputDistributionRequirements,
173-
InputOrderMode, SendableRecordBatchStream, Statistics,
173+
InputOrderMode, Partitioning, SendableRecordBatchStream, Statistics,
174174
};
175175
use datafusion_common::config::ConfigOptions;
176176
use parking_lot::Mutex;
@@ -1015,7 +1015,7 @@ impl AggregateExec {
10151015
.filter(|idx| group_by.groups.iter().all(|group| !group[*idx]))
10161016
.collect();
10171017

1018-
let input_order_mode = if indices.len() == groupby_exprs.len()
1018+
let mut input_order_mode = if indices.len() == groupby_exprs.len()
10191019
&& !indices.is_empty()
10201020
&& group_by.groups.len() == 1
10211021
{
@@ -1026,19 +1026,29 @@ impl AggregateExec {
10261026
InputOrderMode::Linear
10271027
};
10281028

1029+
// To keep the ordering optimization simple, grouping sets are not supported.
1030+
// See [`OrderedPartialAggregateStream`] for more details about this optimization.
1031+
if group_by.has_grouping_set() {
1032+
input_order_mode = InputOrderMode::Linear;
1033+
}
1034+
10291035
// construct a map from the input expression to the output expression of the Aggregation group by
10301036
let group_expr_mapping =
10311037
ProjectionMapping::try_new(group_by.expr.clone(), &input.schema())?;
10321038

1033-
let cache = Self::compute_properties(
1034-
&input,
1035-
Arc::clone(&schema),
1036-
&group_expr_mapping,
1037-
group_by.is_true_no_grouping(),
1038-
&mode,
1039-
&input_order_mode,
1040-
aggr_expr.as_ref(),
1041-
)?;
1039+
let cache = if group_by.has_grouping_set() {
1040+
Self::compute_grouping_set_properties(&input, Arc::clone(&schema))
1041+
} else {
1042+
Self::compute_properties(
1043+
&input,
1044+
Arc::clone(&schema),
1045+
&group_expr_mapping,
1046+
group_by.is_true_no_grouping(),
1047+
&mode,
1048+
&input_order_mode,
1049+
aggr_expr.as_ref(),
1050+
)?
1051+
};
10421052

10431053
let mut exec = AggregateExec {
10441054
mode,
@@ -1438,6 +1448,22 @@ impl AggregateExec {
14381448
))
14391449
}
14401450

1451+
fn compute_grouping_set_properties(
1452+
input: &Arc<dyn ExecutionPlan>,
1453+
schema: SchemaRef,
1454+
) -> PlanProperties {
1455+
// Grouping-set expansion can replace group keys with nulls and adds a
1456+
// grouping ID, so input properties do not project through unchanged.
1457+
PlanProperties::new(
1458+
EquivalenceProperties::new(schema),
1459+
Partitioning::UnknownPartitioning(
1460+
input.output_partitioning().partition_count(),
1461+
),
1462+
EmissionType::Final,
1463+
input.boundedness(),
1464+
)
1465+
}
1466+
14411467
pub fn input_order_mode(&self) -> &InputOrderMode {
14421468
&self.input_order_mode
14431469
}

0 commit comments

Comments
 (0)