Skip to content

Commit 6ff484a

Browse files
authored
fix: ignore constant aggregate completion boundaries (apache#25019)
## Which issue does this PR close? - Closes apache#25018 ## Rationale for this change A grouping column with one fixed value never changes, so it cannot tell aggregation that an earlier group is complete. Currently the logic uses such columns when choosing how aggregation processes its input. This can select the ordered path even when the remaining grouping columns are not ordered. ## What changes are included in this PR? - Ignore fixed grouping columns when deciding when groups can be completed. - Preserve ordered aggregation when all grouping columns that can change are ordered. - Update relevant plans ## What is the testing strategy for this PR? Added tests for a grouping column fixed by a filter. The tests fail without the fix. ## Are there any user-facing changes? No query results or public APIs change. DataFusion now chooses the appropriate aggregation path for fixed grouping columns.
1 parent b0b5471 commit 6ff484a

9 files changed

Lines changed: 138 additions & 22 deletions

File tree

‎datafusion/core/tests/dataframe/mod.rs‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -910,7 +910,7 @@ async fn test_aggregate_with_pk2() -> Result<()> {
910910
assert_snapshot!(
911911
physical_plan_to_string(&df).await,
912912
@r"
913-
AggregateExec: mode=Single, gby=[id@0 as id, name@1 as name], aggr=[], ordering_mode=Sorted
913+
AggregateExec: mode=Single, gby=[id@0 as id, name@1 as name], aggr=[]
914914
FilterExec: id@0 = 1 AND name@1 = a
915915
DataSourceExec: partitions=1, partition_sizes=[1]
916916
"
@@ -958,7 +958,7 @@ async fn test_aggregate_with_pk3() -> Result<()> {
958958
assert_snapshot!(
959959
physical_plan_to_string(&df).await,
960960
@r"
961-
AggregateExec: mode=Single, gby=[id@0 as id, name@1 as name], aggr=[], ordering_mode=PartiallySorted([0])
961+
AggregateExec: mode=Single, gby=[id@0 as id, name@1 as name], aggr=[]
962962
FilterExec: id@0 = 1
963963
DataSourceExec: partitions=1, partition_sizes=[1]
964964
"
@@ -1008,7 +1008,7 @@ async fn test_aggregate_with_pk4() -> Result<()> {
10081008
assert_snapshot!(
10091009
physical_plan_to_string(&df).await,
10101010
@r"
1011-
AggregateExec: mode=Single, gby=[id@0 as id], aggr=[], ordering_mode=Sorted
1011+
AggregateExec: mode=Single, gby=[id@0 as id], aggr=[]
10121012
FilterExec: id@0 = 1
10131013
DataSourceExec: partitions=1, partition_sizes=[1]
10141014
"

‎datafusion/core/tests/physical_optimizer/filter_pushdown.rs‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -740,12 +740,12 @@ fn test_pushdown_through_aggregates_on_grouping_columns() {
740740
OptimizationTest:
741741
input:
742742
- FilterExec: b@1 = bar
743-
- AggregateExec: mode=Final, gby=[a@0 as a, b@1 as b], aggr=[cnt], ordering_mode=PartiallySorted([0])
743+
- AggregateExec: mode=Final, gby=[a@0 as a, b@1 as b], aggr=[cnt]
744744
- FilterExec: a@0 = foo
745745
- DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true
746746
output:
747747
Ok:
748-
- AggregateExec: mode=Final, gby=[a@0 as a, b@1 as b], aggr=[cnt], ordering_mode=Sorted
748+
- AggregateExec: mode=Final, gby=[a@0 as a, b@1 as b], aggr=[cnt]
749749
- DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true, predicate=a@0 = foo AND b@1 = bar
750750
"
751751
);
@@ -804,7 +804,7 @@ fn test_pushdown_through_aggregates_preserves_parent_filter_order() {
804804
output:
805805
Ok:
806806
- FilterExec: cnt@2 = 1
807-
- AggregateExec: mode=Final, gby=[a@0 as a, b@1 as b], aggr=[cnt], ordering_mode=PartiallySorted([1])
807+
- AggregateExec: mode=Final, gby=[a@0 as a, b@1 as b], aggr=[cnt]
808808
- DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true, predicate=b@1 = bar
809809
"
810810
);
@@ -3206,7 +3206,7 @@ fn test_pushdown_filter_on_non_first_grouping_column() {
32063206
- DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true
32073207
output:
32083208
Ok:
3209-
- AggregateExec: mode=Final, gby=[a@0 as a, b@1 as b], aggr=[cnt], ordering_mode=PartiallySorted([1])
3209+
- AggregateExec: mode=Final, gby=[a@0 as a, b@1 as b], aggr=[cnt]
32103210
- DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true, predicate=b@1 = bar
32113211
"
32123212
);
@@ -3542,7 +3542,7 @@ fn test_pushdown_through_aggregate_with_reordered_input_columns() {
35423542
- DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true
35433543
output:
35443544
Ok:
3545-
- AggregateExec: mode=Final, gby=[a@1 as a, b@2 as b], aggr=[cnt], ordering_mode=PartiallySorted([1])
3545+
- AggregateExec: mode=Final, gby=[a@1 as a, b@2 as b], aggr=[cnt]
35463546
- ProjectionExec: expr=[c@2 as c, a@0 as a, b@1 as b]
35473547
- DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true, predicate=b@1 = bar
35483548
"

‎datafusion/core/tests/physical_optimizer/sanity_checker.rs‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -321,6 +321,27 @@ async fn test_aggregate() -> Result<()> {
321321
Ok(())
322322
}
323323

324+
#[tokio::test]
325+
async fn test_constant_grouping_expr_on_unbounded_input() -> Result<()> {
326+
let case = QueryCase {
327+
sql: "SELECT c1, c2, MIN(c4) FROM test WHERE c2 = 1 GROUP BY c1, c2".to_string(),
328+
cases: vec![
329+
Arc::new(UnaryTestCase {
330+
source_type: SourceType::Bounded,
331+
expect_fail: false,
332+
}),
333+
Arc::new(UnaryTestCase {
334+
source_type: SourceType::Unbounded,
335+
expect_fail: true,
336+
}),
337+
],
338+
error_operator: "operator: AggregateExec".to_string(),
339+
};
340+
341+
case.run().await?;
342+
Ok(())
343+
}
344+
324345
#[tokio::test]
325346
async fn test_window_agg_hash_partition() -> Result<()> {
326347
let test1 = UnaryTestCase {

‎datafusion/physical-plan/src/aggregates/mod.rs‎

Lines changed: 100 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1033,6 +1033,8 @@ impl AggregateExec {
10331033
let required_input_ordering =
10341034
LexRequirement::new(new_requirements).map(OrderingRequirements::new_soft);
10351035

1036+
// Constant expressions never change, so they cannot mark a completed group.
1037+
// Exclude them from both the ordering indices and the group expression count.
10361038
// If our aggregation has grouping sets then our base grouping exprs will
10371039
// be expanded based on the flags in `group_by.groups` where for each
10381040
// group we swap the grouping expr for `null` if the flag is `true`
@@ -1041,9 +1043,18 @@ impl AggregateExec {
10411043
let indices: Vec<usize> = indices
10421044
.into_iter()
10431045
.filter(|idx| group_by.groups.iter().all(|group| !group[*idx]))
1046+
.filter(|idx| {
1047+
input_eq_properties
1048+
.is_expr_constant(&groupby_exprs[*idx])
1049+
.is_none()
1050+
})
10441051
.collect();
10451052

1046-
let mut input_order_mode = if indices.len() == groupby_exprs.len()
1053+
let num_non_constant_groupby_exprs = groupby_exprs
1054+
.iter()
1055+
.filter(|expr| input_eq_properties.is_expr_constant(expr).is_none())
1056+
.count();
1057+
let mut input_order_mode = if indices.len() == num_non_constant_groupby_exprs
10471058
&& !indices.is_empty()
10481059
&& group_by.groups.len() == 1
10491060
{
@@ -3228,7 +3239,7 @@ mod tests {
32283239

32293240
use arrow::array::{
32303241
BooleanArray, DictionaryArray, Float32Array, Float64Array, Int32Array,
3231-
Int64Array, NullArray, StructArray, UInt32Array, UInt64Array,
3242+
Int64Array, NullArray, StringArray, StructArray, UInt32Array, UInt64Array,
32323243
};
32333244
use arrow::compute::{SortOptions, concat_batches};
32343245
use arrow::datatypes::Int32Type;
@@ -3239,7 +3250,7 @@ mod tests {
32393250
use datafusion_execution::runtime_env::RuntimeEnvBuilder;
32403251
use datafusion_expr::function::{AccumulatorArgs, StateFieldsArgs};
32413252
use datafusion_expr::{
3242-
Accumulator, AggregateUDF, AggregateUDFImpl, EmitTo, GroupsAccumulator,
3253+
Accumulator, AggregateUDF, AggregateUDFImpl, EmitTo, GroupsAccumulator, Operator,
32433254
Signature, Volatility,
32443255
};
32453256
use datafusion_functions_aggregate::approx_percentile_cont::approx_percentile_cont_udaf;
@@ -3253,7 +3264,7 @@ mod tests {
32533264
use datafusion_physical_expr::Partitioning;
32543265
use datafusion_physical_expr::PhysicalSortExpr;
32553266
use datafusion_physical_expr::aggregate::AggregateExprBuilder;
3256-
use datafusion_physical_expr::expressions::{Literal, NotExpr};
3267+
use datafusion_physical_expr::expressions::{Literal, NotExpr, binary};
32573268

32583269
use crate::projection::ProjectionExec;
32593270
use crate::repartition::RepartitionExec;
@@ -4697,6 +4708,87 @@ mod tests {
46974708
Ok(())
46984709
}
46994710

4711+
#[tokio::test]
4712+
async fn constant_grouping_expr_is_not_a_completion_boundary() -> Result<()> {
4713+
let schema = Arc::new(Schema::new(vec![
4714+
Field::new("key", DataType::Int32, true),
4715+
Field::new("market", DataType::Utf8, true),
4716+
Field::new("value", DataType::Int64, false),
4717+
]));
4718+
let batch = RecordBatch::try_new(
4719+
Arc::clone(&schema),
4720+
vec![
4721+
Arc::new(Int32Array::from(vec![None, Some(10), Some(10)])),
4722+
Arc::new(StringArray::from(vec![Some("US"), Some("US"), Some("US")])),
4723+
Arc::new(Int64Array::from(vec![3, 1, 2])),
4724+
],
4725+
)?;
4726+
4727+
let build_aggregate = |input: Arc<dyn ExecutionPlan>| -> Result<AggregateExec> {
4728+
let predicate =
4729+
binary(col("market", &schema)?, Operator::Eq, lit("US"), &schema)?;
4730+
let input = Arc::new(FilterExecBuilder::new(predicate, input).build()?);
4731+
AggregateExec::try_new(
4732+
AggregateMode::Single,
4733+
PhysicalGroupBy::new_single(vec![
4734+
(col("key", &schema)?, "key".to_string()),
4735+
(col("market", &schema)?, "market".to_string()),
4736+
]),
4737+
vec![Arc::new(
4738+
AggregateExprBuilder::new(count_udaf(), vec![col("value", &schema)?])
4739+
.schema(Arc::clone(&schema))
4740+
.alias("COUNT(value)")
4741+
.build()?,
4742+
)],
4743+
vec![None],
4744+
input,
4745+
Arc::clone(&schema),
4746+
)
4747+
};
4748+
4749+
let unordered_input = TestMemoryExec::try_new_exec(
4750+
&[vec![batch.clone()]],
4751+
Arc::clone(&schema),
4752+
None,
4753+
)?;
4754+
let aggregate = build_aggregate(unordered_input)?;
4755+
assert_eq!(aggregate.input_order_mode(), &InputOrderMode::Linear);
4756+
assert_eq!(
4757+
aggregate.schema().as_ref(),
4758+
&Schema::new(vec![
4759+
Field::new("key", DataType::Int32, true),
4760+
Field::new("market", DataType::Utf8, true),
4761+
Field::new("COUNT(value)", DataType::Int64, false),
4762+
])
4763+
);
4764+
4765+
let output =
4766+
collect(aggregate.execute(0, Arc::new(TaskContext::default()))?).await?;
4767+
assert_eq!(output.iter().map(RecordBatch::num_rows).sum::<usize>(), 2);
4768+
assert_snapshot!(batches_to_sort_string(&output), @r"
4769+
+-----+--------+--------------+
4770+
| key | market | COUNT(value) |
4771+
+-----+--------+--------------+
4772+
| | US | 1 |
4773+
| 10 | US | 2 |
4774+
+-----+--------+--------------+
4775+
");
4776+
4777+
let ordering = LexOrdering::new([PhysicalSortExpr::new_default(Arc::new(
4778+
Column::new("key", 0),
4779+
))])
4780+
.unwrap();
4781+
let ordered_input =
4782+
TestMemoryExec::try_new(&[vec![batch]], Arc::clone(&schema), None)?
4783+
.try_with_sort_information(vec![ordering])?;
4784+
let ordered_input =
4785+
Arc::new(TestMemoryExec::update_cache(&Arc::new(ordered_input)));
4786+
let aggregate = build_aggregate(ordered_input)?;
4787+
assert_eq!(aggregate.input_order_mode(), &InputOrderMode::Sorted);
4788+
4789+
Ok(())
4790+
}
4791+
47004792
fn partial_reduce_test_aggregate() -> Result<AggregateExec> {
47014793
partial_reduce_test_aggregate_with_batches(1)
47024794
}
@@ -6584,7 +6676,10 @@ mod tests {
65846676
");
65856677
}
65866678
}
6587-
Err(e) => assert!(matches!(e, DataFusionError::ResourcesExhausted(_))),
6679+
Err(e) => assert!(
6680+
matches!(e.find_root(), DataFusionError::ResourcesExhausted(_)),
6681+
"unexpected error: {e}"
6682+
),
65886683
}
65896684

65906685
Ok(())

‎datafusion/sqllogictest/test_files/aggregate.slt‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8789,9 +8789,9 @@ logical_plan
87898789
15)------------EmptyRelation: rows=1
87908790
physical_plan
87918791
01)ProjectionExec: expr=[last_value(a.foo) ORDER BY [a.foo ASC NULLS LAST]@1 as last_value(a.foo) ORDER BY [a.foo ASC NULLS LAST], sum(DISTINCT Int64(1))@2 as sum(DISTINCT Int64(1))]
8792-
02)--AggregateExec: mode=FinalPartitioned, gby=[id@0 as id], aggr=[last_value(a.foo) ORDER BY [a.foo ASC NULLS LAST], sum(DISTINCT Int64(1))], ordering_mode=Sorted
8792+
02)--AggregateExec: mode=FinalPartitioned, gby=[id@0 as id], aggr=[last_value(a.foo) ORDER BY [a.foo ASC NULLS LAST], sum(DISTINCT Int64(1))]
87938793
03)----RepartitionExec: partitioning=Hash([id@0], 4), input_partitions=5
8794-
04)------AggregateExec: mode=Partial, gby=[id@0 as id], aggr=[last_value(a.foo) ORDER BY [a.foo ASC NULLS LAST], sum(DISTINCT Int64(1))], ordering_mode=Sorted
8794+
04)------AggregateExec: mode=Partial, gby=[id@0 as id], aggr=[last_value(a.foo) ORDER BY [a.foo ASC NULLS LAST], sum(DISTINCT Int64(1))]
87958795
05)--------UnionExec
87968796
06)----------ProjectionExec: expr=[1 as id, 2 as foo]
87978797
07)------------PlaceholderRowExec

‎datafusion/sqllogictest/test_files/array_agg.slt‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -574,9 +574,9 @@ logical_plan
574574
15)------------EmptyRelation: rows=1
575575
physical_plan
576576
01)ProjectionExec: expr=[array_length(array_agg(DISTINCT a.foo)@1) as array_length(array_agg(DISTINCT a.foo)), sum(DISTINCT Int64(1))@2 as sum(DISTINCT Int64(1))]
577-
02)--AggregateExec: mode=FinalPartitioned, gby=[id@0 as id], aggr=[array_agg(DISTINCT a.foo), sum(DISTINCT Int64(1))], ordering_mode=Sorted
577+
02)--AggregateExec: mode=FinalPartitioned, gby=[id@0 as id], aggr=[array_agg(DISTINCT a.foo), sum(DISTINCT Int64(1))]
578578
03)----RepartitionExec: partitioning=Hash([id@0], 4), input_partitions=5
579-
04)------AggregateExec: mode=Partial, gby=[id@0 as id], aggr=[array_agg(DISTINCT a.foo), sum(DISTINCT Int64(1))], ordering_mode=Sorted
579+
04)------AggregateExec: mode=Partial, gby=[id@0 as id], aggr=[array_agg(DISTINCT a.foo), sum(DISTINCT Int64(1))]
580580
05)--------UnionExec
581581
06)----------ProjectionExec: expr=[1 as id, CAST(2 AS Int64) as foo]
582582
07)------------PlaceholderRowExec

‎datafusion/sqllogictest/test_files/push_down_filter_regression.slt‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -512,9 +512,9 @@ query TT
512512
EXPLAIN SELECT a, count(b) FROM agg_filter_pushdown WHERE a = 'x' GROUP BY a;
513513
----
514514
physical_plan
515-
01)AggregateExec: mode=FinalPartitioned, gby=[a@0 as a], aggr=[count(agg_filter_pushdown.b)], ordering_mode=Sorted
515+
01)AggregateExec: mode=FinalPartitioned, gby=[a@0 as a], aggr=[count(agg_filter_pushdown.b)]
516516
02)--RepartitionExec: partitioning=Hash([a@0], 4), input_partitions=4
517-
03)----AggregateExec: mode=Partial, gby=[a@0 as a], aggr=[count(agg_filter_pushdown.b)], ordering_mode=Sorted
517+
03)----AggregateExec: mode=Partial, gby=[a@0 as a], aggr=[count(agg_filter_pushdown.b)]
518518
04)------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1
519519
05)--------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_regression/agg_filter_pushdown.parquet]]}, projection=[a, b], file_type=parquet, predicate=a@0 = x, pruning_predicate=a_null_count@2 != row_count@3 AND a_min@0 <= x AND x <= a_max@1, required_guarantees=[a in (x)]
520520

‎datafusion/sqllogictest/test_files/union.slt‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -783,7 +783,7 @@ logical_plan
783783
physical_plan
784784
01)UnionExec
785785
02)--ProjectionExec: expr=[Int64(1)@0 as a]
786-
03)----AggregateExec: mode=SinglePartitioned, gby=[1 as Int64(1)], aggr=[], ordering_mode=Sorted
786+
03)----AggregateExec: mode=SinglePartitioned, gby=[1 as Int64(1)], aggr=[]
787787
04)------PlaceholderRowExec
788788
05)--ProjectionExec: expr=[2 as a]
789789
06)----PlaceholderRowExec
@@ -811,7 +811,7 @@ logical_plan
811811
physical_plan
812812
01)UnionExec
813813
02)--ProjectionExec: expr=[count(Int64(1))@1 as count, CAST(n@0 AS Int64) as n]
814-
03)----AggregateExec: mode=SinglePartitioned, gby=[n@0 as n], aggr=[count(Int64(1))], ordering_mode=Sorted
814+
03)----AggregateExec: mode=SinglePartitioned, gby=[n@0 as n], aggr=[count(Int64(1))]
815815
04)------ProjectionExec: expr=[5 as n]
816816
05)--------PlaceholderRowExec
817817
06)--ProjectionExec: expr=[1 as count, max(Int64(10))@0 as n]

‎datafusion/sqllogictest/test_files/window.slt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -276,7 +276,7 @@ physical_plan
276276
03)----SortExec: expr=[b@0 ASC NULLS LAST], preserve_partitioning=[true]
277277
04)------AggregateExec: mode=FinalPartitioned, gby=[b@0 as b], aggr=[max(d.a)]
278278
05)--------RepartitionExec: partitioning=Hash([b@0], 4), input_partitions=4
279-
06)----------AggregateExec: mode=Partial, gby=[b@1 as b], aggr=[max(d.a)], ordering_mode=Sorted
279+
06)----------AggregateExec: mode=Partial, gby=[b@1 as b], aggr=[max(d.a)]
280280
07)------------UnionExec
281281
08)--------------ProjectionExec: expr=[1 as a, aa as b]
282282
09)----------------PlaceholderRowExec

0 commit comments

Comments
 (0)