Skip to content

Commit 64a5955

Browse files
committed
feat: stream aggregates over grouped input
1 parent 32bd81f commit 64a5955

1 file changed

Lines changed: 89 additions & 15 deletions

File tree

  • datafusion/physical-plan/src/aggregates

‎datafusion/physical-plan/src/aggregates/mod.rs‎

Lines changed: 89 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -1044,6 +1044,10 @@ impl AggregateExec {
10441044
// states without using group boundaries to recognize completed groups.
10451045
if group_by.has_grouping_set() || mode == AggregateMode::PartialReduce {
10461046
group_clustering_mode = GroupClusteringMode::None;
1047+
} else if num_non_constant_groupby_exprs > 0
1048+
&& input_eq_properties.grouping_satisfy(groupby_exprs.iter().cloned())?
1049+
{
1050+
group_clustering_mode = GroupClusteringMode::Full;
10471051
}
10481052

10491053
// construct a map from the input expression to the output expression of the Aggregation group by
@@ -2457,7 +2461,7 @@ impl ExecutionPlan for AggregateExec {
24572461
metrics: _,
24582462
// Derived at construction from the input ordering and `group_by`.
24592463
required_input_ordering: _,
2460-
// Derived at construction from the input ordering and `group_by`.
2464+
// Derived at construction from the input properties and `group_by`.
24612465
group_clustering_mode: _,
24622466
// Derived at construction by `Self::compute_properties`.
24632467
cache: _,
@@ -3381,7 +3385,7 @@ mod tests {
33813385
Int64Array, NullArray, StringArray, StructArray, UInt32Array, UInt64Array,
33823386
};
33833387
use arrow::compute::{SortOptions, concat_batches};
3384-
use arrow::datatypes::{Float64Type, Int32Type, Int64Type, UInt32Type};
3388+
use arrow::datatypes::{Float64Type, Int32Type, Int64Type, TimeUnit, UInt32Type};
33853389
use datafusion_common::test_util::{batches_to_sort_string, batches_to_string};
33863390
use datafusion_common::{DataFusionError, assert_contains, internal_err};
33873391
use datafusion_execution::config::SessionConfig;
@@ -3394,6 +3398,7 @@ mod tests {
33943398
Accumulator, AggregateUDF, AggregateUDFImpl, EmitTo, GroupsAccumulator, Operator,
33953399
Signature, Volatility,
33963400
};
3401+
use datafusion_functions::datetime::date_bin;
33973402
use datafusion_functions_aggregate::approx_percentile_cont::approx_percentile_cont_udaf;
33983403
use datafusion_functions_aggregate::array_agg::array_agg_udaf;
33993404
use datafusion_functions_aggregate::average::avg_udaf;
@@ -3402,10 +3407,9 @@ mod tests {
34023407
use datafusion_functions_aggregate::median::median_udaf;
34033408
use datafusion_functions_aggregate::min_max::min_udaf;
34043409
use datafusion_functions_aggregate::sum::sum_udaf;
3405-
use datafusion_physical_expr::Partitioning;
3406-
use datafusion_physical_expr::PhysicalSortExpr;
34073410
use datafusion_physical_expr::aggregate::AggregateExprBuilder;
34083411
use datafusion_physical_expr::expressions::{Literal, NotExpr, binary};
3412+
use datafusion_physical_expr::{Partitioning, PhysicalSortExpr, ScalarFunctionExpr};
34093413

34103414
use crate::projection::ProjectionExec;
34113415
use crate::repartition::RepartitionExec;
@@ -5961,7 +5965,7 @@ mod tests {
59615965
}
59625966

59635967
#[tokio::test]
5964-
async fn unsorted_contiguous_groups_use_final_emission() -> Result<()> {
5968+
async fn unsorted_contiguous_groups_use_incremental_emission() -> Result<()> {
59655969
let schema = Arc::new(Schema::new(vec![
59665970
Field::new("key", DataType::Int32, false),
59675971
Field::new("time_bin", DataType::Int64, false),
@@ -5989,18 +5993,21 @@ mod tests {
59895993
],
59905994
)?,
59915995
];
5996+
let key = col("key", &schema)?;
5997+
let time_bin = col("time_bin", &schema)?;
59925998
let group_by = PhysicalGroupBy::new_single(vec![
5993-
(col("key", &schema)?, "key".to_string()),
5994-
(col("time_bin", &schema)?, "time_bin".to_string()),
5999+
(Arc::clone(&key), "key".to_string()),
6000+
(Arc::clone(&time_bin), "time_bin".to_string()),
59956001
]);
59966002
let aggr_expr = Arc::new(
59976003
AggregateExprBuilder::new(sum_udaf(), vec![col("value", &schema)?])
59986004
.schema(Arc::clone(&schema))
59996005
.alias("SUM(value)")
60006006
.build()?,
60016007
);
6002-
let input: Arc<dyn ExecutionPlan> =
6003-
TestMemoryExec::try_new_exec(&[input_batches], Arc::clone(&schema), None)?;
6008+
let input = TestMemoryExec::try_new(&[input_batches], Arc::clone(&schema), None)?
6009+
.try_with_grouping_information(vec![vec![key, time_bin]])?;
6010+
let input: Arc<dyn ExecutionPlan> = Arc::new(input);
60046011
assert_eq!(input.output_partitioning().partition_count(), 1);
60056012

60066013
let aggregate = AggregateExec::try_new(
@@ -6014,16 +6021,21 @@ mod tests {
60146021

60156022
assert_eq!(
60166023
aggregate.group_clustering_mode(),
6017-
&GroupClusteringMode::None
6024+
&GroupClusteringMode::Full
6025+
);
6026+
assert_eq!(aggregate.cache().emission_type, EmissionType::Incremental);
6027+
assert!(
6028+
aggregate
6029+
.cache()
6030+
.equivalence_properties()
6031+
.geq_class()
6032+
.is_empty()
60186033
);
6019-
// This captures the behavior before #24438. When the source can declare
6020-
// `(key, time_bin)` group-contiguous, the corresponding case can use
6021-
// `EmissionType::Incremental`.
6022-
assert_eq!(aggregate.cache().emission_type, EmissionType::Final);
6034+
assert!(aggregate.cache().output_ordering().is_none());
60236035

60246036
let task_ctx = new_migrated_hash_ctx(1024);
60256037
let stream = aggregate.execute_typed(0, &task_ctx)?;
6026-
assert!(matches!(stream, StreamType::SingleHash(_)));
6038+
assert!(matches!(stream, StreamType::ClusteredSingleAggregate(_)));
60276039
let stream: SendableRecordBatchStream = stream.into();
60286040
let output = collect(stream).await?;
60296041
assert_snapshot!(batches_to_sort_string(&output), @r"
@@ -6040,6 +6052,68 @@ mod tests {
60406052
Ok(())
60416053
}
60426054

6055+
#[test]
6056+
fn grouped_date_bin_projects_to_aggregate() -> Result<()> {
6057+
let schema = Arc::new(Schema::new(vec![
6058+
Field::new("key", DataType::Int32, false),
6059+
Field::new("time", DataType::Timestamp(TimeUnit::Second, None), false),
6060+
]));
6061+
let time_bin_expr = || -> Result<Arc<dyn PhysicalExpr>> {
6062+
Ok(Arc::new(ScalarFunctionExpr::try_new(
6063+
date_bin(),
6064+
vec![
6065+
lit(ScalarValue::new_interval_dt(0, 10_000)),
6066+
col("time", &schema)?,
6067+
],
6068+
&schema,
6069+
Arc::new(ConfigOptions::default()),
6070+
)?))
6071+
};
6072+
let input = TestMemoryExec::try_new(&[vec![]], Arc::clone(&schema), None)?
6073+
.try_with_grouping_information(vec![vec![
6074+
col("key", &schema)?,
6075+
time_bin_expr()?,
6076+
]])?;
6077+
let projection = ProjectionExec::try_new(
6078+
[
6079+
ProjectionExpr::new(col("key", &schema)?, "key"),
6080+
// Build this expression independently from the source
6081+
// assertion so the test exercises semantic expression matching.
6082+
ProjectionExpr::new(time_bin_expr()?, "time_bin"),
6083+
],
6084+
Arc::new(input),
6085+
)?;
6086+
6087+
let projected_schema = projection.schema();
6088+
let key = col("key", &projected_schema)?;
6089+
let time_bin = col("time_bin", &projected_schema)?;
6090+
assert!(
6091+
projection
6092+
.properties()
6093+
.equivalence_properties()
6094+
.grouping_satisfy([Arc::clone(&key), Arc::clone(&time_bin)])?
6095+
);
6096+
let aggregate = AggregateExec::try_new(
6097+
AggregateMode::Single,
6098+
PhysicalGroupBy::new_single(vec![
6099+
(key, "key".to_string()),
6100+
(time_bin, "time_bin".to_string()),
6101+
]),
6102+
vec![],
6103+
vec![],
6104+
Arc::new(projection),
6105+
projected_schema,
6106+
)?;
6107+
6108+
assert!(aggregate.input().output_ordering().is_none());
6109+
assert_eq!(
6110+
aggregate.group_clustering_mode(),
6111+
&GroupClusteringMode::Full
6112+
);
6113+
assert_eq!(aggregate.cache().emission_type, EmissionType::Incremental);
6114+
Ok(())
6115+
}
6116+
60436117
/// Ensures for ordered input, `ClusteredPartialAggregateStream` is used.
60446118
#[tokio::test]
60456119
async fn ordered_partial_aggregate_planning() -> Result<()> {

0 commit comments

Comments
 (0)