Skip to content

Commit 3c3898e

Browse files
committed
fix: guard grouped aggregate completion and test early emission
Keep PartialReduce on hash execution and require a non-constant grouping expression for full group completion. Cover both cases and gate the second input batch to verify completed groups are emitted before input finishes.
1 parent 64a5955 commit 3c3898e

2 files changed

Lines changed: 179 additions & 47 deletions

File tree

‎datafusion/physical-plan/src/aggregates/mod.rs‎

Lines changed: 147 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -3376,7 +3376,8 @@ mod tests {
33763376
use crate::test::TestMemoryExec;
33773377
use crate::test::assert_is_pending;
33783378
use crate::test::exec::{
3379-
BlockingExec, PanicExec, StatisticsExec, assert_strong_count_converges_to_zero,
3379+
BarrierExec, BlockingExec, PanicExec, StatisticsExec,
3380+
assert_strong_count_converges_to_zero,
33803381
};
33813382

33823383
use arrow::array::AsArray;
@@ -5569,7 +5570,7 @@ mod tests {
55695570
}
55705571

55715572
#[test]
5572-
fn partial_reduce_does_not_advertise_input_ordering() -> Result<()> {
5573+
fn partial_reduce_does_not_use_group_clustering() -> Result<()> {
55735574
let schema = Arc::new(Schema::new(vec![
55745575
Field::new("a", DataType::UInt32, false),
55755576
Field::new("b", DataType::Float64, false),
@@ -5578,39 +5579,113 @@ mod tests {
55785579
Column::new("a", 0),
55795580
))])
55805581
.unwrap();
5581-
let input = TestMemoryExec::try_new(&[vec![]], Arc::clone(&schema), None)?
5582-
.try_with_sort_information(vec![ordering])?;
5583-
let input = Arc::new(TestMemoryExec::update_cache(&Arc::new(input)));
5584-
assert!(
5585-
input.properties().output_ordering().is_some(),
5586-
"test setup: the input is ordered by the group key"
5587-
);
5582+
let ordered_input =
5583+
TestMemoryExec::try_new(&[vec![]], Arc::clone(&schema), None)?
5584+
.try_with_sort_information(vec![ordering])?;
5585+
let grouped_input =
5586+
TestMemoryExec::try_new(&[vec![]], Arc::clone(&schema), None)?
5587+
.try_with_grouping_information(vec![vec![col("a", &schema)?]])?;
55885588

5589-
let partial_reduce = AggregateExec::try_new(
5590-
AggregateMode::PartialReduce,
5591-
PhysicalGroupBy::new_single(vec![(col("a", &schema)?, "a".to_string())]),
5592-
vec![Arc::new(
5593-
AggregateExprBuilder::new(sum_udaf(), vec![col("b", &schema)?])
5594-
.schema(Arc::clone(&schema))
5595-
.alias("SUM(b)")
5596-
.build()?,
5597-
)],
5598-
vec![None],
5599-
input,
5589+
for input in [ordered_input, grouped_input] {
5590+
assert!(
5591+
input
5592+
.properties()
5593+
.equivalence_properties()
5594+
.grouping_satisfy([col("a", &schema)?])?
5595+
);
5596+
let partial_reduce = AggregateExec::try_new(
5597+
AggregateMode::PartialReduce,
5598+
PhysicalGroupBy::new_single(vec![(col("a", &schema)?, "a".to_string())]),
5599+
vec![Arc::new(
5600+
AggregateExprBuilder::new(sum_udaf(), vec![col("b", &schema)?])
5601+
.schema(Arc::clone(&schema))
5602+
.alias("SUM(b)")
5603+
.build()?,
5604+
)],
5605+
vec![None],
5606+
Arc::new(input),
5607+
Arc::clone(&schema),
5608+
)?;
5609+
5610+
assert_eq!(
5611+
partial_reduce.group_clustering_mode(),
5612+
&GroupClusteringMode::None
5613+
);
5614+
assert_eq!(partial_reduce.maintains_input_order(), vec![false]);
5615+
assert!(
5616+
partial_reduce.properties().output_ordering().is_none(),
5617+
"partial reduce advertised an ordering it does not maintain: {:?}",
5618+
partial_reduce.properties().output_ordering()
5619+
);
5620+
5621+
let stream = partial_reduce.execute_typed(0, &new_migrated_hash_ctx(1024))?;
5622+
assert!(matches!(stream, StreamType::PartialReduceHash(_)));
5623+
}
5624+
5625+
Ok(())
5626+
}
5627+
5628+
#[tokio::test]
5629+
async fn constant_only_grouping_uses_final_emission() -> Result<()> {
5630+
let schema = Arc::new(Schema::new(vec![
5631+
Field::new("key", DataType::Int32, false),
5632+
Field::new("value", DataType::Int64, false),
5633+
]));
5634+
let batch = RecordBatch::try_new(
5635+
Arc::clone(&schema),
5636+
vec![
5637+
Arc::new(Int32Array::from(vec![1, 1])),
5638+
Arc::new(Int64Array::from(vec![10, 20])),
5639+
],
5640+
)?;
5641+
let input = TestMemoryExec::try_new_exec(
5642+
&[vec![batch.clone(), batch]],
56005643
Arc::clone(&schema),
5644+
None,
56015645
)?;
5646+
let predicate = binary(col("key", &schema)?, Operator::Eq, lit(1i32), &schema)?;
5647+
let input: Arc<dyn ExecutionPlan> =
5648+
Arc::new(FilterExecBuilder::new(predicate, input).build()?);
56025649

5603-
assert_eq!(
5604-
partial_reduce.group_clustering_mode(),
5605-
&GroupClusteringMode::None
5606-
);
5607-
assert_eq!(partial_reduce.maintains_input_order(), vec![false]);
5608-
assert!(
5609-
partial_reduce.properties().output_ordering().is_none(),
5610-
"partial reduce advertised an ordering it does not maintain: {:?}",
5611-
partial_reduce.properties().output_ordering()
5612-
);
5650+
// Cover both a literal and a column known to be constant after filtering.
5651+
for key in [lit(1i32), col("key", &schema)?] {
5652+
assert!(
5653+
input
5654+
.equivalence_properties()
5655+
.grouping_satisfy([Arc::clone(&key)])?
5656+
);
5657+
let aggregate = AggregateExec::try_new(
5658+
AggregateMode::Single,
5659+
PhysicalGroupBy::new_single(vec![(key, "key".to_string())]),
5660+
vec![Arc::new(
5661+
AggregateExprBuilder::new(sum_udaf(), vec![col("value", &schema)?])
5662+
.schema(Arc::clone(&schema))
5663+
.alias("SUM(value)")
5664+
.build()?,
5665+
)],
5666+
vec![None],
5667+
Arc::clone(&input),
5668+
Arc::clone(&schema),
5669+
)?;
5670+
assert_eq!(
5671+
aggregate.group_clustering_mode(),
5672+
&GroupClusteringMode::None
5673+
);
5674+
assert_eq!(aggregate.cache().emission_type, EmissionType::Final);
56135675

5676+
let stream = aggregate.execute_typed(0, &new_migrated_hash_ctx(1024))?;
5677+
assert!(matches!(stream, StreamType::SingleHash(_)));
5678+
let output = collect(stream.into()).await?;
5679+
allow_duplicates! {
5680+
assert_snapshot!(batches_to_string(&output), @r"
5681+
+-----+------------+
5682+
| key | SUM(value) |
5683+
+-----+------------+
5684+
| 1 | 60 |
5685+
+-----+------------+
5686+
");
5687+
}
5688+
}
56145689
Ok(())
56155690
}
56165691

@@ -5971,25 +6046,24 @@ mod tests {
59716046
Field::new("time_bin", DataType::Int64, false),
59726047
Field::new("value", DataType::Int64, false),
59736048
]));
5974-
// Two sorted logical runs are emitted as batches in one DataFusion
5975-
// partition. Every distinct grouping tuple occupies one contiguous range,
5976-
// but tuple order resets at the batch boundary, so (key, time_bin) is not
5977-
// globally sorted.
6049+
// Every grouping tuple occupies one contiguous range in a single
6050+
// partition, with (2, 20) spanning both batches. Tuple order resets
6051+
// within the second batch, so (key, time_bin) is not globally sorted.
59786052
let input_batches = vec![
59796053
RecordBatch::try_new(
59806054
Arc::clone(&schema),
59816055
vec![
5982-
Arc::new(Int32Array::from(vec![1, 1, 2, 2])),
5983-
Arc::new(Int64Array::from(vec![20, 20, 20, 20])),
5984-
Arc::new(Int64Array::from(vec![10, 20, 30, 40])),
6056+
Arc::new(Int32Array::from(vec![1, 1, 2])),
6057+
Arc::new(Int64Array::from(vec![20, 20, 20])),
6058+
Arc::new(Int64Array::from(vec![10, 20, 30])),
59856059
],
59866060
)?,
59876061
RecordBatch::try_new(
59886062
Arc::clone(&schema),
59896063
vec![
5990-
Arc::new(Int32Array::from(vec![1, 1, 2, 2])),
5991-
Arc::new(Int64Array::from(vec![0, 0, 0, 0])),
5992-
Arc::new(Int64Array::from(vec![50, 60, 70, 80])),
6064+
Arc::new(Int32Array::from(vec![2, 1, 1, 2, 2])),
6065+
Arc::new(Int64Array::from(vec![20, 0, 0, 0, 0])),
6066+
Arc::new(Int64Array::from(vec![40, 50, 60, 70, 80])),
59936067
],
59946068
)?,
59956069
];
@@ -6005,17 +6079,26 @@ mod tests {
60056079
.alias("SUM(value)")
60066080
.build()?,
60076081
);
6008-
let input = TestMemoryExec::try_new(&[input_batches], Arc::clone(&schema), None)?
6009-
.try_with_grouping_information(vec![vec![key, time_bin]])?;
6010-
let input: Arc<dyn ExecutionPlan> = Arc::new(input);
6011-
assert_eq!(input.output_partitioning().partition_count(), 1);
6082+
let mut eq_properties = EquivalenceProperties::new(Arc::clone(&schema));
6083+
eq_properties.add_groupings([vec![key, time_bin]]);
6084+
let input = Arc::new(
6085+
BarrierExec::new(vec![input_batches], Arc::clone(&schema))
6086+
.without_start_barrier()
6087+
.with_batch_barrier()
6088+
.with_equivalence_properties(eq_properties)
6089+
.with_log(false),
6090+
);
6091+
assert_eq!(
6092+
input.properties().output_partitioning().partition_count(),
6093+
1
6094+
);
60126095

60136096
let aggregate = AggregateExec::try_new(
60146097
AggregateMode::Single,
60156098
group_by,
60166099
vec![aggr_expr],
60176100
vec![None],
6018-
input,
6101+
Arc::clone(&input) as Arc<dyn ExecutionPlan>,
60196102
schema,
60206103
)?;
60216104

@@ -6036,8 +6119,25 @@ mod tests {
60366119
let task_ctx = new_migrated_hash_ctx(1024);
60376120
let stream = aggregate.execute_typed(0, &task_ctx)?;
60386121
assert!(matches!(stream, StreamType::ClusteredSingleAggregate(_)));
6039-
let stream: SendableRecordBatchStream = stream.into();
6040-
let output = collect(stream).await?;
6122+
let mut stream: SendableRecordBatchStream = stream.into();
6123+
input.wait_batch().await;
6124+
// The second batch and EOF remain blocked until the first completed
6125+
// group has been emitted.
6126+
let first =
6127+
tokio::time::timeout(std::time::Duration::from_secs(5), stream.next())
6128+
.await
6129+
.expect("completed group was not emitted before the next input batch")
6130+
.expect("stream ended before the next input batch")?;
6131+
assert_snapshot!(batches_to_string(std::slice::from_ref(&first)), @r"
6132+
+-----+----------+------------+
6133+
| key | time_bin | SUM(value) |
6134+
+-----+----------+------------+
6135+
| 1 | 20 | 30 |
6136+
+-----+----------+------------+
6137+
");
6138+
input.wait_batch().await;
6139+
let mut output = vec![first];
6140+
output.extend(collect(stream).await?);
60416141
assert_snapshot!(batches_to_sort_string(&output), @r"
60426142
+-----+----------+------------+
60436143
| key | time_bin | SUM(value) |

‎datafusion/physical-plan/src/test/exec.rs‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -318,6 +318,9 @@ pub struct BarrierExec {
318318
/// all streams wait on this barrier to produce
319319
start_data_barrier: Option<Arc<Barrier>>,
320320

321+
/// each stream waits on this barrier before sending each batch
322+
batch_barrier: Option<Arc<Barrier>>,
323+
321324
/// the stream wait for this to return Poll::Ready(None)
322325
finish_barrier: Option<Arc<(Barrier, AtomicUsize)>>,
323326

@@ -336,6 +339,7 @@ impl BarrierExec {
336339
data,
337340
schema,
338341
start_data_barrier: barrier,
342+
batch_barrier: None,
339343
cache: Arc::new(cache),
340344
finish_barrier: None,
341345
log: true,
@@ -352,6 +356,30 @@ impl BarrierExec {
352356
self
353357
}
354358

359+
/// Wait for [`Self::wait_batch`] before sending each batch.
360+
pub fn with_batch_barrier(mut self) -> Self {
361+
self.batch_barrier = Some(Arc::new(Barrier::new(self.data.len() + 1)));
362+
self
363+
}
364+
365+
/// Release the next batch in each partition.
366+
pub async fn wait_batch(&self) {
367+
self.batch_barrier
368+
.as_ref()
369+
.expect("Must only be called when having a batch barrier")
370+
.wait()
371+
.await;
372+
}
373+
374+
/// Declare the equivalence properties of the test data.
375+
pub fn with_equivalence_properties(
376+
mut self,
377+
properties: EquivalenceProperties,
378+
) -> Self {
379+
self.cache = Arc::new(self.cache.as_ref().clone().with_eq_properties(properties));
380+
self
381+
}
382+
355383
pub fn with_finish_barrier(mut self) -> Self {
356384
let barrier = Arc::new((
357385
// wait for all streams and the input
@@ -486,6 +514,7 @@ impl ExecutionPlan for BarrierExec {
486514
// task simply sends data in order after barrier is reached
487515
let data = self.data[partition].clone();
488516
let start_barrier = self.start_data_barrier.as_ref().map(Arc::clone);
517+
let batch_barrier = self.batch_barrier.as_ref().map(Arc::clone);
489518
let finish_barrier = self.finish_barrier.as_ref().map(Arc::clone);
490519
let log = self.log;
491520
let tx = builder.tx();
@@ -497,6 +526,9 @@ impl ExecutionPlan for BarrierExec {
497526
barrier.wait().await;
498527
}
499528
for batch in data {
529+
if let Some(barrier) = &batch_barrier {
530+
barrier.wait().await;
531+
}
500532
if log {
501533
println!("Partition {partition} sending batch");
502534
}

0 commit comments

Comments
 (0)