Skip to content

Commit b8c4187

Browse files
authored
test: add regression coverage for terminal hash aggregate drain (apache#24853)
## Which issue does this PR close? - Closes apache#24849. ## Rationale for this change apache#23182 fixed a performance regression in terminal hash aggregate output by avoiding repeated destructive `EmitTo::First` calls. However, the remaining `MaterializedAggregateOutput` unit test only exercises the slicing helper and does not verify the behavior through `AggregateExec`. A result-only integration test would pass with both the regressed and fixed implementations, while a wall-clock assertion would be unreliable in CI. This PR therefore uses a test `GroupsAccumulator` that returns an error if terminal output falls back to destructive `EmitTo::First`. ## What changes are included in this PR? - Adds `single_grouped_aggregate_avoids_destructive_terminal_drain`, which runs unordered Single hash aggregation with more groups than the configured `batch_size`. - Verifies that `AggregateExec` selects `SingleHash`, emits multiple bounded batches, and returns the correct aggregate results for repeated group keys. - Keeps the test independent of `MaterializedAggregateOutput`; Single and Final hash aggregation exercise the same terminal drain helper. ## What is the testing strategy for this PR? - Verified that temporarily restoring destructive `EmitTo::First(batch_size)` causes the new test to fail with the expected accumulator error, while the current implementation passes. - Applied an equivalent FinalHash accumulator oracle to the historical regression: it fails on `d58e0c6d4^` and passes on `d58e0c6d4`. The migrated SingleHash stream did not exist at that historical revision. - Ran `cargo fmt --all`. - Ran `cargo clippy --all-targets --all-features -- -D warnings`. - Ran the extended workspace test suite with `avro,json,backtrace,extended_tests,recursive_protection,parquet_encryption`; all tests passed after initializing the required Arrow and Parquet test fixtures. ## Are there any user-facing changes? No. This PR only adds regression test coverage.
1 parent b3cb365 commit b8c4187

1 file changed

Lines changed: 67 additions & 1 deletion

File tree

  • datafusion/physical-plan/src/aggregates

‎datafusion/physical-plan/src/aggregates/mod.rs‎

Lines changed: 67 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4352,6 +4352,72 @@ mod tests {
43524352
Ok(())
43534353
}
43544354

4355+
/// Single and Final hash aggregation share the same terminal drain helper. Single
4356+
/// exercises it directly from raw input without coupling this test to partial state.
4357+
#[tokio::test]
4358+
async fn single_grouped_aggregate_avoids_destructive_terminal_drain() -> Result<()> {
4359+
const BATCH_SIZE: usize = 2;
4360+
const NUM_GROUPS: usize = 3;
4361+
4362+
let schema = Arc::new(Schema::new(vec![
4363+
Field::new("key", DataType::Int32, false),
4364+
Field::new("value", DataType::Int32, false),
4365+
]));
4366+
let input_batches = vec![RecordBatch::try_new(
4367+
Arc::clone(&schema),
4368+
vec![
4369+
Arc::new(Int32Array::from(vec![1, 2, 3, 1, 2, 3])),
4370+
Arc::new(Int32Array::from(vec![10, 20, 30, 40, 50, 60])),
4371+
],
4372+
)?];
4373+
let input =
4374+
TestMemoryExec::try_new_exec(&[input_batches], Arc::clone(&schema), None)?;
4375+
let group_by =
4376+
PhysicalGroupBy::new_single(vec![(col("key", &schema)?, "key".to_string())]);
4377+
let udaf = Arc::new(AggregateUDF::from(NoFirstEmitUdaf::new()));
4378+
let aggregates: Vec<Arc<AggregateFunctionExpr>> = vec![Arc::new(
4379+
AggregateExprBuilder::new(udaf, vec![col("value", &schema)?])
4380+
.schema(Arc::clone(&schema))
4381+
.alias("no_first_emit(value)")
4382+
.build()?,
4383+
)];
4384+
let aggregate = Arc::new(AggregateExec::try_new(
4385+
AggregateMode::Single,
4386+
group_by,
4387+
aggregates,
4388+
vec![None],
4389+
input,
4390+
Arc::clone(&schema),
4391+
)?);
4392+
let task_ctx = new_migrated_hash_ctx(BATCH_SIZE);
4393+
4394+
let stream = aggregate.execute_typed(0, &task_ctx)?;
4395+
assert!(matches!(stream, StreamType::SingleHash(_)));
4396+
4397+
let stream: SendableRecordBatchStream = stream.into();
4398+
let batches = collect(stream).await?;
4399+
assert!(batches.len() > 1, "expected terminal output to be chunked");
4400+
assert!(
4401+
batches.iter().all(|batch| batch.num_rows() <= BATCH_SIZE),
4402+
"terminal output exceeded the configured batch size"
4403+
);
4404+
assert_eq!(
4405+
batches.iter().map(RecordBatch::num_rows).sum::<usize>(),
4406+
NUM_GROUPS
4407+
);
4408+
assert_snapshot!(batches_to_sort_string(&batches), @r"
4409+
+-----+----------------------+
4410+
| key | no_first_emit(value) |
4411+
+-----+----------------------+
4412+
| 1 | 2 |
4413+
| 2 | 2 |
4414+
| 3 | 2 |
4415+
+-----+----------------------+
4416+
");
4417+
4418+
Ok(())
4419+
}
4420+
43554421
#[tokio::test]
43564422
async fn limited_distinct_aggregate_uses_migrated_hash_streams() -> Result<()> {
43574423
let schema =
@@ -8278,7 +8344,7 @@ mod tests {
82788344
Ok(Arc::new(Int64Array::from(counts)))
82798345
}
82808346
EmitTo::First(_) => internal_err!(
8281-
"partial grouped aggregate output must materialize with EmitTo::All before slicing"
8347+
"grouped aggregate terminal output must not use EmitTo::First"
82828348
),
82838349
}
82848350
}

0 commit comments

Comments
 (0)