Skip to content

Commit 9c44fc0

Browse files
authored
fix: count aggregation_time once per batch instead of once per accumulator (#24736)
## Which issue does this PR close? - Closes #24735. ## Rationale for this change `aggregation_time` on `GroupedHashAggregateStream` is added inside the per-accumulator loop while `agg_start_time` is taken once before it, so with N aggregate functions the k-th accumulator re-adds the elapsed time of all k accumulators so far, inflating the metric by up to (N+1)/2. On a 28-function GROUP BY a single task reported 4m22s of `aggregation_time` inside a 2m0s stage. Single-aggregate plans are unaffected. The newer `aggregate_hash_table` implementations already scope the timer around the whole loop. ## What changes are included in this PR? Move the `add_elapsed` call after the accumulator loop, once per interned batch. `time_calculating_group_ids`, which shares `agg_start_time` as its end point, is unchanged, and the per-accumulator `aggregate_accumulator_metrics` timings are unaffected. ## Are these changes tested? Covered by the existing aggregate tests (`cargo test -p datafusion-physical-plan aggregates`, 193 passed). The metric's value is wall-clock time, which the existing tests do not assert exactly; the inflation itself was measured on a 28-aggregate workload where the fixed metric now stays within the operator's elapsed time. ## Are there any user-facing changes? `aggregation_time` (shown in `EXPLAIN ANALYZE` as elapsed_compute subset time) now reports the actual time spent in accumulators on plans with more than one aggregate function; previously it over-reported on such plans.
1 parent e4cf35c commit 9c44fc0

2 files changed

Lines changed: 69 additions & 3 deletions

File tree

‎datafusion/physical-plan/src/aggregates/group_values/metrics.rs‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -475,6 +475,71 @@ mod tests {
475475
Ok(())
476476
}
477477

478+
#[tokio::test]
479+
async fn test_groupby_aggregation_time_not_inflated_by_accumulator_count()
480+
-> Result<()> {
481+
let schema = Arc::new(Schema::new(vec![
482+
Field::new("k", DataType::UInt32, false),
483+
Field::new("v", DataType::Float64, false),
484+
]));
485+
486+
const ROWS: usize = 8192;
487+
let batches = (0..20)
488+
.map(|_| {
489+
RecordBatch::try_new(
490+
Arc::clone(&schema),
491+
vec![
492+
Arc::new(UInt32Array::from(
493+
(0..ROWS).map(|r| (r % 8) as u32).collect::<Vec<_>>(),
494+
)),
495+
Arc::new(Float64Array::from(
496+
(0..ROWS).map(|r| r as f64).collect::<Vec<_>>(),
497+
)),
498+
],
499+
)
500+
.unwrap()
501+
})
502+
.collect::<Vec<_>>();
503+
504+
let input = TestMemoryExec::try_new_exec(&[batches], Arc::clone(&schema), None)?;
505+
let group_by =
506+
PhysicalGroupBy::new_single(vec![(col("k", &schema)?, "k".to_string())]);
507+
508+
let aggregates = (0..8)
509+
.map(|i| sum_aggregate(&schema, "v", &format!("SUM{i}(v)")))
510+
.collect::<Result<Vec<_>>>()?;
511+
let filters = vec![None; aggregates.len()];
512+
513+
let aggregate_exec = Arc::new(AggregateExec::try_new(
514+
AggregateMode::Partial,
515+
group_by,
516+
aggregates,
517+
filters,
518+
input,
519+
schema,
520+
)?);
521+
522+
let task_ctx = Arc::new(
523+
TaskContext::default().with_session_config(
524+
SessionConfig::new()
525+
.set_bool("datafusion.execution.enable_migration_aggregate", false),
526+
),
527+
);
528+
let _result =
529+
collect(Arc::clone(&aggregate_exec) as _, Arc::clone(&task_ctx)).await?;
530+
531+
let metrics = aggregate_exec.metrics().unwrap();
532+
let aggregation_time =
533+
metrics.sum_by_name("aggregation_time").unwrap().as_usize();
534+
let elapsed_compute = metrics.elapsed_compute().unwrap();
535+
assert!(
536+
aggregation_time <= elapsed_compute,
537+
"aggregation_time {aggregation_time} exceeds elapsed_compute {elapsed_compute}"
538+
);
539+
540+
Ok(())
541+
}
542+
478543
async fn assert_groupby_metrics_final_mode(
479544
enable_migration_aggregate: bool,
480545
) -> Result<()> {

‎datafusion/physical-plan/src/aggregates/grouped_hash_stream.rs‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -978,10 +978,11 @@ impl GroupedHashAggregateStream {
978978
|| acc.merge_batch(values, group_indices, total_num_groups),
979979
)?;
980980
}
981-
self.group_by_metrics
982-
.aggregation_time
983-
.add_elapsed(agg_start_time);
984981
}
982+
// Recorded once per grouping set, after all its accumulators ran
983+
self.group_by_metrics
984+
.aggregation_time
985+
.add_elapsed(agg_start_time);
985986
}
986987

987988
Ok(())

0 commit comments

Comments
 (0)