Skip to content

Commit 39aab04

Browse files
committed
fix(metrics): ensure metric publish after work and fix boundary handling
- Addressed root cause where metric publish was delayed and errors bypassed it. - Introduced **RAII AggregateMetricRecorder** to manage metric aggregation lifecycle. - Metrics now publish **once on drop**, covering success, error, and zero‑work scenarios. - Implemented **direct DISTINCT update/merge** and **grouped adapter update/merge/convert** for accurate metric state tracking. - Added regression tests for **failed‑convert** and **empty‑batch** cases to prevent future issues.
1 parent bd59438 commit 39aab04

4 files changed

Lines changed: 146 additions & 61 deletions

File tree

‎datafusion/expr-common/src/accumulator.rs‎

Lines changed: 50 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@
1818
//! Accumulator module contains the trait definition for aggregation function's accumulators.
1919
2020
use arrow::array::ArrayRef;
21-
use datafusion_common::{Result, ScalarValue, internal_err};
21+
use datafusion_common::{Result, ScalarValue, instant::Instant, internal_err};
2222
use std::fmt::Debug;
2323
use std::sync::Arc;
2424
use std::time::Duration;
@@ -43,6 +43,55 @@ pub trait AggregateMetrics: Debug + Send + Sync {
4343
fn metric(&self, subphase: &'static str) -> Arc<dyn AggregateMetric>;
4444
}
4545

46+
/// Publishes aggregate-owned timing when dropped.
47+
///
48+
/// Create this before the operation that may return an error, and scope each
49+
/// [`Self::timer`] to only the aggregate-owned work. All timed intervals are
50+
/// combined into one metric update when the recorder is dropped.
51+
pub struct AggregateMetricRecorder {
52+
metric: Option<Arc<dyn AggregateMetric>>,
53+
duration: Duration,
54+
}
55+
56+
impl AggregateMetricRecorder {
57+
/// Creates a recorder for an optional aggregate-owned metric.
58+
pub fn new(metric: Option<Arc<dyn AggregateMetric>>) -> Self {
59+
Self {
60+
metric,
61+
duration: Duration::ZERO,
62+
}
63+
}
64+
65+
/// Starts timing one aggregate-owned interval.
66+
pub fn timer(&mut self) -> Option<AggregateMetricTimer<'_>> {
67+
self.metric.as_ref()?;
68+
Some(AggregateMetricTimer {
69+
duration: &mut self.duration,
70+
start: Instant::now(),
71+
})
72+
}
73+
}
74+
75+
impl Drop for AggregateMetricRecorder {
76+
fn drop(&mut self) {
77+
if let Some(metric) = &self.metric {
78+
metric.add_duration(self.duration);
79+
}
80+
}
81+
}
82+
83+
/// One aggregate-owned interval recorded by [`AggregateMetricRecorder`].
84+
pub struct AggregateMetricTimer<'a> {
85+
duration: &'a mut Duration,
86+
start: Instant,
87+
}
88+
89+
impl Drop for AggregateMetricTimer<'_> {
90+
fn drop(&mut self) {
91+
*self.duration += self.start.elapsed();
92+
}
93+
}
94+
4695
/// Tracks an aggregate function's state.
4796
///
4897
/// `Accumulator`s are stateful objects that implement a single group. They

‎datafusion/expr/src/lib.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -102,7 +102,7 @@ pub use datafusion_doc::{
102102
scalar_doc_sections, window_doc_sections,
103103
};
104104
pub use datafusion_expr_common::accumulator::{
105-
Accumulator, AggregateMetric, AggregateMetrics,
105+
Accumulator, AggregateMetric, AggregateMetricRecorder, AggregateMetrics,
106106
};
107107
pub use datafusion_expr_common::columnar_value::ColumnarValue;
108108
pub use datafusion_expr_common::groups_accumulator::{

‎datafusion/functions-aggregate-common/src/aggregate/groups_accumulator.rs‎

Lines changed: 80 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,6 @@ pub mod prim_op;
2525

2626
use std::mem::{size_of, size_of_val};
2727
use std::sync::{Arc, OnceLock};
28-
use std::time::Duration;
2928

3029
use arrow::array::new_empty_array;
3130
use arrow::{
@@ -34,9 +33,9 @@ use arrow::{
3433
compute::take_arrays,
3534
datatypes::UInt32Type,
3635
};
37-
use datafusion_common::{Result, ScalarValue, arrow_datafusion_err, instant::Instant};
36+
use datafusion_common::{Result, ScalarValue, arrow_datafusion_err};
3837
use datafusion_expr_common::accumulator::{
39-
Accumulator, AggregateMetric, AggregateMetrics,
38+
Accumulator, AggregateMetric, AggregateMetricRecorder, AggregateMetrics,
4039
};
4140
use datafusion_expr_common::groups_accumulator::{
4241
EmitTo, GroupSelection, GroupsAccumulator,
@@ -283,9 +282,10 @@ impl GroupsAccumulatorAdapter {
283282

284283
let mut sizes_pre = 0;
285284
let mut sizes_post = 0;
286-
let mut aggregate_duration = Duration::ZERO;
285+
let has_grouped_update_metric = grouped_update_metric.is_some();
286+
let mut metric_recorder = AggregateMetricRecorder::new(grouped_update_metric);
287287
let result: Result<()> = (|| {
288-
if grouped_update_metric.is_none() {
288+
if !has_grouped_update_metric {
289289
// Keep the pre-metrics single-pass path for accumulators that
290290
// do not expose an aggregate-owned submetric.
291291
for (&group_idx, offsets) in
@@ -343,20 +343,20 @@ impl GroupsAccumulatorAdapter {
343343
sizes_pre += self.states[*group_idx].size();
344344
}
345345

346-
let start = grouped_update_metric.as_ref().map(|_| Instant::now());
347-
let mut successful_groups = 0;
348-
let mut chunk_result = Ok(());
349-
for (group_idx, values) in &values_to_accumulate {
350-
chunk_result =
351-
f(self.states[*group_idx].accumulator.as_mut(), values);
352-
if chunk_result.is_err() {
353-
break;
346+
let (successful_groups, chunk_result) = {
347+
let _timer = metric_recorder.timer();
348+
let mut successful_groups = 0;
349+
let mut chunk_result = Ok(());
350+
for (group_idx, values) in &values_to_accumulate {
351+
chunk_result =
352+
f(self.states[*group_idx].accumulator.as_mut(), values);
353+
if chunk_result.is_err() {
354+
break;
355+
}
356+
successful_groups += 1;
354357
}
355-
successful_groups += 1;
356-
}
357-
if let Some(start) = start {
358-
aggregate_duration += start.elapsed();
359-
}
358+
(successful_groups, chunk_result)
359+
};
360360
for (index, (group_idx, _)) in values_to_accumulate.iter().enumerate() {
361361
let state = &mut self.states[*group_idx];
362362
if index < successful_groups {
@@ -371,9 +371,6 @@ impl GroupsAccumulatorAdapter {
371371
Ok(())
372372
})();
373373

374-
if let Some(metric) = grouped_update_metric {
375-
metric.add_duration(aggregate_duration);
376-
}
377374
self.adjust_allocation(sizes_pre, sizes_post);
378375
result
379376
}
@@ -563,8 +560,7 @@ impl GroupsAccumulator for GroupsAccumulatorAdapter {
563560

564561
// Each row has its respective group
565562
let mut results = vec![];
566-
let mut grouped_update_metric = None;
567-
let mut aggregate_duration = Duration::ZERO;
563+
let mut metric_recorder = None;
568564
for chunk_start in (0..num_rows).step_by(GROUPED_METRIC_PREPARATION_CHUNK_SIZE) {
569565
let chunk_end =
570566
(chunk_start + GROUPED_METRIC_PREPARATION_CHUNK_SIZE).min(num_rows);
@@ -575,24 +571,26 @@ impl GroupsAccumulator for GroupsAccumulatorAdapter {
575571
// outside the aggregate submetric.
576572
let accumulator = self.create_accumulator()?;
577573
if row_idx == 0 {
578-
grouped_update_metric =
579-
self.grouped_update_metric.get().and_then(Clone::clone);
574+
metric_recorder = Some(AggregateMetricRecorder::new(
575+
self.grouped_update_metric.get().and_then(Clone::clone),
576+
));
580577
}
581578
let values_to_accumulate =
582579
slice_and_maybe_filter(values, opt_filter, &[row_idx, row_idx + 1])?;
583580
prepared.push((accumulator, values_to_accumulate));
584581
}
585582

586583
// Time only aggregate-owned deduplication, once per bounded chunk.
587-
let start = grouped_update_metric.as_ref().map(|_| Instant::now());
588-
let update_result: Result<()> = prepared.iter_mut().try_for_each(
589-
|(accumulator, values_to_accumulate)| {
590-
accumulator.update_batch_grouped(values_to_accumulate)
591-
},
592-
);
593-
if let Some(start) = start {
594-
aggregate_duration += start.elapsed();
595-
}
584+
let update_result: Result<()> = {
585+
let _timer = metric_recorder
586+
.as_mut()
587+
.and_then(AggregateMetricRecorder::timer);
588+
prepared
589+
.iter_mut()
590+
.try_for_each(|(accumulator, values_to_accumulate)| {
591+
accumulator.update_batch_grouped(values_to_accumulate)
592+
})
593+
};
596594
update_result?;
597595

598596
for (mut accumulator, _) in prepared {
@@ -608,10 +606,6 @@ impl GroupsAccumulator for GroupsAccumulatorAdapter {
608606
}
609607
}
610608

611-
if let Some(metric) = grouped_update_metric {
612-
metric.add_duration(aggregate_duration);
613-
}
614-
615609
let arrays = results
616610
.into_iter()
617611
.map(ScalarValue::iter_to_array)
@@ -850,6 +844,29 @@ mod tests {
850844
Ok(())
851845
}
852846

847+
#[test]
848+
fn adapter_grouped_update_records_zero_duration_for_empty_batch() -> Result<()> {
849+
let metric_updates = Arc::new(AtomicUsize::new(0));
850+
let mut accumulator = GroupsAccumulatorAdapter::new({
851+
let metric_updates = Arc::clone(&metric_updates);
852+
move || {
853+
Ok(Box::new(TimedAccumulator {
854+
metric: Arc::new(CountingMetric(Arc::clone(&metric_updates))),
855+
}) as Box<dyn Accumulator>)
856+
}
857+
});
858+
859+
let values: ArrayRef = Arc::new(Int64Array::from(vec![1]));
860+
accumulator.update_batch(&[values], &[0], None, 1)?;
861+
metric_updates.store(0, Ordering::Relaxed);
862+
863+
let empty_values: ArrayRef = Arc::new(Int64Array::from(Vec::<i64>::new()));
864+
accumulator.update_batch(&[empty_values], &[], None, 1)?;
865+
866+
assert_eq!(metric_updates.load(Ordering::Relaxed), 1);
867+
Ok(())
868+
}
869+
853870
#[test]
854871
fn adapter_convert_to_state_records_metric_once() -> Result<()> {
855872
let metric_updates = Arc::new(AtomicUsize::new(0));
@@ -1109,6 +1126,31 @@ mod tests {
11091126
Ok(())
11101127
}
11111128

1129+
#[test]
1130+
fn adapter_convert_to_state_records_metric_on_update_error() -> Result<()> {
1131+
let created = Arc::new(AtomicUsize::new(0));
1132+
let metric_updates = Arc::new(AtomicUsize::new(0));
1133+
let metric: Arc<dyn AggregateMetric> =
1134+
Arc::new(CountingMetric(Arc::clone(&metric_updates)));
1135+
let accumulator = GroupsAccumulatorAdapter::new({
1136+
let created = Arc::clone(&created);
1137+
move || {
1138+
let group = created.fetch_add(1, Ordering::Relaxed);
1139+
Ok(Box::new(FailOnceAccumulator {
1140+
fail_first_update: group == 1,
1141+
successful_group_rows: None,
1142+
allocation_bytes: 0,
1143+
metric: Some(Arc::clone(&metric)),
1144+
}) as Box<dyn Accumulator>)
1145+
}
1146+
});
1147+
1148+
let values: ArrayRef = Arc::new(Int64Array::from(vec![1, 2]));
1149+
assert!(accumulator.convert_to_state(&[values], None).is_err());
1150+
assert_eq!(metric_updates.load(Ordering::Relaxed), 1);
1151+
Ok(())
1152+
}
1153+
11121154
#[test]
11131155
fn adapter_preserving_evaluation_uses_accumulator_contract() -> Result<()> {
11141156
let mut accumulator = GroupsAccumulatorAdapter::new(|| {

‎datafusion/functions-aggregate/src/array_agg.rs‎

Lines changed: 15 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -38,14 +38,13 @@ use datafusion_common::scalar::copy_array_data;
3838
use datafusion_common::utils::SingleRowListArrayBuilder;
3939
use datafusion_common::utils::proxy::HashTableAllocExt;
4040
use datafusion_common::{
41-
Result, ScalarValue, assert_eq_or_internal_err, exec_err, instant::Instant,
42-
internal_err,
41+
Result, ScalarValue, assert_eq_or_internal_err, exec_err, internal_err,
4342
};
4443
use datafusion_expr::function::{AccumulatorArgs, StateFieldsArgs};
4544
use datafusion_expr::utils::format_state_name;
4645
use datafusion_expr::{
47-
Accumulator, AggregateMetric, AggregateMetrics, AggregateUDFImpl, Documentation,
48-
EmitTo, GroupsAccumulator, Signature, Volatility,
46+
Accumulator, AggregateMetric, AggregateMetricRecorder, AggregateMetrics,
47+
AggregateUDFImpl, Documentation, EmitTo, GroupsAccumulator, Signature, Volatility,
4948
};
5049
use datafusion_functions_aggregate_common::aggregate::groups_accumulator::nulls::filter_to_nulls;
5150
use datafusion_functions_aggregate_common::order::AggregateOrderSensitivity;
@@ -954,8 +953,9 @@ impl DistinctArrayAggAccumulator {
954953
let distinct_metric = record_metric
955954
.then(|| self.distinct_metric.clone())
956955
.flatten();
957-
let distinct_start = distinct_metric.as_ref().map(|_| Instant::now());
958-
let result = (|| {
956+
let mut metric_recorder = AggregateMetricRecorder::new(distinct_metric);
957+
(|| {
958+
let _timer = metric_recorder.timer();
959959
self.ensure_state(col.data_type())?;
960960

961961
// Encode the entire incoming batch into rows_buffer in one pass.
@@ -1003,11 +1003,7 @@ impl DistinctArrayAggAccumulator {
10031003
}
10041004
}
10051005
Ok(())
1006-
})();
1007-
if let (Some(metric), Some(start)) = (distinct_metric, distinct_start) {
1008-
metric.add_duration(start.elapsed());
1009-
}
1010-
result
1006+
})()
10111007
}
10121008
}
10131009

@@ -1026,21 +1022,19 @@ impl DistinctArrayAggAccumulator {
10261022
let distinct_metric = record_metric
10271023
.then(|| self.distinct_metric.clone())
10281024
.flatten();
1029-
let distinct_start = distinct_metric.as_ref().map(|_| Instant::now());
1025+
let mut metric_recorder = AggregateMetricRecorder::new(distinct_metric);
10301026

10311027
// The DISTINCT state is `List<value>`. This calls the update
10321028
// implementation once per state row, so record the submetric once for
10331029
// the entire merge rather than once per row.
1034-
let result = states[0]
1035-
.as_list::<i32>()
1036-
.iter()
1037-
.flatten()
1038-
.try_for_each(|val| self.update_batch_impl(&[val], false));
1039-
1040-
if let (Some(metric), Some(start)) = (distinct_metric, distinct_start) {
1041-
metric.add_duration(start.elapsed());
1030+
{
1031+
let _timer = metric_recorder.timer();
1032+
states[0]
1033+
.as_list::<i32>()
1034+
.iter()
1035+
.flatten()
1036+
.try_for_each(|val| self.update_batch_impl(&[val], false))
10421037
}
1043-
result
10441038
}
10451039
}
10461040

0 commit comments

Comments
 (0)