Skip to content

Commit f63b437

Browse files
committed
feat(adapter): add metrics ownership, grouped timer cache, and multi-group regression
- Adapter owns metrics + grouped timer cache. - Every created accumulator gets `set_metrics`. - Removed factory-side shared `OnceLock`. - Added multi-group regression. - Updated tests/docs.
1 parent f3e7063 commit f63b437

3 files changed

Lines changed: 106 additions & 58 deletions

File tree

‎datafusion/expr-common/src/accumulator.rs‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -71,8 +71,9 @@ pub trait AggregateMetrics: Debug + Send + Sync {
7171
pub trait Accumulator: Send + Sync + Debug + std::any::Any {
7272
/// Supplies optional metrics owned by this aggregate expression.
7373
///
74-
/// The default preserves compatibility for accumulators without internal
75-
/// submetrics.
74+
/// The grouped accumulator adapter supplies these metrics to every
75+
/// accumulator it creates. The default preserves compatibility for
76+
/// accumulators without internal submetrics.
7677
fn set_metrics(&mut self, _metrics: Arc<dyn AggregateMetrics>) {}
7778

7879
/// Updates the accumulator's state from its input.

‎datafusion/functions-aggregate-common/src/aggregate/groups_accumulator.rs‎

Lines changed: 90 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,9 @@ use arrow::{
3535
datatypes::UInt32Type,
3636
};
3737
use datafusion_common::{Result, ScalarValue, arrow_datafusion_err, instant::Instant};
38-
use datafusion_expr_common::accumulator::{Accumulator, AggregateMetric};
38+
use datafusion_expr_common::accumulator::{
39+
Accumulator, AggregateMetric, AggregateMetrics,
40+
};
3941
use datafusion_expr_common::groups_accumulator::{
4042
EmitTo, GroupSelection, GroupsAccumulator,
4143
};
@@ -105,12 +107,11 @@ pub struct GroupsAccumulatorAdapter {
105107
/// distinct groups.
106108
allocation_bytes: usize,
107109

110+
/// Metrics supplied by the aggregate execution operator.
111+
metrics: Option<Arc<dyn AggregateMetrics>>,
112+
108113
/// Optional aggregate-owned metric timed once for a grouped update batch.
109-
///
110-
/// The cache is shared with factories that install metrics on only the
111-
/// first accumulator. This avoids resolving the same expression metric for
112-
/// every group while allowing `convert_to_state(&self)` to use it.
113-
grouped_update_metric: Arc<OnceLock<Option<Arc<dyn AggregateMetric>>>>,
114+
grouped_update_metric: OnceLock<Option<Arc<dyn AggregateMetric>>>,
114115
}
115116

116117
/// Maximum number of prepared group inputs retained while timing an
@@ -145,32 +146,29 @@ impl GroupsAccumulatorAdapter {
145146
/// Create a new adapter that will create a new [`Accumulator`]
146147
/// for each group, using the specified factory function
147148
pub fn new<F>(factory: F) -> Self
148-
where
149-
F: Fn() -> Result<Box<dyn Accumulator>> + Send + 'static,
150-
{
151-
Self::new_with_grouped_update_metric_cache(factory, Arc::new(OnceLock::new()))
152-
}
153-
154-
/// Creates an adapter with a metric cache shared by its accumulator factory.
155-
///
156-
/// The factory must initialize the cache before returning its first
157-
/// metric-enabled accumulator. Later accumulators may be created without
158-
/// metrics because this adapter records the grouped batch timing.
159-
pub fn new_with_grouped_update_metric_cache<F>(
160-
factory: F,
161-
grouped_update_metric: Arc<OnceLock<Option<Arc<dyn AggregateMetric>>>>,
162-
) -> Self
163149
where
164150
F: Fn() -> Result<Box<dyn Accumulator>> + Send + 'static,
165151
{
166152
Self {
167153
factory: Box::new(factory),
168154
states: vec![],
169155
allocation_bytes: 0,
170-
grouped_update_metric,
156+
metrics: None,
157+
grouped_update_metric: OnceLock::new(),
171158
}
172159
}
173160

161+
/// Creates an accumulator with this adapter's optional metrics.
162+
fn create_accumulator(&self) -> Result<Box<dyn Accumulator>> {
163+
let mut accumulator = (self.factory)()?;
164+
if let Some(metrics) = &self.metrics {
165+
accumulator.set_metrics(Arc::clone(metrics));
166+
}
167+
self.grouped_update_metric
168+
.get_or_init(|| accumulator.grouped_update_batch_metric());
169+
Ok(accumulator)
170+
}
171+
174172
/// Ensure that self.accumulators has total_num_groups
175173
fn make_accumulators_if_needed(&mut self, total_num_groups: usize) -> Result<()> {
176174
// can't shrink
@@ -180,9 +178,7 @@ impl GroupsAccumulatorAdapter {
180178
// instantiate new accumulators
181179
let new_accumulators = total_num_groups - self.states.len();
182180
for _ in 0..new_accumulators {
183-
let accumulator = (self.factory)()?;
184-
self.grouped_update_metric
185-
.get_or_init(|| accumulator.grouped_update_batch_metric());
181+
let accumulator = self.create_accumulator()?;
186182
let state = AccumulatorState::new(accumulator);
187183
self.add_allocation(state.size());
188184
self.states.push(state);
@@ -408,6 +404,10 @@ impl GroupsAccumulatorAdapter {
408404
}
409405

410406
impl GroupsAccumulator for GroupsAccumulatorAdapter {
407+
fn set_metrics(&mut self, metrics: Arc<dyn AggregateMetrics>) {
408+
self.metrics = Some(metrics);
409+
}
410+
411411
fn update_batch(
412412
&mut self,
413413
values: &[ArrayRef],
@@ -453,7 +453,7 @@ impl GroupsAccumulator for GroupsAccumulatorAdapter {
453453
if selected_len == 0 {
454454
// ScalarValue::iter_to_array needs at least one value to infer the
455455
// output type, so evaluate a temporary empty accumulator.
456-
let mut accumulator = (self.factory)()?;
456+
let mut accumulator = self.create_accumulator()?;
457457
return Ok(ScalarValue::iter_to_array([accumulator.evaluate()?])?.slice(0, 0));
458458
}
459459

@@ -543,7 +543,7 @@ impl GroupsAccumulator for GroupsAccumulatorAdapter {
543543
// If there are no rows, return empty arrays
544544
if num_rows == 0 {
545545
// create empty accumulator to get the state types
546-
let empty_state = (self.factory)()?.state()?;
546+
let empty_state = self.create_accumulator()?.state()?;
547547
let empty_arrays = empty_state
548548
.into_iter()
549549
.map(|state_val| new_empty_array(&state_val.data_type()))
@@ -564,12 +564,10 @@ impl GroupsAccumulator for GroupsAccumulatorAdapter {
564564
for row_idx in chunk_start..chunk_end {
565565
// Create the empty accumulator and prepare adapter-owned input
566566
// outside the aggregate submetric.
567-
let accumulator = (self.factory)()?;
567+
let accumulator = self.create_accumulator()?;
568568
if row_idx == 0 {
569-
let metric = self
570-
.grouped_update_metric
571-
.get_or_init(|| accumulator.grouped_update_batch_metric());
572-
grouped_update_metric.clone_from(metric);
569+
grouped_update_metric =
570+
self.grouped_update_metric.get().and_then(Clone::clone);
573571
}
574572
let values_to_accumulate =
575573
slice_and_maybe_filter(values, opt_filter, &[row_idx, row_idx + 1])?;
@@ -691,6 +689,15 @@ mod tests {
691689
}
692690
}
693691

692+
#[derive(Debug)]
693+
struct CountingMetrics(Arc<AtomicUsize>);
694+
695+
impl AggregateMetrics for CountingMetrics {
696+
fn metric(&self, _subphase: &'static str) -> Arc<dyn AggregateMetric> {
697+
Arc::new(CountingMetric(Arc::clone(&self.0)))
698+
}
699+
}
700+
694701
#[derive(Debug)]
695702
struct DurationMetric(Arc<AtomicU64>);
696703

@@ -737,6 +744,57 @@ mod tests {
737744
}
738745
}
739746

747+
#[derive(Debug)]
748+
struct MetricsAwareAccumulator {
749+
metric: Option<Arc<dyn AggregateMetric>>,
750+
}
751+
752+
impl Accumulator for MetricsAwareAccumulator {
753+
fn set_metrics(&mut self, metrics: Arc<dyn AggregateMetrics>) {
754+
self.metric = Some(metrics.metric("update"));
755+
}
756+
757+
fn update_batch(&mut self, _values: &[ArrayRef]) -> Result<()> {
758+
self.metric
759+
.as_ref()
760+
.expect("adapter must supply metrics to every group")
761+
.add_duration(Duration::ZERO);
762+
Ok(())
763+
}
764+
765+
fn evaluate(&mut self) -> Result<ScalarValue> {
766+
Ok(ScalarValue::Int64(None))
767+
}
768+
769+
fn size(&self) -> usize {
770+
size_of_val(self)
771+
}
772+
773+
fn state(&mut self) -> Result<Vec<ScalarValue>> {
774+
Ok(vec![ScalarValue::Int64(None)])
775+
}
776+
777+
fn merge_batch(&mut self, _states: &[ArrayRef]) -> Result<()> {
778+
Ok(())
779+
}
780+
}
781+
782+
#[test]
783+
fn adapter_supplies_metrics_to_every_group() -> Result<()> {
784+
let metric_updates = Arc::new(AtomicUsize::new(0));
785+
let mut adapter = GroupsAccumulatorAdapter::new(|| {
786+
Ok(Box::new(MetricsAwareAccumulator { metric: None })
787+
as Box<dyn Accumulator>)
788+
});
789+
adapter.set_metrics(Arc::new(CountingMetrics(Arc::clone(&metric_updates))));
790+
791+
let values: ArrayRef = Arc::new(Int64Array::from(vec![1, 2]));
792+
adapter.update_batch(&[values], &[0, 1], None, 2)?;
793+
794+
assert_eq!(metric_updates.load(Ordering::Relaxed), 2);
795+
Ok(())
796+
}
797+
740798
#[test]
741799
fn adapter_charges_retained_indices_once_and_releases_them() -> Result<()> {
742800
let mut adapter = GroupsAccumulatorAdapter::new(|| {

‎datafusion/physical-plan/src/aggregates/grouped_hash_stream.rs‎

Lines changed: 13 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@
3030
//!
3131
//! See issue for details: <https://github.com/apache/datafusion/issues/22710>
3232
33-
use std::sync::{Arc, OnceLock};
33+
use std::sync::Arc;
3434
use std::task::{Context, Poll};
3535
use std::vec;
3636

@@ -667,25 +667,11 @@ pub(crate) fn create_group_accumulator(
667667
"Creating GroupsAccumulatorAdapter for {}: {agg_expr:?}",
668668
agg_expr.name()
669669
);
670-
let grouped_update_metric = Arc::new(OnceLock::new());
671-
let factory_metric = Arc::clone(&grouped_update_metric);
672-
let agg_expr_captured = Arc::clone(agg_expr);
673-
let factory = move || {
674-
let mut accumulator = agg_expr_captured.create_accumulator()?;
675-
if factory_metric.get().is_none() {
676-
accumulator.set_metrics(Arc::clone(&metrics));
677-
// This factory is called serially by one adapter, so the first
678-
// accumulator is the only one that resolves the metric handle.
679-
let _ = factory_metric.set(accumulator.grouped_update_batch_metric());
680-
}
681-
Ok(accumulator)
682-
};
683-
Ok(Box::new(
684-
GroupsAccumulatorAdapter::new_with_grouped_update_metric_cache(
685-
factory,
686-
grouped_update_metric,
687-
),
688-
))
670+
let agg_expr = Arc::clone(agg_expr);
671+
let mut adapter =
672+
GroupsAccumulatorAdapter::new(move || agg_expr.create_accumulator());
673+
adapter.set_metrics(metrics);
674+
Ok(Box::new(adapter))
689675
}
690676
}
691677

@@ -1552,8 +1538,8 @@ mod tests {
15521538
}
15531539

15541540
#[test]
1555-
fn legacy_grouped_distinct_resolves_metric_once_for_conversion_and_update()
1556-
-> Result<()> {
1541+
fn legacy_grouped_distinct_installs_metrics_for_conversion_and_update() -> Result<()>
1542+
{
15571543
let schema = Arc::new(Schema::new(vec![Field::new(
15581544
"value",
15591545
DataType::UInt32,
@@ -1580,7 +1566,9 @@ mod tests {
15801566
accumulator.convert_to_state(&[Arc::clone(&values)], None)?;
15811567
accumulator.update_batch(&[values], &[0, 1, 2], None, 3)?;
15821568

1583-
assert_eq!(metric_resolutions.load(Ordering::Relaxed), 1);
1569+
// Three temporary conversion accumulators and three grouped accumulators
1570+
// each receive the aggregate expression's metrics.
1571+
assert_eq!(metric_resolutions.load(Ordering::Relaxed), 6);
15841572
assert_eq!(metric_durations.load(Ordering::Relaxed), 2);
15851573
Ok(())
15861574
}
@@ -1615,7 +1603,8 @@ mod tests {
16151603
}
16161604
accumulator.merge_batch(&[Arc::new(state.finish())], &[1, 2, 3], 4)?;
16171605

1618-
assert_eq!(metric_resolutions.load(Ordering::Relaxed), 1);
1606+
// Four group accumulators each receive the aggregate expression's metrics.
1607+
assert_eq!(metric_resolutions.load(Ordering::Relaxed), 4);
16191608
assert_eq!(metric_durations.load(Ordering::Relaxed), 1);
16201609
Ok(())
16211610
}

0 commit comments

Comments
 (0)