Skip to content

Commit da75266

Browse files
jayzhan211Omega359
authored andcommitted
minor: move create_group_accumulator out of the legacy aggregate stream (apache#25539)
## Which issue does this PR close? - Part of apache#25537. Follow-up to apache#22710. ## Rationale for this change `create_group_accumulator` is defined in `grouped_hash_stream.rs`, the legacy `GroupedHashAggregateStream` that is only reachable with `datafusion.execution.enable_migration_aggregate = false` and is scheduled for removal. It is, however, the one piece of that file the new code depends on: both `AggregateHashTable` (`aggregate_hash_table/common.rs`) and `OrderedAggregateTable` (`aggregate_hash_table/common_ordered.rs`) import it from there. Moving it next to its real users removes the only dependency of the new aggregation code on the legacy file, so the legacy stream can later be deleted without touching the new streams. ## What changes are included in this PR? Pure code move, no behaviour change. - Move `create_group_accumulator` to `aggregate_hash_table/common.rs` and re-export it from `aggregate_hash_table`. - `grouped_hash_stream.rs` now imports it from there (dependency direction reversed: legacy → new). ## What is the testing strategy for this PR? Existing tests; no new behaviour. ## Are there any user-facing changes? No.
1 parent b6f4037 commit da75266

4 files changed

Lines changed: 31 additions & 32 deletions

File tree

‎datafusion/physical-plan/src/aggregates/aggregate_hash_table/common.rs‎

Lines changed: 25 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,14 +29,15 @@ use arrow::record_batch::RecordBatch;
2929
use datafusion_common::{Result, internal_err};
3030
use datafusion_execution::memory_pool::proxy::VecAllocExt;
3131
use datafusion_expr::{AggregateMetrics, EmitTo, GroupsAccumulator};
32+
use datafusion_physical_expr::GroupsAccumulatorAdapter;
3233
use datafusion_physical_expr::aggregate::AggregateFunctionExpr;
34+
use log::debug;
3335

3436
use crate::PhysicalExpr;
3537
use crate::aggregates::group_values::{
3638
AccumulatorPhase, AggregateAccumulatorMetrics, AggregateArgumentMetrics,
3739
GroupByMetrics, GroupValues, new_group_values,
3840
};
39-
use crate::aggregates::grouped_hash_stream::create_group_accumulator;
4041
use crate::aggregates::order::GroupOrdering;
4142
use crate::aggregates::{
4243
AggregateExec, PhysicalGroupBy, aggregate_expressions, evaluate_group_by,
@@ -56,6 +57,29 @@ pub(in crate::aggregates) struct PartialSkipMarker;
5657
/// Marker for partial state -> final value aggregation.
5758
pub(in crate::aggregates) struct FinalMarker;
5859

60+
/// Create an accumulator for `agg_expr` -- a [`GroupsAccumulator`] if
61+
/// that is supported by the aggregate, or a
62+
/// [`GroupsAccumulatorAdapter`] if not.
63+
pub(in crate::aggregates) fn create_group_accumulator(
64+
agg_expr: &Arc<AggregateFunctionExpr>,
65+
metrics: Arc<dyn AggregateMetrics>,
66+
) -> Result<Box<dyn GroupsAccumulator>> {
67+
if agg_expr.groups_accumulator_supported() {
68+
agg_expr.create_groups_accumulator_with_metrics(metrics)
69+
} else {
70+
// Note in the log when the slow path is used
71+
debug!(
72+
"Creating GroupsAccumulatorAdapter for {}: {agg_expr:?}",
73+
agg_expr.name()
74+
);
75+
let agg_expr = Arc::clone(agg_expr);
76+
let mut adapter =
77+
GroupsAccumulatorAdapter::new(move || agg_expr.create_accumulator());
78+
adapter.set_metrics(metrics);
79+
Ok(Box::new(adapter))
80+
}
81+
}
82+
5983
/// Grouped hash table shared by the partial and final paths.
6084
///
6185
/// While building, it consumes input batches and updates group / accumulator

‎datafusion/physical-plan/src/aggregates/aggregate_hash_table/common_ordered.rs‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,6 @@ use crate::aggregates::group_values::{
3434
AccumulatorPhase, AggregateAccumulatorMetrics, AggregateArgumentMetrics,
3535
GroupByMetrics, GroupValues, new_group_values,
3636
};
37-
use crate::aggregates::grouped_hash_stream::create_group_accumulator;
3837
use crate::aggregates::order::GroupOrdering;
3938
use crate::aggregates::{
4039
AggregateExec, AggregateMode, PhysicalGroupBy, aggregate_expressions,
@@ -44,7 +43,7 @@ use crate::aggregates::{
4443
use super::AggregateTableMetrics;
4544
use super::common::{
4645
AggregateAccumulator, AggregateBatchFn, AggregateHashTable, EvaluatedAggregateBatch,
47-
MaterializeAccumulatorFn,
46+
MaterializeAccumulatorFn, create_group_accumulator,
4847
};
4948

5049
#[derive(Clone)]

‎datafusion/physical-plan/src/aggregates/aggregate_hash_table/mod.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -99,7 +99,7 @@ impl AggregateTableMetrics {
9999

100100
pub(super) use common::{
101101
AggregateHashTable, FinalMarker, PartialMarker, PartialReduceMarker,
102-
PartialSkipMarker, SingleMarker,
102+
PartialSkipMarker, SingleMarker, create_group_accumulator,
103103
};
104104
pub(super) use common_ordered::{OrderedAggregateTable, OrderedAggregateTableMetrics};
105105

‎datafusion/physical-plan/src/aggregates/grouped_hash_stream.rs‎

Lines changed: 4 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ use std::sync::Arc;
3434
use std::task::{Context, Poll};
3535
use std::vec;
3636

37-
use super::aggregate_hash_table::accumulator_phases;
37+
use super::aggregate_hash_table::{accumulator_phases, create_group_accumulator};
3838
use super::order::GroupOrdering;
3939
use super::skip_partial::SkipAggregationProbe;
4040
use super::{AggregateExec, format_human_display};
@@ -64,10 +64,9 @@ use datafusion_common::{
6464
use datafusion_execution::TaskContext;
6565
use datafusion_execution::memory_pool::proxy::VecAllocExt;
6666
use datafusion_execution::memory_pool::{MemoryConsumer, MemoryReservation};
67-
use datafusion_expr::{AggregateMetrics, EmitTo, GroupsAccumulator};
68-
use datafusion_physical_expr::aggregate::AggregateFunctionExpr;
67+
use datafusion_expr::{EmitTo, GroupsAccumulator};
68+
use datafusion_physical_expr::PhysicalSortExpr;
6969
use datafusion_physical_expr::expressions::Column;
70-
use datafusion_physical_expr::{GroupsAccumulatorAdapter, PhysicalSortExpr};
7170
use datafusion_physical_expr_common::sort_expr::LexOrdering;
7271
use datafusion_physical_expr_common::utils::evaluate_expressions_to_arrays;
7372

@@ -652,29 +651,6 @@ impl GroupedHashAggregateStream {
652651
}
653652
}
654653

655-
/// Create an accumulator for `agg_expr` -- a [`GroupsAccumulator`] if
656-
/// that is supported by the aggregate, or a
657-
/// [`GroupsAccumulatorAdapter`] if not.
658-
pub(crate) fn create_group_accumulator(
659-
agg_expr: &Arc<AggregateFunctionExpr>,
660-
metrics: Arc<dyn AggregateMetrics>,
661-
) -> Result<Box<dyn GroupsAccumulator>> {
662-
if agg_expr.groups_accumulator_supported() {
663-
agg_expr.create_groups_accumulator_with_metrics(metrics)
664-
} else {
665-
// Note in the log when the slow path is used
666-
debug!(
667-
"Creating GroupsAccumulatorAdapter for {}: {agg_expr:?}",
668-
agg_expr.name()
669-
);
670-
let agg_expr = Arc::clone(agg_expr);
671-
let mut adapter =
672-
GroupsAccumulatorAdapter::new(move || agg_expr.create_accumulator());
673-
adapter.set_metrics(metrics);
674-
Ok(Box::new(adapter))
675-
}
676-
}
677-
678654
impl Stream for GroupedHashAggregateStream {
679655
type Item = Result<RecordBatch>;
680656

@@ -1512,7 +1488,7 @@ mod tests {
15121488
use arrow::array::{Int32Array, Int64Array, UInt32Array};
15131489
use arrow::datatypes::{DataType, Field, Schema};
15141490
use datafusion_execution::runtime_env::RuntimeEnvBuilder;
1515-
use datafusion_expr::AggregateMetric;
1491+
use datafusion_expr::{AggregateMetric, AggregateMetrics};
15161492
use datafusion_functions_aggregate::{array_agg::array_agg_udaf, count::count_udaf};
15171493
use datafusion_physical_expr::aggregate::AggregateExprBuilder;
15181494
use datafusion_physical_expr::expressions::col;

0 commit comments

Comments
 (0)