Skip to content

Commit 3b0d0c8

Browse files
authored
refactor: share aggregate return field metadata handling (#27193)
## Which issue does this PR close? - Closes #27013. ## Rationale for this change Aggregates that return a value from their first argument need to preserve that field's metadata, including Arrow extension type metadata. The same return-field logic is currently implemented separately by several aggregates, while aggregate-form `nth_value` does not preserve the metadata. ## What changes are included in this PR? This adds `udaf_return_field_preserving_arg_metadata` alongside `udaf_default_return_field` and uses it for `any_value`, `first_value`, `last_value`, `nth_value`, `min`, and `max`. ## What is the testing strategy for this PR? The aggregate metadata cases in `metadata.slt` cover all six functions. The new `nth_value` case specifically exercises aggregate form rather than the separate window function. ## Are there any user-facing changes? Yes. This adds the public `udaf_return_field_preserving_arg_metadata` helper, and aggregate-form `nth_value` now preserves the first argument's field metadata.
1 parent d3ae55e commit 3b0d0c8

7 files changed

Lines changed: 56 additions & 47 deletions

File tree

‎datafusion/expr/src/lib.rs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -136,6 +136,7 @@ pub use udaf::{
136136
StatisticsArgs, UdafDisplayNameBuilder, UdafHumanDisplayBuilder,
137137
UdafSchemaNameBuilder, UdafWindowFunctionDisplayNameBuilder,
138138
UdafWindowFunctionSchemaNameBuilder, udaf_default_return_field,
139+
udaf_return_field_preserving_arg_metadata,
139140
};
140141
#[expect(deprecated)]
141142
pub use udaf::{

‎datafusion/expr/src/udaf.rs‎

Lines changed: 21 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,10 @@ use std::vec;
1313

1414
use arrow::datatypes::{DataType, Field, FieldRef};
1515

16-
use datafusion_common::{Result, ScalarValue, Statistics, exec_err, not_impl_err};
16+
use datafusion_common::{
17+
Result, ScalarValue, Statistics, exec_err, internal_err, metadata::FieldMetadata,
18+
not_impl_err,
19+
};
1720
use datafusion_expr_common::dyn_eq::{DynEq, DynHash};
1821
use datafusion_expr_common::operator::Operator;
1922
use datafusion_physical_expr_common::physical_expr::PhysicalExpr;
@@ -1509,6 +1512,23 @@ pub fn udaf_default_return_field<F: AggregateUDFImpl + ?Sized>(
15091512
)))
15101513
}
15111514

1515+
/// Returns the default field for `func` with metadata from its first argument.
1516+
///
1517+
/// This is useful for aggregate functions that return a value from their first
1518+
/// argument, such as `min`, `max`, and `first_value`.
1519+
pub fn udaf_return_field_preserving_arg_metadata<F: AggregateUDFImpl + ?Sized>(
1520+
func: &F,
1521+
arg_fields: &[FieldRef],
1522+
) -> Result<FieldRef> {
1523+
let Some(arg_field) = arg_fields.first() else {
1524+
return internal_err!(
1525+
"udaf_return_field_preserving_arg_metadata requires at least one argument"
1526+
);
1527+
};
1528+
let field = udaf_default_return_field(func, arg_fields)?;
1529+
Ok(FieldMetadata::from(arg_field.as_ref()).add_to_field_ref(field))
1530+
}
1531+
15121532
pub enum ReversedUDAF {
15131533
/// The expression is the same as the original expression, like SUM, COUNT
15141534
Identical,

‎datafusion/functions-aggregate/src/any_value.rs‎

Lines changed: 5 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -6,14 +6,14 @@
66
77
use std::fmt::Debug;
88
use std::hash::Hash;
9-
use std::sync::Arc;
109

1110
use arrow::datatypes::{DataType, Field, FieldRef};
12-
use datafusion_common::{Result, not_impl_err};
11+
use datafusion_common::Result;
1312
use datafusion_expr::function::{AccumulatorArgs, StateFieldsArgs};
1413
use datafusion_expr::utils::{AggregateOrderSensitivity, format_state_name};
1514
use datafusion_expr::{
1615
Accumulator, AggregateUDFImpl, Documentation, Signature, Volatility,
16+
udaf_return_field_preserving_arg_metadata,
1717
};
1818
use datafusion_macros::user_doc;
1919

@@ -69,15 +69,12 @@ impl AggregateUDFImpl for AnyValue {
6969
&self.signature
7070
}
7171

72-
fn return_type(&self, _arg_types: &[DataType]) -> Result<DataType> {
73-
not_impl_err!("Not called because return_field is implemented")
72+
fn return_type(&self, arg_types: &[DataType]) -> Result<DataType> {
73+
Ok(arg_types[0].clone())
7474
}
7575

7676
fn return_field(&self, arg_fields: &[FieldRef]) -> Result<FieldRef> {
77-
Ok(Arc::new(
78-
Field::new(self.name(), arg_fields[0].data_type().clone(), true)
79-
.with_metadata(arg_fields[0].metadata().clone()),
80-
))
77+
udaf_return_field_preserving_arg_metadata(self, arg_fields)
8178
}
8279

8380
fn accumulator(&self, acc_args: AccumulatorArgs) -> Result<Box<dyn Accumulator>> {

‎datafusion/functions-aggregate/src/first_last.rs‎

Lines changed: 7 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,6 @@ use datafusion_common::cast::as_boolean_array;
2424
use datafusion_common::utils::{compare_rows, extract_row_at_idx_to_buf, get_row_at_idx};
2525
use datafusion_common::{
2626
DataFusionError, Result, ScalarValue, arrow_datafusion_err, internal_err,
27-
not_impl_err,
2827
};
2928
use datafusion_expr::function::{AccumulatorArgs, StateFieldsArgs};
3029
use datafusion_expr::utils::{
@@ -33,6 +32,7 @@ use datafusion_expr::utils::{
3332
use datafusion_expr::{
3433
Accumulator, AggregateUDFImpl, Documentation, EmitTo, Expr, ExprFunctionExt,
3534
GroupsAccumulator, ReversedUDAF, Signature, SortExpr, Volatility,
35+
udaf_return_field_preserving_arg_metadata,
3636
};
3737
use datafusion_functions_aggregate_common::utils::get_sort_options;
3838
use datafusion_macros::user_doc;
@@ -284,20 +284,12 @@ impl AggregateUDFImpl for FirstValue {
284284
&self.signature
285285
}
286286

287-
fn return_type(&self, _arg_types: &[DataType]) -> Result<DataType> {
288-
not_impl_err!("Not called because the return_field_from_args is implemented")
287+
fn return_type(&self, arg_types: &[DataType]) -> Result<DataType> {
288+
Ok(arg_types[0].clone())
289289
}
290290

291291
fn return_field(&self, arg_fields: &[FieldRef]) -> Result<FieldRef> {
292-
// Preserve metadata from the first argument field
293-
Ok(Arc::new(
294-
Field::new(
295-
self.name(),
296-
arg_fields[0].data_type().clone(),
297-
true, // always nullable, there may be no rows
298-
)
299-
.with_metadata(arg_fields[0].metadata().clone()),
300-
))
292+
udaf_return_field_preserving_arg_metadata(self, arg_fields)
301293
}
302294

303295
fn accumulator(&self, acc_args: AccumulatorArgs) -> Result<Box<dyn Accumulator>> {
@@ -1342,20 +1334,12 @@ impl AggregateUDFImpl for LastValue {
13421334
&self.signature
13431335
}
13441336

1345-
fn return_type(&self, _arg_types: &[DataType]) -> Result<DataType> {
1346-
not_impl_err!("Not called because the return_field_from_args is implemented")
1337+
fn return_type(&self, arg_types: &[DataType]) -> Result<DataType> {
1338+
Ok(arg_types[0].clone())
13471339
}
13481340

13491341
fn return_field(&self, arg_fields: &[FieldRef]) -> Result<FieldRef> {
1350-
// Preserve metadata from the first argument field
1351-
Ok(Arc::new(
1352-
Field::new(
1353-
self.name(),
1354-
arg_fields[0].data_type().clone(),
1355-
true, // always nullable, there may be no rows
1356-
)
1357-
.with_metadata(arg_fields[0].metadata().clone()),
1358-
))
1342+
udaf_return_field_preserving_arg_metadata(self, arg_fields)
13591343
}
13601344

13611345
fn accumulator(&self, acc_args: AccumulatorArgs) -> Result<Box<dyn Accumulator>> {

‎datafusion/functions-aggregate/src/min_max.rs‎

Lines changed: 3 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,6 @@ mod min_max_struct;
1111
use arrow::array::{ArrayRef, ArrowPrimitiveType, downcast_primitive};
1212
use arrow::datatypes::{DataType, FieldRef, Float16Type, Float32Type, Float64Type};
1313
use datafusion_common::float_semantics::{float_gt, float_lt};
14-
use datafusion_common::metadata::FieldMetadata;
1514
use datafusion_common::stats::Precision;
1615
use datafusion_common::{ColumnStatistics, Result, exec_err, internal_err};
1716
use datafusion_functions_aggregate_common::aggregate::groups_accumulator::prim_op::PrimitiveGroupsAccumulator;
@@ -25,7 +24,7 @@ use datafusion_common::ScalarValue;
2524
use datafusion_expr::DistinctHandling;
2625
use datafusion_expr::{
2726
Accumulator, AggregateUDFImpl, Documentation, SetMonotonicity, Signature, Volatility,
28-
function::AccumulatorArgs, udaf_default_return_field,
27+
function::AccumulatorArgs, udaf_return_field_preserving_arg_metadata,
2928
};
3029
use datafusion_expr::{GroupsAccumulator, StatisticsArgs};
3130
use datafusion_macros::user_doc;
@@ -208,16 +207,6 @@ impl FromColumnStatistics for Max {
208207
}
209208
}
210209

211-
/// Returns the field of the result of `min` or `max`. The result has the type of the argument,
212-
/// so it keeps the metadata of the argument field, such as an extension type.
213-
fn min_max_return_field(
214-
func: &impl AggregateUDFImpl,
215-
arg_fields: &[FieldRef],
216-
) -> Result<FieldRef> {
217-
let field = udaf_default_return_field(func, arg_fields)?;
218-
Ok(FieldMetadata::from(arg_fields[0].as_ref()).add_to_field_ref(field))
219-
}
220-
221210
impl AggregateUDFImpl for Max {
222211
fn name(&self) -> &str {
223212
"max"
@@ -232,7 +221,7 @@ impl AggregateUDFImpl for Max {
232221
}
233222

234223
fn return_field(&self, arg_fields: &[FieldRef]) -> Result<FieldRef> {
235-
min_max_return_field(self, arg_fields)
224+
udaf_return_field_preserving_arg_metadata(self, arg_fields)
236225
}
237226

238227
fn accumulator(&self, acc_args: AccumulatorArgs) -> Result<Box<dyn Accumulator>> {
@@ -464,7 +453,7 @@ impl AggregateUDFImpl for Min {
464453
}
465454

466455
fn return_field(&self, arg_fields: &[FieldRef]) -> Result<FieldRef> {
467-
min_max_return_field(self, arg_fields)
456+
udaf_return_field_preserving_arg_metadata(self, arg_fields)
468457
}
469458

470459
fn accumulator(&self, acc_args: AccumulatorArgs) -> Result<Box<dyn Accumulator>> {

‎datafusion/functions-aggregate/src/nth_value.rs‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ use datafusion_expr::function::{AccumulatorArgs, StateFieldsArgs};
2222
use datafusion_expr::utils::format_state_name;
2323
use datafusion_expr::{
2424
Accumulator, AggregateUDFImpl, Documentation, ExprFunctionExt, ReversedUDAF,
25-
Signature, SortExpr, Volatility, lit,
25+
Signature, SortExpr, Volatility, lit, udaf_return_field_preserving_arg_metadata,
2626
};
2727
use datafusion_functions_aggregate_common::merge_arrays::merge_ordered_arrays;
2828
use datafusion_functions_aggregate_common::utils::ordering_fields;
@@ -114,6 +114,10 @@ impl AggregateUDFImpl for NthValueAgg {
114114
Ok(arg_types[0].clone())
115115
}
116116

117+
fn return_field(&self, arg_fields: &[FieldRef]) -> Result<FieldRef> {
118+
udaf_return_field_preserving_arg_metadata(self, arg_fields)
119+
}
120+
117121
fn accumulator(&self, acc_args: AccumulatorArgs) -> Result<Box<dyn Accumulator>> {
118122
let n = match acc_args.exprs[1]
119123
.downcast_ref::<Literal>()

‎datafusion/sqllogictest/test_files/metadata.slt‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -346,6 +346,20 @@ from table_with_metadata;
346346
----
347347
3 the id field
348348

349+
# Regression test: nth_value aggregate should preserve metadata
350+
query IT
351+
select nth_value(id, 2 order by id asc nulls last), arrow_metadata(nth_value(id, 2 order by id asc nulls last), 'metadata_key')
352+
from table_with_metadata;
353+
----
354+
3 the id field
355+
356+
# Regression test: any_value should preserve metadata
357+
query T
358+
select arrow_metadata(any_value(id), 'metadata_key')
359+
from table_with_metadata;
360+
----
361+
the id field
362+
349363
# Regression test: max and min should preserve metadata
350364
query ITIT
351365
select

0 commit comments

Comments
 (0)