Skip to content

Commit 9411463

Browse files
authored
fix: keep field metadata in the result of min and max (apache#26939)
## Which issue does this PR close? - Closes apache#26916. ## Rationale for this change `max` and `min` return a value of the type of their argument, but they drop the field metadata of the argument, including Arrow extension types: `max(uuid_col)` is a plain `FixedSizeBinary(16)`. `first_value` already keeps the metadata. ## What changes are included in this PR? - `Max` and `Min` implement `return_field`. It builds the same field as before and adds the metadata of the argument field, as `first_value` does. - `AggregateStatistics` replaces aggregates that the statistics answer, such as `min` and `max` over a Parquet file, with literals. These literals had no metadata, so the result lost it even with the change above. They now get the metadata of the aggregate field. This is the same kind of problem as apache#26855. ## What is the testing strategy for this PR? New cases in `metadata.slt`: - `max` and `min` without grouping; - with `GROUP BY`, which uses the groups accumulator; - as window functions; - over a Parquet file, where the statistics answer the query. The case checks with `EXPLAIN` that the plan is a projection of literals. A new case in `cast_extension_type_metadata.slt` for `max` and `min` of a `UUID` column, as in the issue. Its input contains a `NULL`, because over a non-null column the query fails with an internal error on nullability. That is a separate bug, apache#26469. All new cases fail without the change. ## Are there any user-facing changes? The result of `min` and `max` keeps the field metadata of the argument, including extension types. There are no API changes.
1 parent e77a370 commit 9411463

4 files changed

Lines changed: 117 additions & 4 deletions

File tree

‎datafusion/functions-aggregate/src/min_max.rs‎

Lines changed: 21 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,8 +9,9 @@ mod min_max_bytes;
99
mod min_max_struct;
1010

1111
use arrow::array::{ArrayRef, ArrowPrimitiveType, downcast_primitive};
12-
use arrow::datatypes::{DataType, Float16Type, Float32Type, Float64Type};
12+
use arrow::datatypes::{DataType, FieldRef, Float16Type, Float32Type, Float64Type};
1313
use datafusion_common::float_semantics::{float_gt, float_lt};
14+
use datafusion_common::metadata::FieldMetadata;
1415
use datafusion_common::stats::Precision;
1516
use datafusion_common::{ColumnStatistics, Result, exec_err, internal_err};
1617
use datafusion_functions_aggregate_common::aggregate::groups_accumulator::prim_op::PrimitiveGroupsAccumulator;
@@ -24,7 +25,7 @@ use datafusion_common::ScalarValue;
2425
use datafusion_expr::DistinctHandling;
2526
use datafusion_expr::{
2627
Accumulator, AggregateUDFImpl, Documentation, SetMonotonicity, Signature, Volatility,
27-
function::AccumulatorArgs,
28+
function::AccumulatorArgs, udaf_default_return_field,
2829
};
2930
use datafusion_expr::{GroupsAccumulator, StatisticsArgs};
3031
use datafusion_macros::user_doc;
@@ -207,6 +208,16 @@ impl FromColumnStatistics for Max {
207208
}
208209
}
209210

211+
/// Returns the field of the result of `min` or `max`. The result has the type of the argument,
212+
/// so it keeps the metadata of the argument field, such as an extension type.
213+
fn min_max_return_field(
214+
func: &impl AggregateUDFImpl,
215+
arg_fields: &[FieldRef],
216+
) -> Result<FieldRef> {
217+
let field = udaf_default_return_field(func, arg_fields)?;
218+
Ok(FieldMetadata::from(arg_fields[0].as_ref()).add_to_field_ref(field))
219+
}
220+
210221
impl AggregateUDFImpl for Max {
211222
fn name(&self) -> &str {
212223
"max"
@@ -220,6 +231,10 @@ impl AggregateUDFImpl for Max {
220231
Ok(arg_types[0].to_owned())
221232
}
222233

234+
fn return_field(&self, arg_fields: &[FieldRef]) -> Result<FieldRef> {
235+
min_max_return_field(self, arg_fields)
236+
}
237+
223238
fn accumulator(&self, acc_args: AccumulatorArgs) -> Result<Box<dyn Accumulator>> {
224239
Ok(Box::new(MaxAccumulator::try_new(
225240
acc_args.return_field.data_type(),
@@ -448,6 +463,10 @@ impl AggregateUDFImpl for Min {
448463
Ok(arg_types[0].to_owned())
449464
}
450465

466+
fn return_field(&self, arg_fields: &[FieldRef]) -> Result<FieldRef> {
467+
min_max_return_field(self, arg_fields)
468+
}
469+
451470
fn accumulator(&self, acc_args: AccumulatorArgs) -> Result<Box<dyn Accumulator>> {
452471
Ok(Box::new(MinAccumulator::try_new(
453472
acc_args.return_field.data_type(),

‎datafusion/physical-optimizer/src/aggregate_statistics.rs‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,18 +5,20 @@
55
//! Utilizing exact statistics from sources to avoid scanning data
66
use datafusion_common::Result;
77
use datafusion_common::config::ConfigOptions;
8+
use datafusion_common::metadata::FieldMetadata;
89
use datafusion_common::scalar::ScalarValue;
910
use datafusion_common::tree_node::{Transformed, TransformedResult, TreeNode};
11+
use datafusion_physical_plan::ExecutionPlan;
1012
use datafusion_physical_plan::aggregates::{
1113
AggregateExec, AggregateInputMode, AggregateMode,
1214
};
15+
use datafusion_physical_plan::expressions::Literal;
1316
use datafusion_physical_plan::placeholder_row::PlaceholderRowExec;
1417
use datafusion_physical_plan::projection::{ProjectionExec, ProjectionExpr};
1518
use datafusion_physical_plan::statistics::StatisticsArgs;
1619
use datafusion_physical_plan::udaf::{
1720
AggregateFunctionExpr, StatisticsArgs as PlanStatisticsArgs,
1821
};
19-
use datafusion_physical_plan::{ExecutionPlan, expressions};
2022
use std::sync::Arc;
2123

2224
use crate::PhysicalOptimizerRule;
@@ -67,8 +69,12 @@ impl PhysicalOptimizerRule for AggregateStatistics {
6769
if let Some((optimizable_statistic, name)) =
6870
take_optimizable_value_from_statistics(&statistics_args, expr)
6971
{
72+
// Keep the metadata of the aggregate field, such as an extension type
7073
projections.push(ProjectionExpr {
71-
expr: expressions::lit(optimizable_statistic),
74+
expr: Arc::new(Literal::new_with_metadata(
75+
optimizable_statistic,
76+
Some(FieldMetadata::from(field.as_ref())),
77+
)),
7278
alias: name,
7379
});
7480
} else {

‎datafusion/sqllogictest/test_files/cast_extension_type_metadata.slt‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -85,6 +85,26 @@ FROM (
8585
----
8686
00010203040506070809000102030506 NULL
8787

88+
# max and min keep the extension type of their argument. The NULL row makes the column
89+
# nullable, see https://github.com/apache/datafusion/issues/26469 for a non-null column.
90+
query ?T?T
91+
SELECT
92+
max(uuid_val),
93+
arrow_metadata(max(uuid_val), 'ARROW:extension:name'),
94+
min(uuid_val),
95+
arrow_metadata(min(uuid_val), 'ARROW:extension:name')
96+
FROM (
97+
SELECT CAST(arrow_cast(raw, 'FixedSizeBinary(16)') AS UUID) AS uuid_val
98+
FROM (
99+
VALUES
100+
(X'00010203040506070809000102030506'),
101+
(X'00010203040506070809000102030507'),
102+
(NULL)
103+
) AS uuids(raw)
104+
);
105+
----
106+
00010203040506070809000102030507 arrow.uuid 00010203040506070809000102030506 arrow.uuid
107+
88108
# Extension types of Parquet columns are visible to expressions evaluated in the scan,
89109
# also when the scan replaces a constant column with a literal from the statistics
90110
statement ok

‎datafusion/sqllogictest/test_files/metadata.slt‎

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -346,6 +346,30 @@ from table_with_metadata;
346346
----
347347
3 the id field
348348

349+
# Regression test: max and min should preserve metadata
350+
query ITIT
351+
select
352+
max(id), arrow_metadata(max(id), 'metadata_key'),
353+
min(id), arrow_metadata(min(id), 'metadata_key')
354+
from table_with_metadata;
355+
----
356+
3 the id field 1 the id field
357+
358+
# Regression test: max and min with GROUP BY (groups accumulator) should preserve metadata
359+
query ITTT
360+
select
361+
id,
362+
arrow_metadata(max(name), 'metadata_key'),
363+
arrow_metadata(min(name), 'metadata_key'),
364+
max(name)
365+
from table_with_metadata
366+
group by id
367+
order by id asc nulls last;
368+
----
369+
1 the name field the name field NULL
370+
3 the name field the name field baz
371+
NULL the name field the name field bar
372+
349373
# Regression test: DISTINCT ON should preserve metadata (uses first_value internally)
350374
query ITTT
351375
select distinct on (id) id, arrow_metadata(id, 'metadata_key'), name, arrow_metadata(name, 'metadata_key')
@@ -498,6 +522,15 @@ from table_with_metadata limit 1;
498522
----
499523
1 the id field
500524

525+
# Test MAX and MIN window functions preserve metadata
526+
query IT
527+
select
528+
max(id) over (order by id asc nulls last) as mx,
529+
arrow_metadata(min(id) over (order by id asc nulls last), 'metadata_key') as meta
530+
from table_with_metadata limit 1;
531+
----
532+
1 the id field
533+
501534
# Test LAST_VALUE window function preserves metadata
502535
query IT
503536
select
@@ -548,6 +581,41 @@ select arrow_metadata((select id from table_with_metadata where id = 1));
548581
----
549582
{metadata_key: the id field}
550583

584+
# Regression test: max and min answered from Parquet statistics should preserve metadata
585+
statement ok
586+
COPY (SELECT id FROM table_with_metadata) TO 'test_files/scratch/metadata/min_max.parquet';
587+
588+
statement ok
589+
SET datafusion.execution.parquet.skip_metadata = false;
590+
591+
statement ok
592+
CREATE EXTERNAL TABLE min_max_with_metadata STORED AS PARQUET
593+
LOCATION 'test_files/scratch/metadata/min_max.parquet';
594+
595+
query ITIT
596+
SELECT
597+
max(id), arrow_metadata(max(id), 'metadata_key'),
598+
min(id), arrow_metadata(min(id), 'metadata_key')
599+
FROM min_max_with_metadata;
600+
----
601+
3 the id field 1 the id field
602+
603+
query TT
604+
EXPLAIN SELECT max(id), min(id) FROM min_max_with_metadata;
605+
----
606+
logical_plan
607+
01)Aggregate: groupBy=[[]], aggr=[[max(min_max_with_metadata.id), min(min_max_with_metadata.id)]]
608+
02)--TableScan: min_max_with_metadata projection=[id]
609+
physical_plan
610+
01)ProjectionExec: expr=[3 as max(min_max_with_metadata.id), 1 as min(min_max_with_metadata.id)]
611+
02)--PlaceholderRowExec
612+
613+
statement ok
614+
DROP TABLE min_max_with_metadata;
615+
616+
statement ok
617+
RESET datafusion.execution.parquet.skip_metadata;
618+
551619
# coalesce, nvl, ifnull and nvl2: conflicting extension types are a planning error
552620
statement error Error during planning: Cannot merge field metadata with differing values of 'ARROW:extension:name': 'a\.type' and 'b\.type'
553621
select coalesce(

0 commit comments

Comments
 (0)