Skip to content

Commit 1ea88c0

Browse files
committed
fix: propagate field metadata through coalesce, nvl, and nvl2
These functions built their return Field from argument data types only, dropping field metadata (e.g. Arrow extension types) from the planned schema. Attach the metadata shared by every value-contributing argument instead. Part of apache#24860.
1 parent ad5820c commit 1ea88c0

3 files changed

Lines changed: 186 additions & 3 deletions

File tree

‎datafusion/functions/src/core/coalesce.rs‎

Lines changed: 101 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
// specific language governing permissions and limitations
1616
// under the License.
1717

18+
use crate::utils::unanimous_metadata;
1819
use arrow::datatypes::{DataType, Field, FieldRef};
1920
use datafusion_common::{Result, exec_err, internal_err, plan_err};
2021
use datafusion_expr::binary::try_type_union_resolution;
@@ -86,7 +87,12 @@ impl ScalarUDFImpl for CoalesceFunc {
8687
.find_or_first(|d| !d.is_null())
8788
.unwrap()
8889
.clone();
89-
Ok(Field::new(self.name(), return_type, nullable).into())
90+
// Propagate field metadata (e.g. Arrow extension types) only when the
91+
// arguments that can supply a value agree on it.
92+
let metadata = unanimous_metadata(args.arg_fields.iter().map(|f| f.as_ref()));
93+
Ok(Field::new(self.name(), return_type, nullable)
94+
.with_metadata(metadata)
95+
.into())
9096
}
9197

9298
fn simplify(
@@ -146,3 +152,97 @@ impl ScalarUDFImpl for CoalesceFunc {
146152
self.doc()
147153
}
148154
}
155+
156+
#[cfg(test)]
157+
mod tests {
158+
use super::*;
159+
use std::collections::HashMap;
160+
use std::sync::Arc;
161+
162+
const EXTENSION_KEY: &str = "ARROW:extension:name";
163+
164+
fn field(name: &str, dt: DataType, nullable: bool) -> FieldRef {
165+
Arc::new(Field::new(name, dt, nullable))
166+
}
167+
168+
fn ext_field(name: &str, dt: DataType, extension_name: &str) -> FieldRef {
169+
Arc::new(Field::new(name, dt, true).with_metadata(HashMap::from([(
170+
EXTENSION_KEY.to_string(),
171+
extension_name.to_string(),
172+
)])))
173+
}
174+
175+
fn return_field(arg_fields: &[FieldRef]) -> FieldRef {
176+
let scalars = vec![None; arg_fields.len()];
177+
CoalesceFunc::new()
178+
.return_field_from_args(ReturnFieldArgs {
179+
arg_fields,
180+
scalar_arguments: &scalars,
181+
})
182+
.unwrap()
183+
}
184+
185+
#[test]
186+
fn propagates_metadata_when_arguments_agree() {
187+
let ret = return_field(&[
188+
ext_field("a", DataType::Binary, "geoarrow.wkb"),
189+
ext_field("b", DataType::Binary, "geoarrow.wkb"),
190+
]);
191+
assert_eq!(ret.data_type(), &DataType::Binary);
192+
assert!(ret.is_nullable());
193+
assert_eq!(
194+
ret.metadata().get(EXTENSION_KEY).map(String::as_str),
195+
Some("geoarrow.wkb")
196+
);
197+
}
198+
199+
#[test]
200+
fn null_literal_argument_does_not_block_metadata() {
201+
// An untyped NULL argument carries no metadata but cannot contribute
202+
// a typed value either, so it does not participate in the agreement.
203+
let ret = return_field(&[
204+
field("null", DataType::Null, true),
205+
ext_field("b", DataType::Binary, "geoarrow.wkb"),
206+
]);
207+
assert_eq!(ret.data_type(), &DataType::Binary);
208+
assert_eq!(
209+
ret.metadata().get(EXTENSION_KEY).map(String::as_str),
210+
Some("geoarrow.wkb")
211+
);
212+
}
213+
214+
#[test]
215+
fn drops_metadata_when_arguments_disagree() {
216+
let ret = return_field(&[
217+
ext_field("a", DataType::Binary, "geoarrow.wkb"),
218+
field("plain", DataType::Binary, true),
219+
]);
220+
assert_eq!(ret.data_type(), &DataType::Binary);
221+
assert!(ret.metadata().is_empty());
222+
}
223+
224+
#[test]
225+
fn no_metadata_when_all_arguments_are_null() {
226+
let ret = return_field(&[
227+
field("a", DataType::Null, true),
228+
field("b", DataType::Null, true),
229+
]);
230+
assert_eq!(ret.data_type(), &DataType::Null);
231+
assert!(ret.metadata().is_empty());
232+
}
233+
234+
#[test]
235+
fn nullability_is_unchanged_by_metadata_propagation() {
236+
let ret = return_field(&[
237+
ext_field("a", DataType::Binary, "geoarrow.wkb"),
238+
Arc::new(Field::new("b", DataType::Binary, false).with_metadata(
239+
HashMap::from([(EXTENSION_KEY.to_string(), "geoarrow.wkb".to_string())]),
240+
)),
241+
]);
242+
assert!(!ret.is_nullable());
243+
assert_eq!(
244+
ret.metadata().get(EXTENSION_KEY).map(String::as_str),
245+
Some("geoarrow.wkb")
246+
);
247+
}
248+
}

‎datafusion/functions/src/core/nvl2.rs‎

Lines changed: 63 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
// specific language governing permissions and limitations
1616
// under the License.
1717

18+
use crate::utils::unanimous_metadata;
1819
use arrow::datatypes::{DataType, Field, FieldRef};
1920
use datafusion_common::{Result, internal_err, utils::take_function_args};
2021
use datafusion_expr::{
@@ -94,7 +95,15 @@ impl ScalarUDFImpl for NVL2Func {
9495
let nullable =
9596
args.arg_fields[1].is_nullable() || args.arg_fields[2].is_nullable();
9697
let return_type = args.arg_fields[1].data_type().clone();
97-
Ok(Field::new(self.name(), return_type, nullable).into())
98+
// The result value comes from the second or third argument; propagate
99+
// field metadata (e.g. Arrow extension types) only when they agree on
100+
// it. The first argument is only tested for NULL and does not
101+
// contribute a value.
102+
let metadata =
103+
unanimous_metadata(args.arg_fields[1..3].iter().map(|f| f.as_ref()));
104+
Ok(Field::new(self.name(), return_type, nullable)
105+
.with_metadata(metadata)
106+
.into())
98107
}
99108

100109
fn invoke_with_args(&self, _args: ScalarFunctionArgs) -> Result<ColumnarValue> {
@@ -145,3 +154,56 @@ impl ScalarUDFImpl for NVL2Func {
145154
self.doc()
146155
}
147156
}
157+
158+
#[cfg(test)]
159+
mod tests {
160+
use super::*;
161+
use std::collections::HashMap;
162+
use std::sync::Arc;
163+
164+
const EXTENSION_KEY: &str = "ARROW:extension:name";
165+
166+
fn ext_field(name: &str, dt: DataType, extension_name: &str) -> FieldRef {
167+
Arc::new(Field::new(name, dt, true).with_metadata(HashMap::from([(
168+
EXTENSION_KEY.to_string(),
169+
extension_name.to_string(),
170+
)])))
171+
}
172+
173+
fn return_field(arg_fields: &[FieldRef]) -> FieldRef {
174+
let scalars = vec![None; arg_fields.len()];
175+
NVL2Func::new()
176+
.return_field_from_args(ReturnFieldArgs {
177+
arg_fields,
178+
scalar_arguments: &scalars,
179+
})
180+
.unwrap()
181+
}
182+
183+
#[test]
184+
fn propagates_metadata_from_value_arguments() {
185+
// The first argument is only tested for NULL; its metadata must not
186+
// leak into the result even when the value arguments agree.
187+
let ret = return_field(&[
188+
ext_field("test", DataType::Binary, "unrelated.type"),
189+
ext_field("if_non_null", DataType::Binary, "geoarrow.wkb"),
190+
ext_field("if_null", DataType::Binary, "geoarrow.wkb"),
191+
]);
192+
assert_eq!(ret.data_type(), &DataType::Binary);
193+
assert_eq!(
194+
ret.metadata().get(EXTENSION_KEY).map(String::as_str),
195+
Some("geoarrow.wkb")
196+
);
197+
}
198+
199+
#[test]
200+
fn drops_metadata_when_value_arguments_disagree() {
201+
let ret = return_field(&[
202+
Arc::new(Field::new("test", DataType::Boolean, true)),
203+
ext_field("if_non_null", DataType::Binary, "geoarrow.wkb"),
204+
Arc::new(Field::new("if_null", DataType::Binary, true)),
205+
]);
206+
assert_eq!(ret.data_type(), &DataType::Binary);
207+
assert!(ret.metadata().is_empty());
208+
}
209+
}

‎datafusion/functions/src/utils.rs‎

Lines changed: 22 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,12 +17,13 @@
1717

1818
use arrow::array::{Array, ArrayRef, ArrowPrimitiveType, AsArray, PrimitiveArray};
1919
use arrow::compute::try_binary;
20-
use arrow::datatypes::{DataType, DecimalType};
20+
use arrow::datatypes::{DataType, DecimalType, Field};
2121
use arrow::error::ArrowError;
2222
use datafusion_common::{DataFusionError, Result, ScalarValue};
2323
use datafusion_expr::ColumnarValue;
2424
use datafusion_expr::function::Hint;
2525
use std::cmp::Ordering;
26+
use std::collections::HashMap;
2627
use std::sync::Arc;
2728

2829
/// Creates a function to identify the optimal return type of a string function given
@@ -69,6 +70,26 @@ macro_rules! get_optimal_return_type {
6970
};
7071
}
7172

73+
/// Returns the field metadata shared by every argument that can contribute a
74+
/// value to a conditional function's result.
75+
///
76+
/// Fields with a `Null` data type (untyped NULL literals) carry no metadata
77+
/// and are ignored. If the remaining fields disagree on metadata, the result
78+
/// carries none: propagating one argument's metadata (e.g. an Arrow extension
79+
/// type name) would claim a type identity for values that other arguments may
80+
/// supply without it.
81+
pub(crate) fn unanimous_metadata<'a>(
82+
fields: impl Iterator<Item = &'a Field>,
83+
) -> HashMap<String, String> {
84+
let mut candidates = fields.filter(|f| !f.data_type().is_null());
85+
match candidates.next() {
86+
Some(first) if candidates.all(|f| f.metadata() == first.metadata()) => {
87+
first.metadata().clone()
88+
}
89+
_ => HashMap::new(),
90+
}
91+
}
92+
7293
// `utf8_to_str_type`: returns either a Utf8 or LargeUtf8 based on the input type size.
7394
get_optimal_return_type!(utf8_to_str_type, DataType::LargeUtf8, DataType::Utf8);
7495

0 commit comments

Comments
 (0)