Skip to content

Commit 55feb9b

Browse files
mandrushclaude
authored andcommitted
fix: nested Map<K, Struct> cast fix
Cherry-picked from 503f3bb (merged via c823de2) and adapted to 1.x. DataFusion's default physical expr adapter cannot null-fill a struct nested in a Map value (apache/datafusion#20835, fixed in #23914, not yet released). Route columns whose type differs from the table schema through delta-rs's own recursive cast (`cast_array_to_field`). On 1.x this is composed with the existing nested-nullability relaxation in `DeltaPhysicalExprAdapterFactory`: the logical schema is relaxed first, so files that only differ in nested nullability are passed through unchanged. Co-Authored-By: Claude Sonnet 5.5 <[email protected]>
1 parent 9a342fd commit 55feb9b

3 files changed

Lines changed: 378 additions & 9 deletions

File tree

‎crates/core/src/delta_datafusion/table_provider/next/scan/expr_adapter.rs‎

Lines changed: 357 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,23 @@
11
//! Delta-aware physical expression adaptation for parquet scans.
22
3+
use std::fmt::{self, Display, Formatter};
4+
use std::hash::Hash;
35
use std::sync::Arc;
46

7+
use arrow_array::RecordBatch;
58
use arrow_schema::{DataType, Field, FieldRef, Fields, Schema, SchemaRef};
6-
use datafusion::common::Result;
7-
use datafusion_physical_expr_adapter::{
8-
DefaultPhysicalExprAdapterFactory, PhysicalExprAdapter, PhysicalExprAdapterFactory,
9-
};
9+
use datafusion::common::tree_node::{Transformed, TransformedResult, TreeNode};
10+
use datafusion::common::{exec_err, Result, ScalarValue};
11+
use datafusion::functions::core::getfield::GetFieldFunc;
12+
use datafusion::logical_expr::ColumnarValue;
13+
use datafusion::physical_expr::expressions::{lit, Column, Literal};
14+
use datafusion::physical_expr::{PhysicalExpr, ScalarFunctionExpr};
15+
use datafusion_physical_expr_adapter::{PhysicalExprAdapter, PhysicalExprAdapterFactory};
16+
17+
use crate::kernel::schema::cast::cast_array_to_field;
1018

1119
/// A [`PhysicalExprAdapterFactory`] that relaxes nested-field nullability on the
12-
/// logical (table) schema before delegating to DataFusion's default adapter.
20+
/// logical (table) schema before adapting file columns to it.
1321
///
1422
/// In Delta, `nullable = false` is a write-time invariant: engines such as Spark
1523
/// write parquet files whose nested fields are physically optional even when the
@@ -18,6 +26,13 @@ use datafusion_physical_expr_adapter::{
1826
/// a nullability-relaxed target schema instead. The data is guaranteed non-null
1927
/// by the writer, and the logical output schema is restored above the parquet
2028
/// scan by [`DeltaScanExec`](super::DeltaScanExec)'s transforms.
29+
///
30+
/// Columns whose type still differs after relaxation are cast with delta-rs's own recursive,
31+
/// Map-aware cast instead of DataFusion's, which cannot null-fill a struct nested in a Map
32+
/// value. Upstream DataFusion fix: <https://github.com/apache/datafusion/pull/23914>, tracking
33+
/// <https://github.com/apache/datafusion/issues/20835>. Once this crate builds against a
34+
/// DataFusion release containing that fix, the adapter below can be replaced by
35+
/// `DefaultPhysicalExprAdapterFactory`.
2136
#[derive(Debug)]
2237
pub(crate) struct DeltaPhysicalExprAdapterFactory;
2338

@@ -27,10 +42,220 @@ impl PhysicalExprAdapterFactory for DeltaPhysicalExprAdapterFactory {
2742
logical_file_schema: SchemaRef,
2843
physical_file_schema: SchemaRef,
2944
) -> Result<Arc<dyn PhysicalExprAdapter>> {
30-
DefaultPhysicalExprAdapterFactory.create(
31-
Arc::new(relax_schema_nested_nullability(&logical_file_schema)),
45+
Ok(Arc::new(DeltaExprAdapter {
46+
logical_file_schema: Arc::new(relax_schema_nested_nullability(&logical_file_schema)),
3247
physical_file_schema,
33-
)
48+
}))
49+
}
50+
}
51+
52+
#[derive(Debug)]
53+
struct DeltaExprAdapter {
54+
logical_file_schema: SchemaRef,
55+
physical_file_schema: SchemaRef,
56+
}
57+
58+
impl PhysicalExprAdapter for DeltaExprAdapter {
59+
fn rewrite(&self, expr: Arc<dyn PhysicalExpr>) -> Result<Arc<dyn PhysicalExpr>> {
60+
expr.transform(|e| {
61+
if let Some(narrowed) = self.try_narrow_struct_cast(&e)? {
62+
return Ok(Transformed::yes(narrowed));
63+
}
64+
65+
let Some(col) = e.downcast_ref::<Column>() else {
66+
return Ok(Transformed::no(e));
67+
};
68+
69+
let Ok(logical_field) = self.logical_file_schema.field_with_name(col.name()) else {
70+
return Ok(Transformed::no(e));
71+
};
72+
73+
let physical_idx = match self.physical_file_schema.index_of(col.name()) {
74+
Ok(idx) => idx,
75+
Err(_) if !logical_field.is_nullable() => {
76+
return exec_err!(
77+
"Non-nullable column '{}' is missing from the physical schema",
78+
col.name()
79+
);
80+
}
81+
// missing entirely: read it as nulls.
82+
Err(_) => {
83+
let null = ScalarValue::Null.cast_to(logical_field.data_type())?;
84+
return Ok(Transformed::yes(lit(null)));
85+
}
86+
};
87+
88+
if logical_field.data_type()
89+
== self.physical_file_schema.field(physical_idx).data_type()
90+
{
91+
if col.index() == physical_idx {
92+
return Ok(Transformed::no(e));
93+
}
94+
return Ok(Transformed::yes(Arc::new(Column::new(
95+
col.name(),
96+
physical_idx,
97+
))));
98+
}
99+
100+
// Types differ. Cast with delta-rs's own recursive, Map-aware cast rather than
101+
// DataFusion's.
102+
let resolved = if col.index() == physical_idx {
103+
col.clone()
104+
} else {
105+
Column::new(col.name(), physical_idx)
106+
};
107+
Ok(Transformed::yes(Arc::new(DeltaCastColumn {
108+
expr: Arc::new(resolved),
109+
target_field: Arc::new(logical_field.clone()),
110+
})))
111+
})
112+
.data()
113+
}
114+
}
115+
116+
impl DeltaExprAdapter {
117+
/// Rewrite `get_field(delta_cast(s), 'a', ...)` into `delta_cast(get_field(s, 'a', ...))`,
118+
/// mirroring `DefaultPhysicalExprAdapter`. Besides avoiding a whole-struct cast, this keeps
119+
/// the column visible to the parquet scan, which otherwise reports a struct-field predicate
120+
/// as handled and then silently drops it: <https://github.com/apache/datafusion/issues/24109>.
121+
/// Map lookups are left alone, since their keys are runtime values.
122+
fn try_narrow_struct_cast(
123+
&self,
124+
expr: &Arc<dyn PhysicalExpr>,
125+
) -> Result<Option<Arc<dyn PhysicalExpr>>> {
126+
let Some(get_field) = ScalarFunctionExpr::try_downcast_func::<GetFieldFunc>(expr.as_ref())
127+
else {
128+
return Ok(None);
129+
};
130+
let Some((source, key_exprs)) = get_field.args().split_first() else {
131+
return Ok(None);
132+
};
133+
let Some(cast) = source.downcast_ref::<DeltaCastColumn>() else {
134+
return Ok(None);
135+
};
136+
137+
let mut path = Vec::with_capacity(key_exprs.len());
138+
for key in key_exprs {
139+
let Some(name) = key
140+
.downcast_ref::<Literal>()
141+
.and_then(|l| l.value().try_as_str().flatten())
142+
else {
143+
return Ok(None);
144+
};
145+
path.push(name);
146+
}
147+
let FieldPath::Found(logical_leaf) =
148+
resolve_field_path(cast.target_field.data_type(), &path)
149+
else {
150+
return Ok(None);
151+
};
152+
let physical_type = cast.expr.data_type(&self.physical_file_schema)?;
153+
let physical_leaf = match resolve_field_path(&physical_type, &path) {
154+
FieldPath::Found(field) => field,
155+
// The file predates this field, so it reads as null.
156+
FieldPath::Missing => {
157+
let null = ScalarValue::Null.cast_to(logical_leaf.data_type())?;
158+
return Ok(Some(lit(null)));
159+
}
160+
FieldPath::Unresolvable => return Ok(None),
161+
};
162+
163+
let mut args = Vec::with_capacity(get_field.args().len());
164+
args.push(Arc::clone(&cast.expr));
165+
args.extend(key_exprs.iter().cloned());
166+
let extracted: Arc<dyn PhysicalExpr> = Arc::new(ScalarFunctionExpr::try_new(
167+
Arc::new(get_field.fun().clone()),
168+
args,
169+
&self.physical_file_schema,
170+
Arc::new(get_field.config_options().clone()),
171+
)?);
172+
173+
if physical_leaf.data_type() == logical_leaf.data_type() {
174+
return Ok(Some(extracted));
175+
}
176+
Ok(Some(Arc::new(DeltaCastColumn {
177+
expr: extracted,
178+
target_field: Arc::clone(logical_leaf),
179+
})))
180+
}
181+
}
182+
183+
enum FieldPath<'a> {
184+
Found(&'a FieldRef),
185+
Missing,
186+
/// Empty path, or a non-struct along the way (e.g. a Map).
187+
Unresolvable,
188+
}
189+
190+
/// Follows a `get_field` key path through nested structs.
191+
fn resolve_field_path<'a>(dt: &'a DataType, path: &[&str]) -> FieldPath<'a> {
192+
let (DataType::Struct(fields), Some((first, rest))) = (dt, path.split_first()) else {
193+
return FieldPath::Unresolvable;
194+
};
195+
match fields.iter().find(|f| f.name() == first) {
196+
None => FieldPath::Missing,
197+
Some(field) if rest.is_empty() => FieldPath::Found(field),
198+
Some(field) => resolve_field_path(field.data_type(), rest),
199+
}
200+
}
201+
202+
/// Evaluates `expr` and casts the result to `target_field`, null-filling fields the file
203+
/// predates.
204+
#[derive(Debug, Clone, Eq)]
205+
struct DeltaCastColumn {
206+
expr: Arc<dyn PhysicalExpr>,
207+
target_field: FieldRef,
208+
}
209+
210+
impl PartialEq for DeltaCastColumn {
211+
fn eq(&self, other: &Self) -> bool {
212+
self.expr.eq(&other.expr) && self.target_field == other.target_field
213+
}
214+
}
215+
216+
impl Hash for DeltaCastColumn {
217+
fn hash<H: std::hash::Hasher>(&self, state: &mut H) {
218+
self.expr.hash(state);
219+
self.target_field.hash(state);
220+
}
221+
}
222+
223+
impl Display for DeltaCastColumn {
224+
fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
225+
write!(f, "delta_cast({}, {})", self.expr, self.target_field.name())
226+
}
227+
}
228+
229+
impl PhysicalExpr for DeltaCastColumn {
230+
fn evaluate(&self, batch: &RecordBatch) -> Result<ColumnarValue> {
231+
let array = self.expr.evaluate(batch)?.into_array(batch.num_rows())?;
232+
let cast = cast_array_to_field(&array, &self.target_field, false, true)?;
233+
Ok(ColumnarValue::Array(cast))
234+
}
235+
236+
fn return_field(&self, _input_schema: &Schema) -> Result<FieldRef> {
237+
Ok(Arc::clone(&self.target_field))
238+
}
239+
240+
fn children(&self) -> Vec<&Arc<dyn PhysicalExpr>> {
241+
vec![&self.expr]
242+
}
243+
244+
fn with_new_children(
245+
self: Arc<Self>,
246+
mut children: Vec<Arc<dyn PhysicalExpr>>,
247+
) -> Result<Arc<dyn PhysicalExpr>> {
248+
if children.len() != 1 {
249+
return exec_err!("DeltaCastColumn expects exactly one child");
250+
}
251+
Ok(Arc::new(Self {
252+
expr: children.remove(0),
253+
target_field: Arc::clone(&self.target_field),
254+
}))
255+
}
256+
257+
fn fmt_sql(&self, f: &mut Formatter<'_>) -> fmt::Result {
258+
write!(f, "{self}")
34259
}
35260
}
36261

@@ -218,4 +443,128 @@ mod tests {
218443
};
219444
assert!(fields.iter().all(|f| f.is_nullable()));
220445
}
446+
447+
#[test]
448+
fn narrows_cast_under_nested_struct_field_access() {
449+
use datafusion::config::ConfigOptions;
450+
use datafusion::functions::core::get_field;
451+
452+
let inner = |extra: bool| {
453+
let mut fields = vec![Field::new("name", DataType::Utf8, true)];
454+
if extra {
455+
fields.push(Field::new("extra", DataType::Int32, true));
456+
}
457+
Field::new("a", DataType::Struct(Fields::from(fields)), true)
458+
};
459+
let physical = Arc::new(Schema::new(vec![Field::new(
460+
"s",
461+
DataType::Struct(Fields::from(vec![inner(false)])),
462+
true,
463+
)]));
464+
let logical = Arc::new(Schema::new(vec![Field::new(
465+
"s",
466+
DataType::Struct(Fields::from(vec![
467+
inner(true),
468+
Field::new("b", DataType::Int32, true),
469+
])),
470+
true,
471+
)]));
472+
473+
let access = |keys: &[&str]| -> Arc<dyn PhysicalExpr> {
474+
let mut args: Vec<Arc<dyn PhysicalExpr>> = vec![Arc::new(Column::new("s", 0))];
475+
args.extend(keys.iter().map(|k| lit(*k)));
476+
Arc::new(
477+
ScalarFunctionExpr::try_new(
478+
get_field(),
479+
args,
480+
&logical,
481+
Arc::new(ConfigOptions::default()),
482+
)
483+
.unwrap(),
484+
)
485+
};
486+
487+
let adapter = DeltaPhysicalExprAdapterFactory
488+
.create(logical.clone(), physical)
489+
.unwrap();
490+
491+
let name = adapter.rewrite(access(&["a", "name"])).unwrap();
492+
assert!(
493+
ScalarFunctionExpr::try_downcast_func::<GetFieldFunc>(name.as_ref())
494+
.is_some_and(|f| f.args()[0].downcast_ref::<Column>().is_some()),
495+
"expected get_field directly over the column, got {name}"
496+
);
497+
498+
let missing = adapter.rewrite(access(&["b"])).unwrap();
499+
let missing = missing.downcast_ref::<Literal>().expect("null literal");
500+
assert_eq!(missing.value(), &ScalarValue::Int32(None));
501+
}
502+
503+
#[test]
504+
fn adapts_map_value_struct_missing_a_field() {
505+
use arrow_array::builder::{
506+
ArrayBuilder, Int32Builder, MapBuilder, MapFieldNames, StringBuilder, StructBuilder,
507+
};
508+
509+
fn map_type(value_fields: Fields) -> DataType {
510+
let entries = Field::new(
511+
"entries",
512+
DataType::Struct(Fields::from(vec![
513+
Field::new("key", DataType::Utf8, false),
514+
Field::new("value", DataType::Struct(value_fields), true),
515+
])),
516+
false,
517+
);
518+
DataType::Map(Arc::new(entries), false)
519+
}
520+
521+
let physical_value = Fields::from(vec![Field::new("a", DataType::Int32, true)]);
522+
let logical_value = Fields::from(vec![
523+
Field::new("a", DataType::Int32, true),
524+
Field::new("b", DataType::Utf8, true),
525+
]);
526+
let physical = Arc::new(Schema::new(vec![Field::new(
527+
"m",
528+
map_type(physical_value.clone()),
529+
true,
530+
)]));
531+
let logical = Arc::new(Schema::new(vec![Field::new(
532+
"m",
533+
map_type(logical_value),
534+
true,
535+
)]));
536+
537+
let values = StructBuilder::new(
538+
physical_value,
539+
vec![Box::new(Int32Builder::new()) as Box<dyn ArrayBuilder>],
540+
);
541+
let names = MapFieldNames {
542+
entry: "entries".into(),
543+
key: "key".into(),
544+
value: "value".into(),
545+
};
546+
let mut builder = MapBuilder::new(Some(names), StringBuilder::new(), values);
547+
builder.keys().append_value("k");
548+
builder
549+
.values()
550+
.field_builder::<Int32Builder>(0)
551+
.unwrap()
552+
.append_value(1);
553+
builder.values().append(true);
554+
builder.append(true).unwrap();
555+
let batch =
556+
RecordBatch::try_new(physical.clone(), vec![Arc::new(builder.finish())]).unwrap();
557+
558+
let adapter = DeltaPhysicalExprAdapterFactory
559+
.create(logical.clone(), physical)
560+
.unwrap();
561+
let rewritten = adapter.rewrite(Arc::new(Column::new("m", 0))).unwrap();
562+
let out = rewritten
563+
.evaluate(&batch)
564+
.unwrap()
565+
.into_array(batch.num_rows())
566+
.unwrap();
567+
568+
assert_eq!(out.data_type(), logical.field(0).data_type());
569+
}
221570
}

0 commit comments

Comments
 (0)