11//! Delta-aware physical expression adaptation for parquet scans.
22
3+ use std:: fmt:: { self , Display , Formatter } ;
4+ use std:: hash:: Hash ;
35use std:: sync:: Arc ;
46
7+ use arrow_array:: RecordBatch ;
58use arrow_schema:: { DataType , Field , FieldRef , Fields , Schema , SchemaRef } ;
6- use datafusion:: common:: Result ;
7- use datafusion_physical_expr_adapter:: {
8- DefaultPhysicalExprAdapterFactory , PhysicalExprAdapter , PhysicalExprAdapterFactory ,
9- } ;
9+ use datafusion:: common:: tree_node:: { Transformed , TransformedResult , TreeNode } ;
10+ use datafusion:: common:: { exec_err, Result , ScalarValue } ;
11+ use datafusion:: functions:: core:: getfield:: GetFieldFunc ;
12+ use datafusion:: logical_expr:: ColumnarValue ;
13+ use datafusion:: physical_expr:: expressions:: { lit, Column , Literal } ;
14+ use datafusion:: physical_expr:: { PhysicalExpr , ScalarFunctionExpr } ;
15+ use datafusion_physical_expr_adapter:: { PhysicalExprAdapter , PhysicalExprAdapterFactory } ;
16+
17+ use crate :: kernel:: schema:: cast:: cast_array_to_field;
1018
1119/// A [`PhysicalExprAdapterFactory`] that relaxes nested-field nullability on the
12- /// logical (table) schema before delegating to DataFusion's default adapter .
20+ /// logical (table) schema before adapting file columns to it .
1321///
1422/// In Delta, `nullable = false` is a write-time invariant: engines such as Spark
1523/// write parquet files whose nested fields are physically optional even when the
@@ -18,6 +26,13 @@ use datafusion_physical_expr_adapter::{
1826/// a nullability-relaxed target schema instead. The data is guaranteed non-null
1927/// by the writer, and the logical output schema is restored above the parquet
2028/// scan by [`DeltaScanExec`](super::DeltaScanExec)'s transforms.
29+ ///
30+ /// Columns whose type still differs after relaxation are cast with delta-rs's own recursive,
31+ /// Map-aware cast instead of DataFusion's, which cannot null-fill a struct nested in a Map
32+ /// value. Upstream DataFusion fix: <https://github.com/apache/datafusion/pull/23914>, tracking
33+ /// <https://github.com/apache/datafusion/issues/20835>. Once this crate builds against a
34+ /// DataFusion release containing that fix, the adapter below can be replaced by
35+ /// `DefaultPhysicalExprAdapterFactory`.
2136#[ derive( Debug ) ]
2237pub ( crate ) struct DeltaPhysicalExprAdapterFactory ;
2338
@@ -27,10 +42,220 @@ impl PhysicalExprAdapterFactory for DeltaPhysicalExprAdapterFactory {
2742 logical_file_schema : SchemaRef ,
2843 physical_file_schema : SchemaRef ,
2944 ) -> Result < Arc < dyn PhysicalExprAdapter > > {
30- DefaultPhysicalExprAdapterFactory . create (
31- Arc :: new ( relax_schema_nested_nullability ( & logical_file_schema) ) ,
45+ Ok ( Arc :: new ( DeltaExprAdapter {
46+ logical_file_schema : Arc :: new ( relax_schema_nested_nullability ( & logical_file_schema) ) ,
3247 physical_file_schema,
33- )
48+ } ) )
49+ }
50+ }
51+
52+ #[ derive( Debug ) ]
53+ struct DeltaExprAdapter {
54+ logical_file_schema : SchemaRef ,
55+ physical_file_schema : SchemaRef ,
56+ }
57+
58+ impl PhysicalExprAdapter for DeltaExprAdapter {
59+ fn rewrite ( & self , expr : Arc < dyn PhysicalExpr > ) -> Result < Arc < dyn PhysicalExpr > > {
60+ expr. transform ( |e| {
61+ if let Some ( narrowed) = self . try_narrow_struct_cast ( & e) ? {
62+ return Ok ( Transformed :: yes ( narrowed) ) ;
63+ }
64+
65+ let Some ( col) = e. downcast_ref :: < Column > ( ) else {
66+ return Ok ( Transformed :: no ( e) ) ;
67+ } ;
68+
69+ let Ok ( logical_field) = self . logical_file_schema . field_with_name ( col. name ( ) ) else {
70+ return Ok ( Transformed :: no ( e) ) ;
71+ } ;
72+
73+ let physical_idx = match self . physical_file_schema . index_of ( col. name ( ) ) {
74+ Ok ( idx) => idx,
75+ Err ( _) if !logical_field. is_nullable ( ) => {
76+ return exec_err ! (
77+ "Non-nullable column '{}' is missing from the physical schema" ,
78+ col. name( )
79+ ) ;
80+ }
81+ // missing entirely: read it as nulls.
82+ Err ( _) => {
83+ let null = ScalarValue :: Null . cast_to ( logical_field. data_type ( ) ) ?;
84+ return Ok ( Transformed :: yes ( lit ( null) ) ) ;
85+ }
86+ } ;
87+
88+ if logical_field. data_type ( )
89+ == self . physical_file_schema . field ( physical_idx) . data_type ( )
90+ {
91+ if col. index ( ) == physical_idx {
92+ return Ok ( Transformed :: no ( e) ) ;
93+ }
94+ return Ok ( Transformed :: yes ( Arc :: new ( Column :: new (
95+ col. name ( ) ,
96+ physical_idx,
97+ ) ) ) ) ;
98+ }
99+
100+ // Types differ. Cast with delta-rs's own recursive, Map-aware cast rather than
101+ // DataFusion's.
102+ let resolved = if col. index ( ) == physical_idx {
103+ col. clone ( )
104+ } else {
105+ Column :: new ( col. name ( ) , physical_idx)
106+ } ;
107+ Ok ( Transformed :: yes ( Arc :: new ( DeltaCastColumn {
108+ expr : Arc :: new ( resolved) ,
109+ target_field : Arc :: new ( logical_field. clone ( ) ) ,
110+ } ) ) )
111+ } )
112+ . data ( )
113+ }
114+ }
115+
116+ impl DeltaExprAdapter {
117+ /// Rewrite `get_field(delta_cast(s), 'a', ...)` into `delta_cast(get_field(s, 'a', ...))`,
118+ /// mirroring `DefaultPhysicalExprAdapter`. Besides avoiding a whole-struct cast, this keeps
119+ /// the column visible to the parquet scan, which otherwise reports a struct-field predicate
120+ /// as handled and then silently drops it: <https://github.com/apache/datafusion/issues/24109>.
121+ /// Map lookups are left alone, since their keys are runtime values.
122+ fn try_narrow_struct_cast (
123+ & self ,
124+ expr : & Arc < dyn PhysicalExpr > ,
125+ ) -> Result < Option < Arc < dyn PhysicalExpr > > > {
126+ let Some ( get_field) = ScalarFunctionExpr :: try_downcast_func :: < GetFieldFunc > ( expr. as_ref ( ) )
127+ else {
128+ return Ok ( None ) ;
129+ } ;
130+ let Some ( ( source, key_exprs) ) = get_field. args ( ) . split_first ( ) else {
131+ return Ok ( None ) ;
132+ } ;
133+ let Some ( cast) = source. downcast_ref :: < DeltaCastColumn > ( ) else {
134+ return Ok ( None ) ;
135+ } ;
136+
137+ let mut path = Vec :: with_capacity ( key_exprs. len ( ) ) ;
138+ for key in key_exprs {
139+ let Some ( name) = key
140+ . downcast_ref :: < Literal > ( )
141+ . and_then ( |l| l. value ( ) . try_as_str ( ) . flatten ( ) )
142+ else {
143+ return Ok ( None ) ;
144+ } ;
145+ path. push ( name) ;
146+ }
147+ let FieldPath :: Found ( logical_leaf) =
148+ resolve_field_path ( cast. target_field . data_type ( ) , & path)
149+ else {
150+ return Ok ( None ) ;
151+ } ;
152+ let physical_type = cast. expr . data_type ( & self . physical_file_schema ) ?;
153+ let physical_leaf = match resolve_field_path ( & physical_type, & path) {
154+ FieldPath :: Found ( field) => field,
155+ // The file predates this field, so it reads as null.
156+ FieldPath :: Missing => {
157+ let null = ScalarValue :: Null . cast_to ( logical_leaf. data_type ( ) ) ?;
158+ return Ok ( Some ( lit ( null) ) ) ;
159+ }
160+ FieldPath :: Unresolvable => return Ok ( None ) ,
161+ } ;
162+
163+ let mut args = Vec :: with_capacity ( get_field. args ( ) . len ( ) ) ;
164+ args. push ( Arc :: clone ( & cast. expr ) ) ;
165+ args. extend ( key_exprs. iter ( ) . cloned ( ) ) ;
166+ let extracted: Arc < dyn PhysicalExpr > = Arc :: new ( ScalarFunctionExpr :: try_new (
167+ Arc :: new ( get_field. fun ( ) . clone ( ) ) ,
168+ args,
169+ & self . physical_file_schema ,
170+ Arc :: new ( get_field. config_options ( ) . clone ( ) ) ,
171+ ) ?) ;
172+
173+ if physical_leaf. data_type ( ) == logical_leaf. data_type ( ) {
174+ return Ok ( Some ( extracted) ) ;
175+ }
176+ Ok ( Some ( Arc :: new ( DeltaCastColumn {
177+ expr : extracted,
178+ target_field : Arc :: clone ( logical_leaf) ,
179+ } ) ) )
180+ }
181+ }
182+
183+ enum FieldPath < ' a > {
184+ Found ( & ' a FieldRef ) ,
185+ Missing ,
186+ /// Empty path, or a non-struct along the way (e.g. a Map).
187+ Unresolvable ,
188+ }
189+
190+ /// Follows a `get_field` key path through nested structs.
191+ fn resolve_field_path < ' a > ( dt : & ' a DataType , path : & [ & str ] ) -> FieldPath < ' a > {
192+ let ( DataType :: Struct ( fields) , Some ( ( first, rest) ) ) = ( dt, path. split_first ( ) ) else {
193+ return FieldPath :: Unresolvable ;
194+ } ;
195+ match fields. iter ( ) . find ( |f| f. name ( ) == first) {
196+ None => FieldPath :: Missing ,
197+ Some ( field) if rest. is_empty ( ) => FieldPath :: Found ( field) ,
198+ Some ( field) => resolve_field_path ( field. data_type ( ) , rest) ,
199+ }
200+ }
201+
202+ /// Evaluates `expr` and casts the result to `target_field`, null-filling fields the file
203+ /// predates.
204+ #[ derive( Debug , Clone , Eq ) ]
205+ struct DeltaCastColumn {
206+ expr : Arc < dyn PhysicalExpr > ,
207+ target_field : FieldRef ,
208+ }
209+
210+ impl PartialEq for DeltaCastColumn {
211+ fn eq ( & self , other : & Self ) -> bool {
212+ self . expr . eq ( & other. expr ) && self . target_field == other. target_field
213+ }
214+ }
215+
216+ impl Hash for DeltaCastColumn {
217+ fn hash < H : std:: hash:: Hasher > ( & self , state : & mut H ) {
218+ self . expr . hash ( state) ;
219+ self . target_field . hash ( state) ;
220+ }
221+ }
222+
223+ impl Display for DeltaCastColumn {
224+ fn fmt ( & self , f : & mut Formatter < ' _ > ) -> fmt:: Result {
225+ write ! ( f, "delta_cast({}, {})" , self . expr, self . target_field. name( ) )
226+ }
227+ }
228+
229+ impl PhysicalExpr for DeltaCastColumn {
230+ fn evaluate ( & self , batch : & RecordBatch ) -> Result < ColumnarValue > {
231+ let array = self . expr . evaluate ( batch) ?. into_array ( batch. num_rows ( ) ) ?;
232+ let cast = cast_array_to_field ( & array, & self . target_field , false , true ) ?;
233+ Ok ( ColumnarValue :: Array ( cast) )
234+ }
235+
236+ fn return_field ( & self , _input_schema : & Schema ) -> Result < FieldRef > {
237+ Ok ( Arc :: clone ( & self . target_field ) )
238+ }
239+
240+ fn children ( & self ) -> Vec < & Arc < dyn PhysicalExpr > > {
241+ vec ! [ & self . expr]
242+ }
243+
244+ fn with_new_children (
245+ self : Arc < Self > ,
246+ mut children : Vec < Arc < dyn PhysicalExpr > > ,
247+ ) -> Result < Arc < dyn PhysicalExpr > > {
248+ if children. len ( ) != 1 {
249+ return exec_err ! ( "DeltaCastColumn expects exactly one child" ) ;
250+ }
251+ Ok ( Arc :: new ( Self {
252+ expr : children. remove ( 0 ) ,
253+ target_field : Arc :: clone ( & self . target_field ) ,
254+ } ) )
255+ }
256+
257+ fn fmt_sql ( & self , f : & mut Formatter < ' _ > ) -> fmt:: Result {
258+ write ! ( f, "{self}" )
34259 }
35260}
36261
@@ -218,4 +443,128 @@ mod tests {
218443 } ;
219444 assert ! ( fields. iter( ) . all( |f| f. is_nullable( ) ) ) ;
220445 }
446+
447+ #[ test]
448+ fn narrows_cast_under_nested_struct_field_access ( ) {
449+ use datafusion:: config:: ConfigOptions ;
450+ use datafusion:: functions:: core:: get_field;
451+
452+ let inner = |extra : bool | {
453+ let mut fields = vec ! [ Field :: new( "name" , DataType :: Utf8 , true ) ] ;
454+ if extra {
455+ fields. push ( Field :: new ( "extra" , DataType :: Int32 , true ) ) ;
456+ }
457+ Field :: new ( "a" , DataType :: Struct ( Fields :: from ( fields) ) , true )
458+ } ;
459+ let physical = Arc :: new ( Schema :: new ( vec ! [ Field :: new(
460+ "s" ,
461+ DataType :: Struct ( Fields :: from( vec![ inner( false ) ] ) ) ,
462+ true ,
463+ ) ] ) ) ;
464+ let logical = Arc :: new ( Schema :: new ( vec ! [ Field :: new(
465+ "s" ,
466+ DataType :: Struct ( Fields :: from( vec![
467+ inner( true ) ,
468+ Field :: new( "b" , DataType :: Int32 , true ) ,
469+ ] ) ) ,
470+ true ,
471+ ) ] ) ) ;
472+
473+ let access = |keys : & [ & str ] | -> Arc < dyn PhysicalExpr > {
474+ let mut args: Vec < Arc < dyn PhysicalExpr > > = vec ! [ Arc :: new( Column :: new( "s" , 0 ) ) ] ;
475+ args. extend ( keys. iter ( ) . map ( |k| lit ( * k) ) ) ;
476+ Arc :: new (
477+ ScalarFunctionExpr :: try_new (
478+ get_field ( ) ,
479+ args,
480+ & logical,
481+ Arc :: new ( ConfigOptions :: default ( ) ) ,
482+ )
483+ . unwrap ( ) ,
484+ )
485+ } ;
486+
487+ let adapter = DeltaPhysicalExprAdapterFactory
488+ . create ( logical. clone ( ) , physical)
489+ . unwrap ( ) ;
490+
491+ let name = adapter. rewrite ( access ( & [ "a" , "name" ] ) ) . unwrap ( ) ;
492+ assert ! (
493+ ScalarFunctionExpr :: try_downcast_func:: <GetFieldFunc >( name. as_ref( ) )
494+ . is_some_and( |f| f. args( ) [ 0 ] . downcast_ref:: <Column >( ) . is_some( ) ) ,
495+ "expected get_field directly over the column, got {name}"
496+ ) ;
497+
498+ let missing = adapter. rewrite ( access ( & [ "b" ] ) ) . unwrap ( ) ;
499+ let missing = missing. downcast_ref :: < Literal > ( ) . expect ( "null literal" ) ;
500+ assert_eq ! ( missing. value( ) , & ScalarValue :: Int32 ( None ) ) ;
501+ }
502+
503+ #[ test]
504+ fn adapts_map_value_struct_missing_a_field ( ) {
505+ use arrow_array:: builder:: {
506+ ArrayBuilder , Int32Builder , MapBuilder , MapFieldNames , StringBuilder , StructBuilder ,
507+ } ;
508+
509+ fn map_type ( value_fields : Fields ) -> DataType {
510+ let entries = Field :: new (
511+ "entries" ,
512+ DataType :: Struct ( Fields :: from ( vec ! [
513+ Field :: new( "key" , DataType :: Utf8 , false ) ,
514+ Field :: new( "value" , DataType :: Struct ( value_fields) , true ) ,
515+ ] ) ) ,
516+ false ,
517+ ) ;
518+ DataType :: Map ( Arc :: new ( entries) , false )
519+ }
520+
521+ let physical_value = Fields :: from ( vec ! [ Field :: new( "a" , DataType :: Int32 , true ) ] ) ;
522+ let logical_value = Fields :: from ( vec ! [
523+ Field :: new( "a" , DataType :: Int32 , true ) ,
524+ Field :: new( "b" , DataType :: Utf8 , true ) ,
525+ ] ) ;
526+ let physical = Arc :: new ( Schema :: new ( vec ! [ Field :: new(
527+ "m" ,
528+ map_type( physical_value. clone( ) ) ,
529+ true ,
530+ ) ] ) ) ;
531+ let logical = Arc :: new ( Schema :: new ( vec ! [ Field :: new(
532+ "m" ,
533+ map_type( logical_value) ,
534+ true ,
535+ ) ] ) ) ;
536+
537+ let values = StructBuilder :: new (
538+ physical_value,
539+ vec ! [ Box :: new( Int32Builder :: new( ) ) as Box <dyn ArrayBuilder >] ,
540+ ) ;
541+ let names = MapFieldNames {
542+ entry : "entries" . into ( ) ,
543+ key : "key" . into ( ) ,
544+ value : "value" . into ( ) ,
545+ } ;
546+ let mut builder = MapBuilder :: new ( Some ( names) , StringBuilder :: new ( ) , values) ;
547+ builder. keys ( ) . append_value ( "k" ) ;
548+ builder
549+ . values ( )
550+ . field_builder :: < Int32Builder > ( 0 )
551+ . unwrap ( )
552+ . append_value ( 1 ) ;
553+ builder. values ( ) . append ( true ) ;
554+ builder. append ( true ) . unwrap ( ) ;
555+ let batch =
556+ RecordBatch :: try_new ( physical. clone ( ) , vec ! [ Arc :: new( builder. finish( ) ) ] ) . unwrap ( ) ;
557+
558+ let adapter = DeltaPhysicalExprAdapterFactory
559+ . create ( logical. clone ( ) , physical)
560+ . unwrap ( ) ;
561+ let rewritten = adapter. rewrite ( Arc :: new ( Column :: new ( "m" , 0 ) ) ) . unwrap ( ) ;
562+ let out = rewritten
563+ . evaluate ( & batch)
564+ . unwrap ( )
565+ . into_array ( batch. num_rows ( ) )
566+ . unwrap ( ) ;
567+
568+ assert_eq ! ( out. data_type( ) , logical. field( 0 ) . data_type( ) ) ;
569+ }
221570}
0 commit comments