@@ -64,8 +64,15 @@ impl GetArrayStructFields {
6464 }
6565
6666 fn child_field ( & self , input_schema : & Schema ) -> DataFusionResult < FieldRef > {
67- match self . list_field ( input_schema) ?. data_type ( ) {
68- DataType :: Struct ( fields) => Ok ( Arc :: clone ( & fields[ self . ordinal ] ) ) ,
67+ let list_field = self . list_field ( input_schema) ?;
68+ match list_field. data_type ( ) {
69+ DataType :: Struct ( fields) => {
70+ let field = & fields[ self . ordinal ] ;
71+ // A null struct element yields null even when the field itself is required.
72+ Ok ( Arc :: new ( field. as_ref ( ) . clone ( ) . with_nullable (
73+ list_field. is_nullable ( ) || field. is_nullable ( ) ,
74+ ) ) )
75+ }
6976 data_type => Err ( DataFusionError :: Internal ( format ! (
7077 "Unexpected data type in GetArrayStructFields: {data_type:?}"
7178 ) ) ) ,
@@ -86,23 +93,23 @@ impl PhysicalExpr for GetArrayStructFields {
8693 }
8794
8895 fn nullable ( & self , input_schema : & Schema ) -> DataFusionResult < bool > {
89- Ok ( self . list_field ( input_schema) ?. is_nullable ( )
90- || self . child_field ( input_schema) ?. is_nullable ( ) )
96+ self . child . nullable ( input_schema)
9197 }
9298
9399 fn evaluate ( & self , batch : & RecordBatch ) -> DataFusionResult < ColumnarValue > {
94100 let child_value = self . child . evaluate ( batch) ?. into_array ( batch. num_rows ( ) ) ?;
101+ let field = self . child_field ( batch. schema ( ) . as_ref ( ) ) ?;
95102
96103 match child_value. data_type ( ) {
97104 DataType :: List ( _) => {
98105 let list_array = as_list_array ( & child_value) ?;
99106
100- get_array_struct_fields ( list_array, self . ordinal )
107+ get_array_struct_fields ( list_array, self . ordinal , field )
101108 }
102109 DataType :: LargeList ( _) => {
103110 let list_array = as_large_list_array ( & child_value) ?;
104111
105- get_array_struct_fields ( list_array, self . ordinal )
112+ get_array_struct_fields ( list_array, self . ordinal , field )
106113 }
107114 data_type => Err ( DataFusionError :: Internal ( format ! (
108115 "Unexpected child type for ListExtract: {data_type:?}"
@@ -135,14 +142,14 @@ impl PhysicalExpr for GetArrayStructFields {
135142fn get_array_struct_fields < O : OffsetSizeTrait > (
136143 list_array : & GenericListArray < O > ,
137144 ordinal : usize ,
145+ field : FieldRef ,
138146) -> DataFusionResult < ColumnarValue > {
139147 let values = list_array
140148 . values ( )
141149 . as_any ( )
142150 . downcast_ref :: < StructArray > ( )
143151 . expect ( "A StructType is expected" ) ;
144152
145- let field = Arc :: clone ( & values. fields ( ) [ ordinal] ) ;
146153 // A field of a null struct is null, and Arrow keeps the children's validity independent of the
147154 // parent's, so the parent's nulls have to be unioned in. See
148155 // `datafusion_comet_common::struct_nulls`.
@@ -171,3 +178,75 @@ impl Display for GetArrayStructFields {
171178 )
172179 }
173180}
181+
182+ #[ cfg( test) ]
183+ mod tests {
184+ use super :: * ;
185+ use arrow:: array:: Int32Array ;
186+ use arrow:: buffer:: { NullBuffer , OffsetBuffer } ;
187+ use arrow:: datatypes:: Field ;
188+ use datafusion:: physical_expr:: expressions:: Column ;
189+
190+ fn check_nullability < O : OffsetSizeTrait > ( ) {
191+ for list_nullable in [ true , false ] {
192+ for element_nullable in [ true , false ] {
193+ for field_nullable in [ false , true ] {
194+ let field = Arc :: new ( Field :: new ( "a" , DataType :: Int32 , field_nullable) ) ;
195+ let values = Arc :: new ( StructArray :: new (
196+ vec ! [ field] . into ( ) ,
197+ vec ! [ Arc :: new( Int32Array :: from( vec![ 1 , 99 ] ) ) ] ,
198+ element_nullable. then ( || NullBuffer :: from ( vec ! [ true , false ] ) ) ,
199+ ) ) ;
200+ let list = Arc :: new ( GenericListArray :: < O > :: new (
201+ Arc :: new ( Field :: new (
202+ "element" ,
203+ values. data_type ( ) . clone ( ) ,
204+ element_nullable,
205+ ) ) ,
206+ OffsetBuffer :: from_lengths ( [ 1 , 1 , 0 , 0 ] ) ,
207+ values,
208+ list_nullable. then ( || NullBuffer :: from ( vec ! [ true , true , true , false ] ) ) ,
209+ ) ) ;
210+ let schema = Arc :: new ( Schema :: new ( vec ! [ Field :: new(
211+ "l" ,
212+ list. data_type( ) . clone( ) ,
213+ list_nullable,
214+ ) ] ) ) ;
215+ let batch =
216+ RecordBatch :: try_new ( Arc :: clone ( & schema) , vec ! [ list. clone( ) ] ) . unwrap ( ) ;
217+ let expr = GetArrayStructFields :: new ( Arc :: new ( Column :: new ( "l" , 0 ) ) , 0 ) ;
218+ assert_eq ! ( expr. nullable( & schema) . unwrap( ) , list_nullable) ;
219+ let result = expr. evaluate ( & batch) . unwrap ( ) . into_array ( 4 ) . unwrap ( ) ;
220+ assert_eq ! ( expr. data_type( & schema) . unwrap( ) , * result. data_type( ) ) ;
221+ let result = result
222+ . as_any ( )
223+ . downcast_ref :: < GenericListArray < O > > ( )
224+ . unwrap ( ) ;
225+ let output_field = match result. data_type ( ) {
226+ DataType :: List ( field) | DataType :: LargeList ( field) => field,
227+ _ => unreachable ! ( ) ,
228+ } ;
229+ assert_eq ! (
230+ output_field. is_nullable( ) ,
231+ element_nullable || field_nullable
232+ ) ;
233+ let values = result
234+ . values ( )
235+ . as_any ( )
236+ . downcast_ref :: < Int32Array > ( )
237+ . unwrap ( ) ;
238+ assert_eq ! ( values. value( 0 ) , 1 ) ;
239+ assert_eq ! ( values. is_null( 1 ) , element_nullable) ;
240+ assert_eq ! ( result. offsets( ) , list. offsets( ) ) ;
241+ assert_eq ! ( result. nulls( ) , list. nulls( ) ) ;
242+ }
243+ }
244+ }
245+ }
246+
247+ #[ test]
248+ fn get_array_struct_fields_nullability ( ) {
249+ check_nullability :: < i32 > ( ) ;
250+ check_nullability :: < i64 > ( ) ;
251+ }
252+ }
0 commit comments