1919use crate :: execution:: operators:: ExecutionError ;
2020use arrow:: {
2121 array:: ArrayData ,
22+ datatypes:: Field ,
23+ error:: ArrowError ,
2224 ffi:: { FFI_ArrowArray , FFI_ArrowSchema } ,
2325} ;
2426
27+ fn ffi_schema_for_field ( field : & Field ) -> Result < FFI_ArrowSchema , ArrowError > {
28+ if field. name ( ) . contains ( '\0' ) {
29+ // ArrowSchema names are NUL-terminated C strings. Spark owns the logical output name, so
30+ // substitute only the exported name while retaining the Field's type and metadata.
31+ let field = field
32+ . clone ( )
33+ . with_name ( field. name ( ) . replace ( '\0' , "\u{fffd} " ) ) ;
34+ FFI_ArrowSchema :: try_from ( & field)
35+ } else {
36+ FFI_ArrowSchema :: try_from ( field)
37+ }
38+ }
39+
2540pub trait SparkArrowConvert {
2641 /// Move Arrow Arrays to C data interface.
27- fn move_to_spark ( & self , array : i64 , schema : i64 ) -> Result < ( ) , ExecutionError > ;
42+ fn move_to_spark ( & self , field : & Field , array : i64 , schema : i64 ) -> Result < ( ) , ExecutionError > ;
2843}
2944
3045impl SparkArrowConvert for ArrayData {
3146 /// Move this ArrowData to pointers of Arrow C data interface.
32- fn move_to_spark ( & self , array : i64 , schema : i64 ) -> Result < ( ) , ExecutionError > {
47+ fn move_to_spark ( & self , field : & Field , array : i64 , schema : i64 ) -> Result < ( ) , ExecutionError > {
3348 let array_ptr = array as * mut FFI_ArrowArray ;
3449 let schema_ptr = schema as * mut FFI_ArrowSchema ;
3550
3651 let array_align = std:: mem:: align_of :: < FFI_ArrowArray > ( ) ;
3752 let schema_align = std:: mem:: align_of :: < FFI_ArrowSchema > ( ) ;
53+ let ffi_array = FFI_ArrowArray :: new ( self ) ;
54+ let ffi_schema = ffi_schema_for_field ( field) ?;
3855
3956 // Check if the pointer alignment is correct.
4057 if array_ptr. align_offset ( array_align) != 0 || schema_ptr. align_offset ( schema_align) != 0 {
4158 unsafe {
42- std:: ptr:: write_unaligned ( array_ptr, FFI_ArrowArray :: new ( self ) ) ;
43- std:: ptr:: write_unaligned ( schema_ptr, FFI_ArrowSchema :: try_from ( self . data_type ( ) ) ? ) ;
59+ std:: ptr:: write_unaligned ( array_ptr, ffi_array ) ;
60+ std:: ptr:: write_unaligned ( schema_ptr, ffi_schema ) ;
4461 }
4562 } else {
4663 // SAFETY: `array_ptr` and `schema_ptr` are aligned correctly.
@@ -55,8 +72,8 @@ impl SparkArrowConvert for ArrayData {
5572 "move_to_spark: schema_ptr not aligned"
5673 ) ;
5774 unsafe {
58- std:: ptr:: write ( array_ptr, FFI_ArrowArray :: new ( self ) ) ;
59- std:: ptr:: write ( schema_ptr, FFI_ArrowSchema :: try_from ( self . data_type ( ) ) ? ) ;
75+ std:: ptr:: write ( array_ptr, ffi_array ) ;
76+ std:: ptr:: write ( schema_ptr, ffi_schema ) ;
6077 }
6178 }
6279
@@ -65,3 +82,26 @@ impl SparkArrowConvert for ArrayData {
6582}
6683
6784pub use datafusion_comet_common:: bytes_to_i128;
85+
86+ #[ cfg( test) ]
87+ mod tests {
88+ use super :: * ;
89+ use arrow:: datatypes:: DataType ;
90+ use std:: collections:: HashMap ;
91+
92+ #[ test]
93+ fn test_ffi_schema_preserves_field_and_sanitizes_nul_name ( ) {
94+ let field = Field :: new ( "v\0 tail" , DataType :: Int32 , true ) . with_metadata ( HashMap :: from ( [ (
95+ "ARROW:extension:name" . to_string ( ) ,
96+ "example.logical-type" . to_string ( ) ,
97+ ) ] ) ) ;
98+
99+ let ffi_schema = ffi_schema_for_field ( & field) . unwrap ( ) ;
100+ let exported = Field :: try_from ( & ffi_schema) . unwrap ( ) ;
101+
102+ assert_eq ! ( exported. name( ) , "v\u{fffd} tail" ) ;
103+ assert_eq ! ( exported. data_type( ) , field. data_type( ) ) ;
104+ assert_eq ! ( exported. is_nullable( ) , field. is_nullable( ) ) ;
105+ assert_eq ! ( exported. metadata( ) , field. metadata( ) ) ;
106+ }
107+ }
0 commit comments