Skip to content

Commit a0c8f89

Browse files
committed
fix: preserve Arrow Field metadata across C Data exports
1 parent 98cd8c9 commit a0c8f89

3 files changed

Lines changed: 54 additions & 10 deletions

File tree

‎native/core/src/execution/jni_api.rs‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -761,6 +761,7 @@ fn prepare_output(
761761
let schema_addrs = unsafe { schema_addrs.get_elements(env, ReleaseMode::NoCopyBack)? };
762762
let schema_addrs = &*schema_addrs;
763763

764+
let output_schema = output_batch.schema();
764765
let results = output_batch.columns();
765766
let num_rows = output_batch.num_rows();
766767

@@ -787,6 +788,7 @@ fn prepare_output(
787788
let mut i = 0;
788789
while i < results.len() {
789790
let array_ref = results.get(i).ok_or(CometError::IndexOutOfBounds(i))?;
791+
let field = output_schema.field(i);
790792

791793
if array_ref.offset() != 0 {
792794
// https://github.com/apache/datafusion-comet/issues/2051
@@ -803,11 +805,11 @@ fn prepare_output(
803805

804806
new_array
805807
.to_data()
806-
.move_to_spark(array_addrs[i], schema_addrs[i])?;
808+
.move_to_spark(field, array_addrs[i], schema_addrs[i])?;
807809
} else {
808810
array_ref
809811
.to_data()
810-
.move_to_spark(array_addrs[i], schema_addrs[i])?;
812+
.move_to_spark(field, array_addrs[i], schema_addrs[i])?;
811813
}
812814
i += 1;
813815
}

‎native/core/src/execution/utils.rs‎

Lines changed: 46 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -19,28 +19,45 @@
1919
use crate::execution::operators::ExecutionError;
2020
use arrow::{
2121
array::ArrayData,
22+
datatypes::Field,
23+
error::ArrowError,
2224
ffi::{FFI_ArrowArray, FFI_ArrowSchema},
2325
};
2426

27+
fn ffi_schema_for_field(field: &Field) -> Result<FFI_ArrowSchema, ArrowError> {
28+
if field.name().contains('\0') {
29+
// ArrowSchema names are NUL-terminated C strings. Spark owns the logical output name, so
30+
// substitute only the exported name while retaining the Field's type and metadata.
31+
let field = field
32+
.clone()
33+
.with_name(field.name().replace('\0', "\u{fffd}"));
34+
FFI_ArrowSchema::try_from(&field)
35+
} else {
36+
FFI_ArrowSchema::try_from(field)
37+
}
38+
}
39+
2540
pub trait SparkArrowConvert {
2641
/// Move Arrow Arrays to C data interface.
27-
fn move_to_spark(&self, array: i64, schema: i64) -> Result<(), ExecutionError>;
42+
fn move_to_spark(&self, field: &Field, array: i64, schema: i64) -> Result<(), ExecutionError>;
2843
}
2944

3045
impl SparkArrowConvert for ArrayData {
3146
/// Move this ArrowData to pointers of Arrow C data interface.
32-
fn move_to_spark(&self, array: i64, schema: i64) -> Result<(), ExecutionError> {
47+
fn move_to_spark(&self, field: &Field, array: i64, schema: i64) -> Result<(), ExecutionError> {
3348
let array_ptr = array as *mut FFI_ArrowArray;
3449
let schema_ptr = schema as *mut FFI_ArrowSchema;
3550

3651
let array_align = std::mem::align_of::<FFI_ArrowArray>();
3752
let schema_align = std::mem::align_of::<FFI_ArrowSchema>();
53+
let ffi_array = FFI_ArrowArray::new(self);
54+
let ffi_schema = ffi_schema_for_field(field)?;
3855

3956
// Check if the pointer alignment is correct.
4057
if array_ptr.align_offset(array_align) != 0 || schema_ptr.align_offset(schema_align) != 0 {
4158
unsafe {
42-
std::ptr::write_unaligned(array_ptr, FFI_ArrowArray::new(self));
43-
std::ptr::write_unaligned(schema_ptr, FFI_ArrowSchema::try_from(self.data_type())?);
59+
std::ptr::write_unaligned(array_ptr, ffi_array);
60+
std::ptr::write_unaligned(schema_ptr, ffi_schema);
4461
}
4562
} else {
4663
// SAFETY: `array_ptr` and `schema_ptr` are aligned correctly.
@@ -55,8 +72,8 @@ impl SparkArrowConvert for ArrayData {
5572
"move_to_spark: schema_ptr not aligned"
5673
);
5774
unsafe {
58-
std::ptr::write(array_ptr, FFI_ArrowArray::new(self));
59-
std::ptr::write(schema_ptr, FFI_ArrowSchema::try_from(self.data_type())?);
75+
std::ptr::write(array_ptr, ffi_array);
76+
std::ptr::write(schema_ptr, ffi_schema);
6077
}
6178
}
6279

@@ -65,3 +82,26 @@ impl SparkArrowConvert for ArrayData {
6582
}
6683

6784
pub use datafusion_comet_common::bytes_to_i128;
85+
86+
#[cfg(test)]
87+
mod tests {
88+
use super::*;
89+
use arrow::datatypes::DataType;
90+
use std::collections::HashMap;
91+
92+
#[test]
93+
fn test_ffi_schema_preserves_field_and_sanitizes_nul_name() {
94+
let field = Field::new("v\0tail", DataType::Int32, true).with_metadata(HashMap::from([(
95+
"ARROW:extension:name".to_string(),
96+
"example.logical-type".to_string(),
97+
)]));
98+
99+
let ffi_schema = ffi_schema_for_field(&field).unwrap();
100+
let exported = Field::try_from(&ffi_schema).unwrap();
101+
102+
assert_eq!(exported.name(), "v\u{fffd}tail");
103+
assert_eq!(exported.data_type(), field.data_type());
104+
assert_eq!(exported.is_nullable(), field.is_nullable());
105+
assert_eq!(exported.metadata(), field.metadata());
106+
}
107+
}

‎native/core/src/parquet/mod.rs‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -308,8 +308,10 @@ pub extern "system" fn Java_org_apache_comet_parquet_Native_currentColumnBatch(
308308
.ok_or_else(|| CometError::Execution {
309309
source: ExecutionError::GeneralError("There is no more data to read".to_string()),
310310
});
311-
let data = batch_reader?.column(column_idx as usize).into_data();
312-
data.move_to_spark(array_addr, schema_addr)
311+
let batch = batch_reader?;
312+
let field = batch.schema().field(column_idx as usize).clone();
313+
let data = batch.column(column_idx as usize).into_data();
314+
data.move_to_spark(&field, array_addr, schema_addr)
313315
.map_err(|e| e.into())
314316
})
315317
}

0 commit comments

Comments
 (0)