Skip to content

Commit 784c316

Browse files
committed
review
1 parent 45a0ed4 commit 784c316

12 files changed

Lines changed: 1067 additions & 98 deletions

File tree

‎native/core/src/execution/planner.rs‎

Lines changed: 136 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -115,7 +115,7 @@ use crate::parquet::parquet_exec::init_datasource_exec;
115115
use arrow::array::{
116116
new_empty_array, Array, ArrayRef, BinaryBuilder, BooleanArray, Date32Array, Decimal128Array,
117117
Float32Array, Float64Array, Int16Array, Int32Array, Int64Array, Int8Array, ListArray,
118-
NullArray, StringBuilder, TimestampMicrosecondArray,
118+
NullArray, RecordBatch, StringBuilder, TimestampMicrosecondArray,
119119
};
120120
use arrow::buffer::{BooleanBuffer, NullBuffer, OffsetBuffer};
121121
use arrow::row::{OwnedRow, RowConverter, SortField};
@@ -923,6 +923,25 @@ impl PhysicalPlanner {
923923
}
924924
}
925925

926+
/// Decode a Spark existence default into the scalar consumed by the Parquet schema adapter.
927+
/// Ordinary defaults are literals. A Variant default is transported as a constant
928+
/// `CreateNamedStruct(value, metadata)` because Variant has Struct storage in Arrow.
929+
fn create_default_value(
930+
&self,
931+
spark_expr: &Expr,
932+
input_schema: SchemaRef,
933+
) -> Result<ScalarValue, ExecutionError> {
934+
let expr = self.create_expr(spark_expr, Arc::clone(&input_schema))?;
935+
if let Some(literal) = expr.downcast_ref::<DataFusionLiteral>() {
936+
return Ok(literal.value().clone());
937+
}
938+
939+
let array = expr
940+
.evaluate(&RecordBatch::new_empty(input_schema))?
941+
.into_array_of_size(1)?;
942+
Ok(ScalarValue::try_from_array(array.as_ref(), 0)?)
943+
}
944+
926945
/// Create a DataFusion physical sort expression from Spark physical expression
927946
fn create_sort_expr<'a>(
928947
&'a self,
@@ -1580,44 +1599,38 @@ impl PhysicalPlanner {
15801599
.collect()
15811600
};
15821601

1583-
let default_values: Option<HashMap<Column, ScalarValue>> = if !common
1584-
.default_values
1585-
.is_empty()
1586-
{
1587-
// We have default values. Extract the two lists (same length) of values and
1588-
// indexes in the schema, and then create a HashMap to use in the SchemaMapper.
1589-
let default_values: Result<Vec<ScalarValue>, DataFusionError> = common
1590-
.default_values
1591-
.iter()
1592-
.map(|expr| {
1593-
let literal = self.create_expr(expr, Arc::clone(&required_schema))?;
1594-
let df_literal =
1595-
literal.downcast_ref::<DataFusionLiteral>().ok_or_else(|| {
1596-
GeneralError("Expected literal of default value.".to_string())
1602+
if common.default_values.len() != common.default_values_indexes.len() {
1603+
return Err(GeneralError(format!(
1604+
"NativeScan has {} default values but {} default indexes",
1605+
common.default_values.len(),
1606+
common.default_values_indexes.len()
1607+
)));
1608+
}
1609+
let default_values: Option<HashMap<Column, ScalarValue>> =
1610+
if common.default_values.is_empty() {
1611+
None
1612+
} else {
1613+
let defaults = common
1614+
.default_values
1615+
.iter()
1616+
.zip(&common.default_values_indexes)
1617+
.map(|(expr, offset)| {
1618+
let idx = usize::try_from(*offset).map_err(|_| {
1619+
GeneralError(format!("Invalid default value index {offset}"))
15971620
})?;
1598-
Ok(df_literal.value().clone())
1599-
})
1600-
.collect();
1601-
let default_values = default_values?;
1602-
let default_values_indexes: Vec<usize> = common
1603-
.default_values_indexes
1604-
.iter()
1605-
.map(|offset| *offset as usize)
1606-
.collect();
1607-
Some(
1608-
default_values_indexes
1609-
.into_iter()
1610-
.zip(default_values)
1611-
.map(|(idx, scalar_value)| {
1612-
let field = required_schema.field(idx);
1613-
let column = Column::new(field.name().as_str(), idx);
1614-
(column, scalar_value)
1621+
let field = required_schema.fields().get(idx).ok_or_else(|| {
1622+
GeneralError(format!(
1623+
"Default value index {idx} is outside schema with {} fields",
1624+
required_schema.fields().len()
1625+
))
1626+
})?;
1627+
let value =
1628+
self.create_default_value(expr, Arc::clone(&required_schema))?;
1629+
Ok((Column::new(field.name(), idx), value))
16151630
})
1616-
.collect(),
1617-
)
1618-
} else {
1619-
None
1620-
};
1631+
.collect::<Result<HashMap<_, _>, ExecutionError>>()?;
1632+
Some(defaults)
1633+
};
16211634

16221635
// Get one file from this partition (we know it's not empty due to early return above)
16231636
let one_file = partition_files
@@ -4600,7 +4613,8 @@ mod tests {
46004613
use std::{sync::Arc, task::Poll};
46014614

46024615
use arrow::array::{
4603-
Array, DictionaryArray, Int32Array, Int8Array, ListArray, RecordBatch, StringArray,
4616+
Array, BinaryArray, DictionaryArray, Int32Array, Int8Array, ListArray, RecordBatch,
4617+
StringArray,
46044618
};
46054619
use arrow::datatypes::{DataType, Field, FieldRef, Fields, Schema};
46064620
use datafusion::catalog::memory::DataSourceExec;
@@ -4613,7 +4627,10 @@ mod tests {
46134627
use datafusion::error::DataFusionError;
46144628
use datafusion::logical_expr::ScalarUDF;
46154629
use datafusion::physical_plan::ExecutionPlan;
4616-
use datafusion::{assert_batches_eq, physical_plan::common::collect, prelude::SessionContext};
4630+
use datafusion::{
4631+
assert_batches_eq, physical_plan::common::collect, prelude::SessionContext,
4632+
scalar::ScalarValue,
4633+
};
46174634
use datafusion_physical_expr_adapter::PhysicalExprAdapterFactory;
46184635
use tempfile::TempDir;
46194636
use tokio::sync::mpsc;
@@ -4623,6 +4640,7 @@ mod tests {
46234640
use crate::execution::operators::ExecutionError;
46244641
use crate::execution::planner::literal_to_array_ref;
46254642
use crate::execution::planner::parse_file_scan_tasks_from_common;
4643+
use crate::execution::serde::to_arrow_field;
46264644
use crate::parquet::parquet_support::SparkParquetOptions;
46274645
use crate::parquet::schema_adapter::SparkPhysicalExprAdapterFactory;
46284646
use datafusion_comet_proto::spark_expression::expr::ExprStruct;
@@ -4636,6 +4654,85 @@ mod tests {
46364654
};
46374655
use datafusion_comet_spark_expr::EvalMode;
46384656

4657+
#[test]
4658+
fn test_create_default_value_from_literal_and_variant_struct() {
4659+
fn literal(value: literal::Value, type_id: i32) -> Expr {
4660+
Expr {
4661+
expr_struct: Some(ExprStruct::Literal(spark_expression::Literal {
4662+
value: Some(value),
4663+
datatype: Some(spark_expression::DataType {
4664+
type_id,
4665+
type_info: None,
4666+
}),
4667+
is_null: false,
4668+
})),
4669+
query_context: None,
4670+
expr_id: None,
4671+
}
4672+
}
4673+
4674+
let planner = PhysicalPlanner::default();
4675+
let int_field = Field::new("n", DataType::Int32, true);
4676+
let int_default = literal(literal::Value::IntVal(7), 3);
4677+
assert_eq!(
4678+
planner
4679+
.create_default_value(&int_default, Arc::new(Schema::new(vec![int_field.clone()])),)
4680+
.unwrap(),
4681+
ScalarValue::Int32(Some(7))
4682+
);
4683+
4684+
let variant_field = to_arrow_field(
4685+
"v",
4686+
&spark_expression::DataType {
4687+
type_id: 21,
4688+
type_info: None,
4689+
},
4690+
true,
4691+
);
4692+
let variant_default = Expr {
4693+
expr_struct: Some(ExprStruct::CreateNamedStruct(
4694+
spark_expression::CreateNamedStruct {
4695+
values: vec![
4696+
literal(literal::Value::BytesVal(vec![1, 2]), 8),
4697+
literal(literal::Value::BytesVal(vec![3, 4]), 8),
4698+
],
4699+
names: vec!["value".to_string(), "metadata".to_string()],
4700+
},
4701+
)),
4702+
query_context: None,
4703+
expr_id: None,
4704+
};
4705+
let value = planner
4706+
.create_default_value(
4707+
&variant_default,
4708+
Arc::new(Schema::new(vec![variant_field.clone()])),
4709+
)
4710+
.unwrap();
4711+
let ScalarValue::Struct(value) = value else {
4712+
panic!("expected Variant default to use Struct storage")
4713+
};
4714+
assert_eq!(value.fields()[0].name(), "value");
4715+
assert_eq!(value.fields()[1].name(), "metadata");
4716+
assert_eq!(
4717+
value
4718+
.column(0)
4719+
.as_any()
4720+
.downcast_ref::<BinaryArray>()
4721+
.unwrap()
4722+
.value(0),
4723+
&[1, 2]
4724+
);
4725+
assert_eq!(
4726+
value
4727+
.column(1)
4728+
.as_any()
4729+
.downcast_ref::<BinaryArray>()
4730+
.unwrap()
4731+
.value(0),
4732+
&[3, 4]
4733+
);
4734+
}
4735+
46394736
#[test]
46404737
fn test_unpack_dictionary_primitive() {
46414738
let op_scan = Operator {

‎native/core/src/execution/utils.rs‎

Lines changed: 46 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -20,9 +20,26 @@ use crate::execution::operators::ExecutionError;
2020
use arrow::{
2121
array::ArrayData,
2222
datatypes::Field,
23+
error::ArrowError,
2324
ffi::{FFI_ArrowArray, FFI_ArrowSchema},
2425
};
2526

27+
fn ffi_schema_for_field(field: &Field) -> Result<FFI_ArrowSchema, ArrowError> {
28+
if field.name().contains('\0') {
29+
// Spark keeps Parquet field names as strings, while ArrowSchema exports names as C strings:
30+
// https://github.com/apache/spark/blob/v4.1.3/sql/api/src/main/scala/org/apache/spark/sql/types/StructField.scala#L32-L51
31+
// https://github.com/apache/spark/blob/v4.1.3/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSchemaConverter.scala#L576-L647
32+
// https://github.com/apache/arrow-rs/blob/58.4.0/arrow-schema/src/ffi.rs#L168-L175
33+
// The logical output name is owned by Spark's plan, so substitute only at this boundary.
34+
let field = field
35+
.clone()
36+
.with_name(field.name().replace('\0', "\u{fffd}"));
37+
FFI_ArrowSchema::try_from(&field)
38+
} else {
39+
FFI_ArrowSchema::try_from(field)
40+
}
41+
}
42+
2643
pub trait SparkArrowConvert {
2744
/// Move Arrow Arrays to C data interface.
2845
fn move_to_spark(&self, field: &Field, array: i64, schema: i64) -> Result<(), ExecutionError>;
@@ -36,12 +53,14 @@ impl SparkArrowConvert for ArrayData {
3653

3754
let array_align = std::mem::align_of::<FFI_ArrowArray>();
3855
let schema_align = std::mem::align_of::<FFI_ArrowSchema>();
56+
let ffi_array = FFI_ArrowArray::new(self);
57+
let ffi_schema = ffi_schema_for_field(field)?;
3958

4059
// Check if the pointer alignment is correct.
4160
if array_ptr.align_offset(array_align) != 0 || schema_ptr.align_offset(schema_align) != 0 {
4261
unsafe {
43-
std::ptr::write_unaligned(array_ptr, FFI_ArrowArray::new(self));
44-
std::ptr::write_unaligned(schema_ptr, FFI_ArrowSchema::try_from(field)?);
62+
std::ptr::write_unaligned(array_ptr, ffi_array);
63+
std::ptr::write_unaligned(schema_ptr, ffi_schema);
4564
}
4665
} else {
4766
// SAFETY: `array_ptr` and `schema_ptr` are aligned correctly.
@@ -56,8 +75,8 @@ impl SparkArrowConvert for ArrayData {
5675
"move_to_spark: schema_ptr not aligned"
5776
);
5877
unsafe {
59-
std::ptr::write(array_ptr, FFI_ArrowArray::new(self));
60-
std::ptr::write(schema_ptr, FFI_ArrowSchema::try_from(field)?);
78+
std::ptr::write(array_ptr, ffi_array);
79+
std::ptr::write(schema_ptr, ffi_schema);
6180
}
6281
}
6382

@@ -66,3 +85,26 @@ impl SparkArrowConvert for ArrayData {
6685
}
6786

6887
pub use datafusion_comet_common::bytes_to_i128;
88+
89+
#[cfg(test)]
90+
mod tests {
91+
use super::*;
92+
use arrow::datatypes::DataType;
93+
use std::collections::HashMap;
94+
95+
#[test]
96+
fn test_ffi_schema_sanitizes_nul_name_and_preserves_metadata() {
97+
let field = Field::new("v\0tail", DataType::Int32, true).with_metadata(HashMap::from([(
98+
"ARROW:extension:name".to_string(),
99+
"arrow.parquet.variant".to_string(),
100+
)]));
101+
102+
let ffi_schema = ffi_schema_for_field(&field).unwrap();
103+
let exported = Field::try_from(&ffi_schema).unwrap();
104+
105+
assert_eq!(exported.name(), "v\u{fffd}tail");
106+
assert_eq!(exported.data_type(), field.data_type());
107+
assert_eq!(exported.is_nullable(), field.is_nullable());
108+
assert_eq!(exported.metadata(), field.metadata());
109+
}
110+
}

0 commit comments

Comments
 (0)