@@ -115,7 +115,7 @@ use crate::parquet::parquet_exec::init_datasource_exec;
115115use arrow:: array:: {
116116 new_empty_array, Array , ArrayRef , BinaryBuilder , BooleanArray , Date32Array , Decimal128Array ,
117117 Float32Array , Float64Array , Int16Array , Int32Array , Int64Array , Int8Array , ListArray ,
118- NullArray , StringBuilder , TimestampMicrosecondArray ,
118+ NullArray , RecordBatch , StringBuilder , TimestampMicrosecondArray ,
119119} ;
120120use arrow:: buffer:: { BooleanBuffer , NullBuffer , OffsetBuffer } ;
121121use arrow:: row:: { OwnedRow , RowConverter , SortField } ;
@@ -923,6 +923,25 @@ impl PhysicalPlanner {
923923 }
924924 }
925925
926+ /// Decode a Spark existence default into the scalar consumed by the Parquet schema adapter.
927+ /// Ordinary defaults are literals. A Variant default is transported as a constant
928+ /// `CreateNamedStruct(value, metadata)` because Variant has Struct storage in Arrow.
929+ fn create_default_value (
930+ & self ,
931+ spark_expr : & Expr ,
932+ input_schema : SchemaRef ,
933+ ) -> Result < ScalarValue , ExecutionError > {
934+ let expr = self . create_expr ( spark_expr, Arc :: clone ( & input_schema) ) ?;
935+ if let Some ( literal) = expr. downcast_ref :: < DataFusionLiteral > ( ) {
936+ return Ok ( literal. value ( ) . clone ( ) ) ;
937+ }
938+
939+ let array = expr
940+ . evaluate ( & RecordBatch :: new_empty ( input_schema) ) ?
941+ . into_array_of_size ( 1 ) ?;
942+ Ok ( ScalarValue :: try_from_array ( array. as_ref ( ) , 0 ) ?)
943+ }
944+
926945 /// Create a DataFusion physical sort expression from Spark physical expression
927946 fn create_sort_expr < ' a > (
928947 & ' a self ,
@@ -1580,44 +1599,38 @@ impl PhysicalPlanner {
15801599 . collect ( )
15811600 } ;
15821601
1583- let default_values: Option < HashMap < Column , ScalarValue > > = if !common
1584- . default_values
1585- . is_empty ( )
1586- {
1587- // We have default values. Extract the two lists (same length) of values and
1588- // indexes in the schema, and then create a HashMap to use in the SchemaMapper.
1589- let default_values: Result < Vec < ScalarValue > , DataFusionError > = common
1590- . default_values
1591- . iter ( )
1592- . map ( |expr| {
1593- let literal = self . create_expr ( expr, Arc :: clone ( & required_schema) ) ?;
1594- let df_literal =
1595- literal. downcast_ref :: < DataFusionLiteral > ( ) . ok_or_else ( || {
1596- GeneralError ( "Expected literal of default value." . to_string ( ) )
1602+ if common. default_values . len ( ) != common. default_values_indexes . len ( ) {
1603+ return Err ( GeneralError ( format ! (
1604+ "NativeScan has {} default values but {} default indexes" ,
1605+ common. default_values. len( ) ,
1606+ common. default_values_indexes. len( )
1607+ ) ) ) ;
1608+ }
1609+ let default_values: Option < HashMap < Column , ScalarValue > > =
1610+ if common. default_values . is_empty ( ) {
1611+ None
1612+ } else {
1613+ let defaults = common
1614+ . default_values
1615+ . iter ( )
1616+ . zip ( & common. default_values_indexes )
1617+ . map ( |( expr, offset) | {
1618+ let idx = usize:: try_from ( * offset) . map_err ( |_| {
1619+ GeneralError ( format ! ( "Invalid default value index {offset}" ) )
15971620 } ) ?;
1598- Ok ( df_literal. value ( ) . clone ( ) )
1599- } )
1600- . collect ( ) ;
1601- let default_values = default_values?;
1602- let default_values_indexes: Vec < usize > = common
1603- . default_values_indexes
1604- . iter ( )
1605- . map ( |offset| * offset as usize )
1606- . collect ( ) ;
1607- Some (
1608- default_values_indexes
1609- . into_iter ( )
1610- . zip ( default_values)
1611- . map ( |( idx, scalar_value) | {
1612- let field = required_schema. field ( idx) ;
1613- let column = Column :: new ( field. name ( ) . as_str ( ) , idx) ;
1614- ( column, scalar_value)
1621+ let field = required_schema. fields ( ) . get ( idx) . ok_or_else ( || {
1622+ GeneralError ( format ! (
1623+ "Default value index {idx} is outside schema with {} fields" ,
1624+ required_schema. fields( ) . len( )
1625+ ) )
1626+ } ) ?;
1627+ let value =
1628+ self . create_default_value ( expr, Arc :: clone ( & required_schema) ) ?;
1629+ Ok ( ( Column :: new ( field. name ( ) , idx) , value) )
16151630 } )
1616- . collect ( ) ,
1617- )
1618- } else {
1619- None
1620- } ;
1631+ . collect :: < Result < HashMap < _ , _ > , ExecutionError > > ( ) ?;
1632+ Some ( defaults)
1633+ } ;
16211634
16221635 // Get one file from this partition (we know it's not empty due to early return above)
16231636 let one_file = partition_files
@@ -4600,7 +4613,8 @@ mod tests {
46004613 use std:: { sync:: Arc , task:: Poll } ;
46014614
46024615 use arrow:: array:: {
4603- Array , DictionaryArray , Int32Array , Int8Array , ListArray , RecordBatch , StringArray ,
4616+ Array , BinaryArray , DictionaryArray , Int32Array , Int8Array , ListArray , RecordBatch ,
4617+ StringArray ,
46044618 } ;
46054619 use arrow:: datatypes:: { DataType , Field , FieldRef , Fields , Schema } ;
46064620 use datafusion:: catalog:: memory:: DataSourceExec ;
@@ -4613,7 +4627,10 @@ mod tests {
46134627 use datafusion:: error:: DataFusionError ;
46144628 use datafusion:: logical_expr:: ScalarUDF ;
46154629 use datafusion:: physical_plan:: ExecutionPlan ;
4616- use datafusion:: { assert_batches_eq, physical_plan:: common:: collect, prelude:: SessionContext } ;
4630+ use datafusion:: {
4631+ assert_batches_eq, physical_plan:: common:: collect, prelude:: SessionContext ,
4632+ scalar:: ScalarValue ,
4633+ } ;
46174634 use datafusion_physical_expr_adapter:: PhysicalExprAdapterFactory ;
46184635 use tempfile:: TempDir ;
46194636 use tokio:: sync:: mpsc;
@@ -4623,6 +4640,7 @@ mod tests {
46234640 use crate :: execution:: operators:: ExecutionError ;
46244641 use crate :: execution:: planner:: literal_to_array_ref;
46254642 use crate :: execution:: planner:: parse_file_scan_tasks_from_common;
4643+ use crate :: execution:: serde:: to_arrow_field;
46264644 use crate :: parquet:: parquet_support:: SparkParquetOptions ;
46274645 use crate :: parquet:: schema_adapter:: SparkPhysicalExprAdapterFactory ;
46284646 use datafusion_comet_proto:: spark_expression:: expr:: ExprStruct ;
@@ -4636,6 +4654,85 @@ mod tests {
46364654 } ;
46374655 use datafusion_comet_spark_expr:: EvalMode ;
46384656
4657+ #[ test]
4658+ fn test_create_default_value_from_literal_and_variant_struct ( ) {
4659+ fn literal ( value : literal:: Value , type_id : i32 ) -> Expr {
4660+ Expr {
4661+ expr_struct : Some ( ExprStruct :: Literal ( spark_expression:: Literal {
4662+ value : Some ( value) ,
4663+ datatype : Some ( spark_expression:: DataType {
4664+ type_id,
4665+ type_info : None ,
4666+ } ) ,
4667+ is_null : false ,
4668+ } ) ) ,
4669+ query_context : None ,
4670+ expr_id : None ,
4671+ }
4672+ }
4673+
4674+ let planner = PhysicalPlanner :: default ( ) ;
4675+ let int_field = Field :: new ( "n" , DataType :: Int32 , true ) ;
4676+ let int_default = literal ( literal:: Value :: IntVal ( 7 ) , 3 ) ;
4677+ assert_eq ! (
4678+ planner
4679+ . create_default_value( & int_default, Arc :: new( Schema :: new( vec![ int_field. clone( ) ] ) ) , )
4680+ . unwrap( ) ,
4681+ ScalarValue :: Int32 ( Some ( 7 ) )
4682+ ) ;
4683+
4684+ let variant_field = to_arrow_field (
4685+ "v" ,
4686+ & spark_expression:: DataType {
4687+ type_id : 21 ,
4688+ type_info : None ,
4689+ } ,
4690+ true ,
4691+ ) ;
4692+ let variant_default = Expr {
4693+ expr_struct : Some ( ExprStruct :: CreateNamedStruct (
4694+ spark_expression:: CreateNamedStruct {
4695+ values : vec ! [
4696+ literal( literal:: Value :: BytesVal ( vec![ 1 , 2 ] ) , 8 ) ,
4697+ literal( literal:: Value :: BytesVal ( vec![ 3 , 4 ] ) , 8 ) ,
4698+ ] ,
4699+ names : vec ! [ "value" . to_string( ) , "metadata" . to_string( ) ] ,
4700+ } ,
4701+ ) ) ,
4702+ query_context : None ,
4703+ expr_id : None ,
4704+ } ;
4705+ let value = planner
4706+ . create_default_value (
4707+ & variant_default,
4708+ Arc :: new ( Schema :: new ( vec ! [ variant_field. clone( ) ] ) ) ,
4709+ )
4710+ . unwrap ( ) ;
4711+ let ScalarValue :: Struct ( value) = value else {
4712+ panic ! ( "expected Variant default to use Struct storage" )
4713+ } ;
4714+ assert_eq ! ( value. fields( ) [ 0 ] . name( ) , "value" ) ;
4715+ assert_eq ! ( value. fields( ) [ 1 ] . name( ) , "metadata" ) ;
4716+ assert_eq ! (
4717+ value
4718+ . column( 0 )
4719+ . as_any( )
4720+ . downcast_ref:: <BinaryArray >( )
4721+ . unwrap( )
4722+ . value( 0 ) ,
4723+ & [ 1 , 2 ]
4724+ ) ;
4725+ assert_eq ! (
4726+ value
4727+ . column( 1 )
4728+ . as_any( )
4729+ . downcast_ref:: <BinaryArray >( )
4730+ . unwrap( )
4731+ . value( 0 ) ,
4732+ & [ 3 , 4 ]
4733+ ) ;
4734+ }
4735+
46394736 #[ test]
46404737 fn test_unpack_dictionary_primitive ( ) {
46414738 let op_scan = Operator {
0 commit comments