11// SPDX-License-Identifier: Apache-2.0
22// SPDX-FileCopyrightText: Copyright the Vortex contributors
33
4+ //! Converts DataFusion physical expressions for native Vortex evaluation.
5+ //!
6+ //! Unsupported expressions stay in DataFusion. Comparison and membership literals retain their own
7+ //! extension metadata, or inherit missing metadata from the operand they are compared with. This
8+ //! supplies the logical dtype that Vortex requires without adding implicit casts.
9+
410use std:: sync:: Arc ;
511
612use arrow_schema:: DataType ;
713use arrow_schema:: Field ;
814use arrow_schema:: Schema ;
15+ use arrow_schema:: extension:: EXTENSION_TYPE_METADATA_KEY ;
16+ use arrow_schema:: extension:: EXTENSION_TYPE_NAME_KEY ;
917use datafusion_common:: DataFusionError ;
1018use datafusion_common:: Result as DFResult ;
1119use datafusion_common:: ScalarValue ;
@@ -39,18 +47,15 @@ use vortex::expr::nested_case_when;
3947use vortex:: expr:: not;
4048use vortex:: expr:: pack;
4149use vortex:: expr:: root;
50+ use vortex:: scalar:: Scalar ;
4251use vortex:: scalar_fn:: ScalarFnVTableExt ;
4352use vortex:: scalar_fn:: fns:: binary:: Binary ;
4453use vortex:: scalar_fn:: fns:: like:: Like ;
4554use vortex:: scalar_fn:: fns:: like:: LikeOptions ;
46- use vortex:: scalar_fn:: fns:: literal:: Literal ;
4755use vortex:: scalar_fn:: fns:: operators:: Operator ;
4856use vortex:: session:: VortexSession ;
4957use vortex_arrow:: ArrowSessionExt ;
5058
51- #[ cfg( test) ]
52- mod tests;
53-
5459/// Result of splitting a projection into Vortex expressions and leftover DataFusion projections.
5560pub struct ProcessedProjection {
5661 /// Projection evaluated by the Vortex scan.
@@ -309,8 +314,21 @@ impl DefaultExpressionConverter {
309314 } else if !supported_data_types ( & left_type) || !supported_data_types ( & right_type) {
310315 return Err ( Unconverted :: Unsupported ) ;
311316 }
312- let left = self . convert_expr ( binary_expr. left ( ) , schema, input_dtype) ?;
313- let right = self . convert_expr ( binary_expr. right ( ) , schema, input_dtype) ?;
317+ let left = self . convert_binary_operand (
318+ binary_expr. left ( ) ,
319+ binary_expr. right ( ) ,
320+ operator,
321+ schema,
322+ input_dtype,
323+ ) ?;
324+ let right = self . convert_binary_operand (
325+ binary_expr. right ( ) ,
326+ binary_expr. left ( ) ,
327+ operator,
328+ schema,
329+ input_dtype,
330+ ) ?;
331+
314332 // DataFusion may evaluate the RHS only on rows selected by the LHS.
315333 if boolean_operator && !( is_infallible ( & left) && is_infallible ( & right) ) {
316334 return Err ( Unconverted :: Unsupported ) ;
@@ -329,23 +347,8 @@ impl DefaultExpressionConverter {
329347
330348 if let Some ( literal) = expr. downcast_ref :: < df_expr:: Literal > ( ) {
331349 let field = literal. return_field ( schema) ?;
332- let array = literal. value ( ) . to_array ( ) ?;
333- if array. len ( ) != 1 {
334- return Err ( exec_datafusion_err ! (
335- "Literal must contain exactly one value, found {}" ,
336- array. len( )
337- )
338- . into ( ) ) ;
339- }
340- // Literals of unknown Arrow types stay in DataFusion; conversion failures are errors.
341- self . arrow_dtype ( & field) ?;
342- let scalar = self
343- . session
344- . arrow ( )
345- . from_arrow_array ( array, & field)
346- . and_then ( |array| array. execute_scalar ( 0 , & mut self . session . create_execution_ctx ( ) ) )
347- . map_err ( |e| exec_datafusion_err ! ( "Failed to convert literal {expr}: {e}" ) ) ?;
348- return Ok ( lit ( scalar) ) ;
350+
351+ return Ok ( lit ( self . import_literal ( literal, & field) ?) ) ;
349352 }
350353
351354 if let Some ( cast_expr) = expr. downcast_ref :: < df_expr:: CastExpr > ( ) {
@@ -427,6 +430,82 @@ impl DefaultExpressionConverter {
427430 Err ( Unconverted :: Unsupported )
428431 }
429432
433+ /// Converts a binary operand, using the other operand's field to type comparison literals.
434+ ///
435+ /// Arithmetic and Boolean operands convert independently. They do not accept extension dtypes,
436+ /// so the other operand cannot supply an expected extension type for their literals.
437+ fn convert_binary_operand (
438+ & self ,
439+ expr : & Arc < dyn PhysicalExpr > ,
440+ other : & Arc < dyn PhysicalExpr > ,
441+ operator : Operator ,
442+ schema : & Schema ,
443+ input_dtype : & DType ,
444+ ) -> Conversion < Expression > {
445+ if operator. is_comparison ( )
446+ && let Some ( literal) = expr. downcast_ref :: < df_expr:: Literal > ( )
447+ {
448+ let field = other. return_field ( schema) ?;
449+ let scalar = self . import_literal_for_operand ( literal, & field, schema) ?;
450+
451+ return Ok ( lit ( scalar) ) ;
452+ }
453+
454+ self . convert_expr ( expr, schema, input_dtype)
455+ }
456+
457+ /// Imports a literal, using `operand_field` only when the literal has no extension metadata.
458+ ///
459+ /// The literal keeps its Arrow dtype, so different storage types or timestamp units still fail
460+ /// the comparison's dtype check. Explicit metadata, including an unknown extension, is preserved.
461+ /// The inherited extension name and payload both come from `operand_field`, including an absent
462+ /// payload, so the import cannot combine metadata from two different types.
463+ fn import_literal_for_operand (
464+ & self ,
465+ literal : & df_expr:: Literal ,
466+ operand_field : & Field ,
467+ schema : & Schema ,
468+ ) -> Conversion < Scalar > {
469+ let field = literal. return_field ( schema) ?;
470+ if field. extension_type_name ( ) . is_some ( ) || operand_field. extension_type_name ( ) . is_none ( ) {
471+ return self . import_literal ( literal, & field) ;
472+ }
473+
474+ let mut field = field. as_ref ( ) . clone ( ) ;
475+ for key in [ EXTENSION_TYPE_NAME_KEY , EXTENSION_TYPE_METADATA_KEY ] {
476+ field. metadata_mut ( ) . remove ( key) ;
477+ if let Some ( value) = operand_field. metadata ( ) . get ( key) {
478+ field. metadata_mut ( ) . insert ( key. into ( ) , value. clone ( ) ) ;
479+ }
480+ }
481+
482+ self . import_literal ( literal, & field)
483+ }
484+
485+ /// Imports one scalar through the session's Arrow plugins, preserving `field`'s metadata.
486+ fn import_literal ( & self , literal : & df_expr:: Literal , field : & Field ) -> Conversion < Scalar > {
487+ let array = literal. value ( ) . to_array ( ) ?;
488+ if array. len ( ) != 1 {
489+ return Err ( exec_datafusion_err ! (
490+ "Literal must contain exactly one value, got {}" ,
491+ array. len( )
492+ )
493+ . into ( ) ) ;
494+ }
495+
496+ self . arrow_dtype ( field) ?;
497+
498+ self . session
499+ . arrow ( )
500+ . from_arrow_array ( array, field)
501+ . and_then ( |array| array. execute_scalar ( 0 , & mut self . session . create_execution_ctx ( ) ) )
502+ . map_err ( |error| {
503+ Unconverted :: Failed ( DataFusionError :: External ( Box :: new (
504+ error. with_context ( format ! ( "Failed to convert literal {literal}" ) ) ,
505+ ) ) )
506+ } )
507+ }
508+
430509 fn convert_in_list (
431510 & self ,
432511 in_list : & df_expr:: InListExpr ,
@@ -448,23 +527,20 @@ impl DefaultExpressionConverter {
448527 return Err ( Unconverted :: Unsupported ) ;
449528 }
450529 let value_dtype = converted_dtype ( & value, input_dtype) ?;
530+ let value_field = in_list. expr ( ) . return_field ( schema) ?;
451531
452532 // The list of a hash join filter holds every build-side key, which repeat. Each element
453- // converts through Arrow, so equal elements are converted once.
533+ // converts through Arrow, so equal values with equal metadata are converted once.
454534 let elements = in_list
455535 . list ( )
456536 . iter ( )
457- . unique_by ( |element| {
458- element
459- . downcast_ref :: < df_expr:: Literal > ( )
460- . map ( |literal| literal. value ( ) . clone ( ) )
461- } )
537+ . unique_by ( |element| element. downcast_ref :: < df_expr:: Literal > ( ) . cloned ( ) )
462538 . map ( |element| {
463- let element = self . convert_expr ( element, schema , input_dtype ) ? ;
464- element
465- . as_opt :: < Literal > ( )
466- . cloned ( )
467- . ok_or ( Unconverted :: Unsupported )
539+ let literal = element
540+ . downcast_ref :: < df_expr :: Literal > ( )
541+ . ok_or ( Unconverted :: Unsupported ) ? ;
542+
543+ self . import_literal_for_operand ( literal , & value_field , schema )
468544 } )
469545 . collect :: < Conversion < Vec < _ > > > ( ) ?;
470546
@@ -747,3 +823,6 @@ fn array_length_input(scalar_fn: &ScalarFunctionExpr) -> DFResult<Option<&Arc<dy
747823 ) ) ,
748824 }
749825}
826+
827+ #[ cfg( test) ]
828+ mod tests;
0 commit comments