@@ -429,23 +429,6 @@ fn is_string_or_binary(dt: &DataType) -> bool {
429429 )
430430}
431431
432- /// Whether `cast` reconciles a column's Parquet (file) type with its table-schema type, as opposed
433- /// to being part of the pushed-down expression itself.
434- ///
435- /// DataFusion's `DefaultPhysicalExprAdapter` builds the former with
436- /// `CastExpr::new_with_target_field`, passing the table schema's field, so the target is named
437- /// after the column (or nested field). Comet's planner builds the latter with `CastExpr::new`, for
438- /// example to coerce a scalar function's arguments to the types the function requires:
439- /// `substring(s, 1, 2)` casts its Int32 literals to Int64. `CastExpr::new` produces an unnamed
440- /// target field.
441- ///
442- /// Only the former are Parquet read conversions that Spark's vectorized reader validates. Applying
443- /// those rules to the latter rejects valid queries, for example as a disallowed `INT32 -> bigint`
444- /// promotion on Spark 3.x when row-level filter pushdown evaluates the predicate.
445- fn is_schema_adaptation_cast ( cast : & datafusion:: physical_expr:: expressions:: CastExpr ) -> bool {
446- !cast. target_field ( ) . name ( ) . is_empty ( )
447- }
448-
449432/// Build a Spark-shaped `SchemaColumnConvertNotSupportedException` carrier for a
450433/// rejected Parquet -> Spark conversion. `column` is the Spark-style column path (`a`, or
451434/// `s, x` for a nested leaf); the bracketed wrapping mirrors
@@ -1405,17 +1388,21 @@ impl SparkPhysicalExprAdapter {
14051388
14061389 /// Replace CastExpr (DataFusion's cast) with Spark's Cast expression.
14071390 ///
1408- /// Only casts that reconcile a file column with its table-schema type are rewritten (see
1409- /// [`is_schema_adaptation_cast`]). Any other `CastExpr` is part of the predicate itself and is
1410- /// left as the planner built it.
1391+ /// Only the casts that reconcile a file column with its table-schema type are Parquet read
1392+ /// conversions, which Spark's vectorized reader validates. DataFusion's default adapter builds
1393+ /// them with `CastExpr::new_with_target_field`, so `has_explicit_metadata()` is true, also for a
1394+ /// column named `""`. Any other `CastExpr` is part of the predicate itself and is left as the
1395+ /// planner built it. Comet's planner uses `CastExpr::new`, for example to coerce `substring`'s
1396+ /// Int32 literals to Int64, and applying the read rules to that cast rejected valid queries as
1397+ /// a disallowed `INT32 -> bigint` promotion on Spark 3.x.
14111398 fn replace_with_spark_cast (
14121399 & self ,
14131400 expr : Arc < dyn PhysicalExpr > ,
14141401 ) -> DataFusionResult < Transformed < Arc < dyn PhysicalExpr > > > {
14151402 // Check for CastExpr and replace with spark_expr::Cast
14161403 if let Some ( cast) = expr. downcast_ref :: < datafusion:: physical_expr:: expressions:: CastExpr > ( )
14171404 {
1418- if !is_schema_adaptation_cast ( cast) {
1405+ if !cast. has_explicit_metadata ( ) {
14191406 return Ok ( Transformed :: no ( expr) ) ;
14201407 }
14211408
@@ -2094,6 +2081,36 @@ pub(crate) mod test {
20942081 Ok ( err. to_string ( ) )
20952082 }
20962083
2084+ /// Spark allows a Parquet column named `""`, and reading one goes through the same conversion
2085+ /// check as any other column: an `INT64` column named `""` read as `int` is rejected, whether or
2086+ /// not type promotion is allowed, even when every value would fit.
2087+ #[ tokio:: test]
2088+ async fn empty_named_column_long_read_as_int_errors ( ) -> Result < ( ) , DataFusionError > {
2089+ let file_schema = Arc :: new ( Schema :: new ( vec ! [ Field :: new( "" , DataType :: Int64 , true ) ] ) ) ;
2090+ let batch = RecordBatch :: try_new (
2091+ Arc :: clone ( & file_schema) ,
2092+ vec ! [ Arc :: new( Int64Array :: from( vec![ 1i64 , 2 ] ) ) ] ,
2093+ ) ?;
2094+ let required_schema = Arc :: new ( Schema :: new ( vec ! [ Field :: new( "" , DataType :: Int32 , true ) ] ) ) ;
2095+ for allow_type_promotion in [ false , true ] {
2096+ let mut options = default_options ( ) ;
2097+ options. allow_type_promotion = allow_type_promotion;
2098+ let mut stream = scan_parquet ( & batch, Arc :: clone ( & required_schema) , options) ?;
2099+ let msg = match stream. next ( ) . await {
2100+ Some ( Err ( e) ) => e. to_string ( ) ,
2101+ other => panic ! (
2102+ "allow_type_promotion={allow_type_promotion}: expected ParquetSchemaConvert, \
2103+ got {other:?}"
2104+ ) ,
2105+ } ;
2106+ assert ! (
2107+ msg. contains( "Found: INT64" ) && msg. contains( "Expected: int" ) ,
2108+ "allow_type_promotion={allow_type_promotion}: {msg}"
2109+ ) ;
2110+ }
2111+ Ok ( ( ) )
2112+ }
2113+
20972114 /// `INT64 -> INT32` truncates to the lower 32 bits in DataFusion's cast.
20982115 /// Spark's vectorized reader rejects this. See #4297.
20992116 #[ tokio:: test]
@@ -2375,13 +2392,13 @@ pub(crate) mod test {
23752392 Ok ( ( ) )
23762393 }
23772394
2378- /// `is_schema_adaptation_cast` tells the casts DataFusion's default adapter inserts apart from
2379- /// the casts Comet's planner puts in a predicate by whether the target field is named. If a
2380- /// DataFusion upgrade changes how `CastExpr::new` names its target, this fails and the check
2381- /// has to be revisited (newer DataFusion versions also expose an explicit-target marker).
2395+ /// `replace_with_spark_cast` tells the casts DataFusion's default adapter inserts apart from the
2396+ /// casts Comet's planner puts in a predicate by `CastExpr::has_explicit_metadata()`. This pins
2397+ /// what that relies on: `new_with_target_field` sets it, also for a field named `""`, which
2398+ /// Spark allows as a column name. `CastExpr::new` doesn't set it, and `with_new_children` keeps
2399+ /// it.
23822400 #[ test]
2383- fn schema_adaptation_cast_is_told_apart_by_target_field_name ( ) {
2384- use super :: is_schema_adaptation_cast;
2401+ fn schema_casts_are_told_apart_by_explicit_target ( ) -> Result < ( ) , DataFusionError > {
23852402 use datafusion:: physical_expr:: expressions:: CastExpr ;
23862403
23872404 // Built by the planner, from a bare type.
@@ -2390,55 +2407,38 @@ pub(crate) mod test {
23902407 DataType :: Int64 ,
23912408 None ,
23922409 ) ;
2393- assert ! ( !is_schema_adaptation_cast ( & planner_cast) ) ;
2410+ assert ! ( !planner_cast. has_explicit_metadata ( ) ) ;
23942411
23952412 // Built by the default adapter, from the table schema's field.
2396- let adapter_cast = CastExpr :: new_with_target_field (
2397- Arc :: new ( Column :: new ( "a" , 0 ) ) ,
2398- Arc :: new ( Field :: new ( "a" , DataType :: Int64 , true ) ) ,
2399- None ,
2400- ) ;
2401- assert ! ( is_schema_adaptation_cast( & adapter_cast) ) ;
2413+ for name in [ "a" , "" ] {
2414+ let adapter_cast: Arc < dyn PhysicalExpr > = Arc :: new ( CastExpr :: new_with_target_field (
2415+ Arc :: new ( Column :: new ( name, 0 ) ) ,
2416+ Arc :: new ( Field :: new ( name, DataType :: Int64 , true ) ) ,
2417+ None ,
2418+ ) ) ;
2419+ let rebuilt = Arc :: clone ( & adapter_cast)
2420+ . with_new_children ( vec ! [ Arc :: new( Column :: new( name, 0 ) ) ] ) ?;
2421+ for cast in [ adapter_cast, rebuilt] {
2422+ let cast = cast. downcast_ref :: < CastExpr > ( ) . expect ( "a CastExpr" ) ;
2423+ assert ! ( cast. has_explicit_metadata( ) , "target field {name:?}" ) ;
2424+ }
2425+ }
2426+ Ok ( ( ) )
24022427 }
24032428
2404- /// Scans a Parquet file with a pushed-down `substring(s, 1, 2) = 'ab'` data filter, the way
2405- /// `CometNativeScan` sends it, and returns the values the scan produced.
2406- ///
2407- /// The file schema is identical to the requested schema, so there is no schema evolution of any
2408- /// kind. The predicate comes from the real planner, fed the proto `CometSubstring` emits (Int32
2409- /// literals for the position and length). `SparkSubstring` takes Int64 positions, so the planner
2410- /// coerces those literals with `CastExpr::new`.
2411- async fn scan_with_substring_filter (
2412- row_filter_pushdown : bool ,
2413- allow_type_promotion : bool ,
2414- ) -> Result < Vec < String > , String > {
2429+ /// The pushed-down `substring(s, 1, 2) = 'ab'` data filter over `schema`, the way
2430+ /// `CometNativeScan` sends it: built by the real planner from the proto `CometSubstring` emits,
2431+ /// with Int32 literals for the position and length. `SparkSubstring` takes Int64 positions, so
2432+ /// the planner coerces those literals with `CastExpr::new`.
2433+ fn substring_filter ( schema : SchemaRef ) -> Result < Arc < dyn PhysicalExpr > , DataFusionError > {
24152434 use crate :: execution:: planner:: PhysicalPlanner ;
24162435 use datafusion:: common:: tree_node:: { TreeNode , TreeNodeRecursion } ;
2417- use datafusion:: config:: { ConfigOptions , TableParquetOptions } ;
2418- use datafusion:: datasource:: physical_plan:: FileSource ;
24192436 use datafusion:: logical_expr:: ScalarUDF ;
24202437 use datafusion:: physical_expr:: expressions:: CastExpr ;
2421- use datafusion:: physical_plan:: common:: collect;
24222438 use datafusion:: prelude:: SessionContext ;
24232439 use datafusion_comet_proto:: spark_expression:: { self , expr:: ExprStruct , literal, Expr } ;
24242440 use datafusion_spark:: function:: string:: substring:: SparkSubstring ;
24252441
2426- let schema = Arc :: new ( Schema :: new ( vec ! [ Field :: new( "s" , DataType :: Utf8 , true ) ] ) ) ;
2427- let batch = RecordBatch :: try_new (
2428- Arc :: clone ( & schema) ,
2429- vec ! [ Arc :: new( StringArray :: from( vec![
2430- "abc" , "xyz" , "abd" , "qrs" ,
2431- ] ) ) ] ,
2432- )
2433- . map_err ( |e| e. to_string ( ) ) ?;
2434- let filename = get_temp_filename ( ) ;
2435- let filename = filename. as_path ( ) . as_os_str ( ) . to_str ( ) . unwrap ( ) . to_string ( ) ;
2436- let file = File :: create ( & filename) . map_err ( |e| e. to_string ( ) ) ?;
2437- let mut writer =
2438- ArrowWriter :: try_new ( file, Arc :: clone ( & schema) , None ) . map_err ( |e| e. to_string ( ) ) ?;
2439- writer. write ( & batch) . map_err ( |e| e. to_string ( ) ) ?;
2440- writer. close ( ) . map_err ( |e| e. to_string ( ) ) ?;
2441-
24422442 let session_ctx = Arc :: new ( SessionContext :: new ( ) ) ;
24432443 session_ctx. register_udf ( ScalarUDF :: new_from_impl ( SparkSubstring :: default ( ) ) ) ;
24442444 let planner = PhysicalPlanner :: new ( Arc :: clone ( & session_ctx) , 0 ) ;
@@ -2473,120 +2473,90 @@ pub(crate) mod test {
24732473 fail_on_error : false ,
24742474 null_short_circuit : false ,
24752475 } ) ) ;
2476- let predicate = planner
2477- . create_expr (
2478- & proto_expr ( ExprStruct :: Eq ( Box :: new ( spark_expression:: BinaryExpr {
2479- left : Some ( Box :: new ( substring) ) ,
2480- right : Some ( Box :: new ( proto_expr ( ExprStruct :: Literal (
2481- spark_expression:: Literal {
2482- value : Some ( literal:: Value :: StringVal ( "ab" . to_string ( ) ) ) ,
2483- datatype : Some ( proto_type ( 7 ) ) ,
2484- is_null : false ,
2485- } ,
2486- ) ) ) ) ,
2487- } ) ) ) ,
2488- Arc :: clone ( & schema) ,
2489- )
2490- . map_err ( |e| e. to_string ( ) ) ?;
2476+ let predicate = planner. create_expr (
2477+ & proto_expr ( ExprStruct :: Eq ( Box :: new ( spark_expression:: BinaryExpr {
2478+ left : Some ( Box :: new ( substring) ) ,
2479+ right : Some ( Box :: new ( proto_expr ( ExprStruct :: Literal (
2480+ spark_expression:: Literal {
2481+ value : Some ( literal:: Value :: StringVal ( "ab" . to_string ( ) ) ) ,
2482+ datatype : Some ( proto_type ( 7 ) ) ,
2483+ is_null : false ,
2484+ } ,
2485+ ) ) ) ) ,
2486+ } ) ) ) ,
2487+ schema,
2488+ ) ?;
24912489
24922490 // Guard the premise: the planner really does leave DataFusion `CastExpr`s in the predicate.
24932491 let mut casts = 0 ;
2494- predicate
2495- . apply ( |e| {
2496- if e. downcast_ref :: < CastExpr > ( ) . is_some ( ) {
2497- casts += 1 ;
2498- }
2499- Ok ( TreeNodeRecursion :: Continue )
2500- } )
2501- . map_err ( |e| e. to_string ( ) ) ?;
2492+ predicate. apply ( |e| {
2493+ if e. downcast_ref :: < CastExpr > ( ) . is_some ( ) {
2494+ casts += 1 ;
2495+ }
2496+ Ok ( TreeNodeRecursion :: Continue )
2497+ } ) ?;
25022498 assert ! ( casts >= 2 , "expected planner-inserted casts in {predicate}" ) ;
2503-
2504- // Configure the scan the way the native scan does when
2505- // `spark.comet.parquet.rowFilterPushdown.enabled` is set, and push the filter the way
2506- // `init_datasource_exec` does.
2507- let mut table_options = TableParquetOptions :: new ( ) ;
2508- table_options. global . pushdown_filters = row_filter_pushdown;
2509- table_options. global . reorder_filters = row_filter_pushdown;
2510- let mut config = ConfigOptions :: default ( ) ;
2511- config. execution . parquet . pushdown_filters = row_filter_pushdown;
2512- config. execution . parquet . reorder_filters = row_filter_pushdown;
2513- let parquet_source =
2514- ParquetSource :: new ( Arc :: clone ( & schema) ) . with_table_parquet_options ( table_options) ;
2515- let file_source: Arc < dyn FileSource > = parquet_source
2516- . try_pushdown_filters ( vec ! [ predicate] , & config)
2517- . map_err ( |e| e. to_string ( ) ) ?
2518- . updated_node
2519- . unwrap_or_else ( || Arc :: new ( parquet_source) ) ;
2520-
2521- let mut spark_parquet_options = SparkParquetOptions :: new ( EvalMode :: Legacy , "UTC" ) ;
2522- spark_parquet_options. allow_type_promotion = allow_type_promotion;
2523- let expr_adapter_factory: Arc < dyn PhysicalExprAdapterFactory > = Arc :: new (
2524- SparkPhysicalExprAdapterFactory :: new ( spark_parquet_options, None ) ,
2525- ) ;
2526- let files = FileGroup :: new ( vec ! [
2527- PartitionedFile :: from_path( filename) . map_err( |e| e. to_string( ) ) ?
2528- ] ) ;
2529- let file_scan_config =
2530- FileScanConfigBuilder :: new ( ObjectStoreUrl :: local_filesystem ( ) , file_source)
2531- . with_file_groups ( vec ! [ files] )
2532- . with_expr_adapter ( Some ( expr_adapter_factory) )
2533- . build ( ) ;
2534-
2535- let scan = DataSourceExec :: new ( Arc :: new ( file_scan_config) ) ;
2536- let stream = scan
2537- . execute ( 0 , Arc :: new ( TaskContext :: default ( ) ) )
2538- . map_err ( |e| e. to_string ( ) ) ?;
2539- let batches = collect ( stream) . await . map_err ( |e| e. to_string ( ) ) ?;
2540- let mut values = vec ! [ ] ;
2541- for batch in & batches {
2542- let column = arrow:: compute:: cast ( batch. column ( 0 ) , & DataType :: Utf8 )
2543- . map_err ( |e| e. to_string ( ) ) ?;
2544- values. extend (
2545- column
2546- . as_string :: < i32 > ( )
2547- . iter ( )
2548- . flatten ( )
2549- . map ( str:: to_string) ,
2550- ) ;
2551- }
2552- Ok ( values)
2553- }
2554-
2555- /// Control: without row-level filter pushdown (the default) the predicate is only used for
2556- /// row-group and page pruning, so it is never evaluated against a decoded batch. The scan
2557- /// returns every row and leaves the filtering to the Spark `Filter` above it.
2558- #[ tokio:: test]
2559- async fn pushed_substring_filter_without_row_filter_pushdown ( ) {
2560- assert_eq ! (
2561- scan_with_substring_filter( false , false ) . await ,
2562- Ok ( [ "abc" , "xyz" , "abd" , "qrs" ]
2563- . iter( )
2564- . map( |s| s. to_string( ) )
2565- . collect:: <Vec <_>>( ) )
2566- ) ;
2567- }
2568-
2569- /// With `spark.comet.parquet.rowFilterPushdown.enabled=true` the predicate is evaluated on every
2570- /// decoded batch. On Spark 3.x (`allow_type_promotion = false`) the planner's `Int32 -> Int64`
2571- /// coercion of the `substring` literals must not be mistaken for a Parquet schema promotion,
2572- /// which failed the scan with `ParquetSchemaConvert { column: "[]", physical_type: "INT32",
2573- /// spark_type: "bigint" }`.
2574- #[ tokio:: test]
2575- async fn pushed_substring_filter_with_row_filter_pushdown ( ) {
2576- assert_eq ! (
2577- scan_with_substring_filter( true , false ) . await ,
2578- Ok ( vec![ "abc" . to_string( ) , "abd" . to_string( ) ] )
2579- ) ;
2499+ Ok ( predicate)
25802500 }
25812501
2582- /// Same as above for Spark 4.x (`allow_type_promotion = true`), which accepts `Int32 -> Int64`
2583- /// and so never exposed the problem for this particular cast .
2502+ /// The casts Comet's planner puts in a pushed-down predicate are not Parquet read conversions.
2503+ /// The file's schema equals the requested schema, so there is no schema evolution of any kind .
25842504 #[ tokio:: test]
2585- async fn pushed_substring_filter_with_row_filter_pushdown_and_type_promotion ( ) {
2586- assert_eq ! (
2587- scan_with_substring_filter( true , true ) . await ,
2588- Ok ( vec![ "abc" . to_string( ) , "abd" . to_string( ) ] )
2589- ) ;
2505+ async fn pushed_substring_filter_keeps_planner_casts ( ) -> Result < ( ) , DataFusionError > {
2506+ let schema = Arc :: new ( Schema :: new ( vec ! [ Field :: new( "s" , DataType :: Utf8 , true ) ] ) ) ;
2507+ let batch = RecordBatch :: try_new (
2508+ Arc :: clone ( & schema) ,
2509+ vec ! [ Arc :: new( StringArray :: from( vec![
2510+ "abc" , "xyz" , "abd" , "qrs" ,
2511+ ] ) ) ] ,
2512+ ) ?;
2513+ let filename = get_temp_filename ( ) . to_str ( ) . unwrap ( ) . to_string ( ) ;
2514+ let mut writer = ArrowWriter :: try_new ( File :: create ( & filename) ?, Arc :: clone ( & schema) , None ) ?;
2515+ writer. write ( & batch) ?;
2516+ writer. close ( ) ?;
2517+ let predicate = substring_filter ( Arc :: clone ( & schema) ) ?;
2518+
2519+ let every_row: & [ & str ] = & [ "abc" , "xyz" , "abd" , "qrs" ] ;
2520+ let matching: & [ & str ] = & [ "abc" , "abd" ] ;
2521+ for ( row_filters, allow_type_promotion, expected) in [
2522+ // Control: without row-level filter pushdown (the default) the predicate is only used
2523+ // for row-group and page pruning, so the scan returns every row and leaves the
2524+ // filtering to the Spark `Filter` above it.
2525+ ( false , false , every_row) ,
2526+ // With `spark.comet.parquet.rowFilterPushdown.enabled=true` the predicate is evaluated
2527+ // on every decoded batch. On Spark 3.x (`allow_type_promotion = false`) the planner's
2528+ // Int32 -> Int64 coercion was taken for a disallowed `INT32 -> bigint` promotion, which
2529+ // failed the scan with `ParquetSchemaConvert { column: "[]", .. }`.
2530+ ( true , false , matching) ,
2531+ // Control: Spark 4.x allows that promotion, so this case passed before the fix too.
2532+ ( true , true , matching) ,
2533+ ] {
2534+ let mut options = default_options ( ) ;
2535+ options. allow_type_promotion = allow_type_promotion;
2536+ let mut stream = scan_parquet_file (
2537+ filename. clone ( ) ,
2538+ Arc :: clone ( & schema) ,
2539+ options,
2540+ Some ( Arc :: clone ( & predicate) ) ,
2541+ row_filters,
2542+ ) ?;
2543+ let mut values: Vec < String > = vec ! [ ] ;
2544+ while let Some ( batch) = stream. next ( ) . await {
2545+ let column = arrow:: compute:: cast ( batch?. column ( 0 ) , & DataType :: Utf8 ) ?;
2546+ values. extend (
2547+ column
2548+ . as_string :: < i32 > ( )
2549+ . iter ( )
2550+ . flatten ( )
2551+ . map ( str:: to_string) ,
2552+ ) ;
2553+ }
2554+ assert_eq ! (
2555+ values, expected,
2556+ "row_filters={row_filters} allow_type_promotion={allow_type_promotion}"
2557+ ) ;
2558+ }
2559+ Ok ( ( ) )
25902560 }
25912561
25922562 #[ tokio:: test]
0 commit comments