@@ -29,7 +29,7 @@ use datafusion_datasource::{FileExtensions, ListingTableUrl};
2929use datafusion_expr:: { BinaryExpr , Operator , lit, utils} ;
3030
3131use arrow:: {
32- array:: AsArray ,
32+ array:: { Array , AsArray } ,
3333 datatypes:: { DataType , Field , Metadata } ,
3434 record_batch:: RecordBatch ,
3535} ;
@@ -337,23 +337,30 @@ pub fn filter_partitioned_file(
337337 ) ?;
338338
339339 // Since we're only operating on a single file, our batch and resulting "array" holds only one
340- // value indicating if the input file matches the provided filters
340+ // value indicating if the input file matches the provided filters. A NULL is not a match:
341+ // a null partition value makes that reachable, as `part = 1` is NULL for a null partition.
341342 let matches = expr. evaluate ( & batch) ?. into_array ( 1 ) ?;
342- if matches. as_boolean ( ) . value ( 0 ) {
343+ let matches = matches. as_boolean ( ) ;
344+ if matches. is_valid ( 0 ) && matches. value ( 0 ) {
343345 return Ok ( Some ( pf) ) ;
344346 }
345347
346348 Ok ( None )
347349}
348350
349351/// Returns `Ok(None)` when the file is not inside a valid partition path
350- /// (e.g. a stale file in the table root directory). Such files are skipped
351- /// because hive-style partition values are never null and there is no valid
352- /// value to assign for non-partitioned files.
352+ /// (e.g. a stale file in the table root directory), as there is no value to
353+ /// assign for non-partitioned files.
354+ ///
355+ /// A partition directory named `null_partition_name` is read as NULL; see
356+ /// [`ExecutionOptions::listing_table_hive_default_partition_name`].
357+ ///
358+ /// [`ExecutionOptions::listing_table_hive_default_partition_name`]: datafusion_common::config::ExecutionOptions::listing_table_hive_default_partition_name
353359fn try_into_partitioned_file (
354360 object_meta : ObjectMeta ,
355361 partition_cols : & [ ( String , DataType ) ] ,
356362 table_path : & ListingTableUrl ,
363+ null_partition_name : & str ,
357364) -> Result < Option < PartitionedFile > > {
358365 let cols = partition_cols. iter ( ) . map ( |( name, _) | name. as_str ( ) ) ;
359366 let parsed = parse_partitions_for_path ( table_path, & object_meta. location , cols) ;
@@ -367,6 +374,11 @@ fn try_into_partitioned_file(
367374 . into_iter ( )
368375 . zip ( partition_cols)
369376 . map ( |( parsed, ( _, datatype) ) | {
377+ // Hive compliant writers spell a null partition value with a placeholder
378+ // directory name, so there is nothing to parse: the value is NULL.
379+ if !null_partition_name. is_empty ( ) && parsed == null_partition_name {
380+ return ScalarValue :: try_new_null ( datatype) ;
381+ }
370382 ScalarValue :: try_from_string ( parsed. into_owned ( ) , datatype)
371383 } )
372384 . collect :: < Result < Vec < _ > > > ( ) ?;
@@ -426,12 +438,19 @@ pub async fn pruned_partition_list<'a>(
426438 Metadata :: new ( ) ,
427439 ) ?;
428440
441+ let null_partition_name = ctx
442+ . config_options ( )
443+ . execution
444+ . listing_table_hive_default_partition_name
445+ . as_str ( ) ;
446+
429447 Ok ( objects
430- . try_filter_map ( |object_meta| {
448+ . try_filter_map ( move |object_meta| {
431449 futures:: future:: ready ( try_into_partitioned_file (
432450 object_meta,
433451 partition_cols,
434452 table_path,
453+ null_partition_name,
435454 ) )
436455 } )
437456 . try_filter_map ( move |pf| {
@@ -508,6 +527,7 @@ pub fn describe_partition(partition: &Partition) -> (&str, usize, Vec<&str>) {
508527
509528#[ cfg( test) ]
510529mod tests {
530+ use datafusion_common:: config:: HIVE_DEFAULT_PARTITION_NAME ;
511531 use datafusion_datasource:: file_groups:: FileGroup ;
512532 use std:: ops:: Not ;
513533
@@ -672,8 +692,13 @@ mod tests {
672692 version : None ,
673693 } ;
674694
675- let result =
676- try_into_partitioned_file ( meta, & partition_cols, & table_path) . unwrap ( ) ;
695+ let result = try_into_partitioned_file (
696+ meta,
697+ & partition_cols,
698+ & table_path,
699+ HIVE_DEFAULT_PARTITION_NAME ,
700+ )
701+ . unwrap ( ) ;
677702 assert ! ( result. is_some( ) ) ;
678703 let pf = result. unwrap ( ) ;
679704 assert_eq ! ( pf. partition_values. len( ) , 1 ) ;
@@ -698,8 +723,13 @@ mod tests {
698723 version : None ,
699724 } ;
700725
701- let result =
702- try_into_partitioned_file ( meta, & partition_cols, & table_path) . unwrap ( ) ;
726+ let result = try_into_partitioned_file (
727+ meta,
728+ & partition_cols,
729+ & table_path,
730+ HIVE_DEFAULT_PARTITION_NAME ,
731+ )
732+ . unwrap ( ) ;
703733 assert ! ( result. is_some( ) ) ;
704734 let pf = result. unwrap ( ) ;
705735 assert_eq ! ( pf. partition_values. len( ) , 1 ) ;
@@ -709,6 +739,60 @@ mod tests {
709739 ) ;
710740 }
711741
742+ #[ test]
743+ fn test_try_into_partitioned_file_null_partition ( ) {
744+ let table_path = ListingTableUrl :: parse ( "file:///bucket/mytable" ) . unwrap ( ) ;
745+ // A type the placeholder cannot be parsed as, which is how this used to fail.
746+ let partition_cols = vec ! [ ( "day" . to_string( ) , DataType :: Date32 ) ] ;
747+ let meta = ObjectMeta {
748+ location : Path :: from (
749+ "bucket/mytable/day=__HIVE_DEFAULT_PARTITION__/data.parquet" ,
750+ ) ,
751+ last_modified : chrono:: Utc :: now ( ) ,
752+ size : 100 ,
753+ e_tag : None ,
754+ version : None ,
755+ } ;
756+
757+ let pf = try_into_partitioned_file (
758+ meta,
759+ & partition_cols,
760+ & table_path,
761+ HIVE_DEFAULT_PARTITION_NAME ,
762+ )
763+ . unwrap ( )
764+ . expect ( "file should be listed" ) ;
765+
766+ assert_eq ! ( pf. partition_values, vec![ ScalarValue :: Date32 ( None ) ] ) ;
767+ }
768+
769+ #[ test]
770+ fn test_try_into_partitioned_file_null_partition_name_not_matched ( ) {
771+ let table_path = ListingTableUrl :: parse ( "file:///bucket/mytable" ) . unwrap ( ) ;
772+ let partition_cols = vec ! [ ( "category" . to_string( ) , DataType :: Utf8 ) ] ;
773+ let meta = ObjectMeta {
774+ location : Path :: from (
775+ "bucket/mytable/category=__HIVE_DEFAULT_PARTITION__/data.parquet" ,
776+ ) ,
777+ last_modified : chrono:: Utc :: now ( ) ,
778+ size : 100 ,
779+ e_tag : None ,
780+ version : None ,
781+ } ;
782+
783+ // The empty name turns the mapping off, leaving the directory name as the value.
784+ let pf = try_into_partitioned_file ( meta, & partition_cols, & table_path, "" )
785+ . unwrap ( )
786+ . expect ( "file should be listed" ) ;
787+
788+ assert_eq ! (
789+ pf. partition_values,
790+ vec![ ScalarValue :: Utf8 ( Some (
791+ "__HIVE_DEFAULT_PARTITION__" . to_string( )
792+ ) ) ]
793+ ) ;
794+ }
795+
712796 #[ test]
713797 fn test_try_into_partitioned_file_root_file_skipped ( ) {
714798 // File in root directory (not inside any partition path) should be
@@ -724,8 +808,13 @@ mod tests {
724808 version : None ,
725809 } ;
726810
727- let result =
728- try_into_partitioned_file ( meta, & partition_cols, & table_path) . unwrap ( ) ;
811+ let result = try_into_partitioned_file (
812+ meta,
813+ & partition_cols,
814+ & table_path,
815+ HIVE_DEFAULT_PARTITION_NAME ,
816+ )
817+ . unwrap ( ) ;
729818 assert ! (
730819 result. is_none( ) ,
731820 "Files outside partition structure should be skipped"
@@ -745,8 +834,13 @@ mod tests {
745834 version : None ,
746835 } ;
747836
748- let result =
749- try_into_partitioned_file ( meta, & partition_cols, & table_path) . unwrap ( ) ;
837+ let result = try_into_partitioned_file (
838+ meta,
839+ & partition_cols,
840+ & table_path,
841+ HIVE_DEFAULT_PARTITION_NAME ,
842+ )
843+ . unwrap ( ) ;
750844 assert ! (
751845 result. is_none( ) ,
752846 "Files with wrong partition column name should be skipped"
@@ -768,8 +862,13 @@ mod tests {
768862 version : None ,
769863 } ;
770864
771- let result =
772- try_into_partitioned_file ( meta, & partition_cols, & table_path) . unwrap ( ) ;
865+ let result = try_into_partitioned_file (
866+ meta,
867+ & partition_cols,
868+ & table_path,
869+ HIVE_DEFAULT_PARTITION_NAME ,
870+ )
871+ . unwrap ( ) ;
773872 assert ! ( result. is_some( ) ) ;
774873 let pf = result. unwrap ( ) ;
775874 assert_eq ! ( pf. partition_values. len( ) , 2 ) ;
@@ -799,8 +898,13 @@ mod tests {
799898 version : None ,
800899 } ;
801900
802- let result =
803- try_into_partitioned_file ( meta, & partition_cols, & table_path) . unwrap ( ) ;
901+ let result = try_into_partitioned_file (
902+ meta,
903+ & partition_cols,
904+ & table_path,
905+ HIVE_DEFAULT_PARTITION_NAME ,
906+ )
907+ . unwrap ( ) ;
804908 // File has year=2024 but no month= directory — parse_partitions_for_path
805909 // returns None because the path component "data.parquet" doesn't match
806910 // the expected "month=..." pattern.
0 commit comments