@@ -16,7 +16,7 @@ use datafusion_datasource::{FileExtensions, ListingTableUrl};
1616use datafusion_expr:: { BinaryExpr , Operator , lit, utils} ;
1717
1818use arrow:: {
19- array:: AsArray ,
19+ array:: { Array , AsArray } ,
2020 datatypes:: { DataType , Field , Metadata } ,
2121 record_batch:: RecordBatch ,
2222} ;
@@ -324,23 +324,30 @@ pub fn filter_partitioned_file(
324324 ) ?;
325325
326326 // Since we're only operating on a single file, our batch and resulting "array" holds only one
327- // value indicating if the input file matches the provided filters
327+ // value indicating if the input file matches the provided filters. A NULL is not a match:
328+ // a null partition value makes that reachable, as `part = 1` is NULL for a null partition.
328329 let matches = expr. evaluate ( & batch) ?. into_array ( 1 ) ?;
329- if matches. as_boolean ( ) . value ( 0 ) {
330+ let matches = matches. as_boolean ( ) ;
331+ if matches. is_valid ( 0 ) && matches. value ( 0 ) {
330332 return Ok ( Some ( pf) ) ;
331333 }
332334
333335 Ok ( None )
334336}
335337
336338/// Returns `Ok(None)` when the file is not inside a valid partition path
337- /// (e.g. a stale file in the table root directory). Such files are skipped
338- /// because hive-style partition values are never null and there is no valid
339- /// value to assign for non-partitioned files.
339+ /// (e.g. a stale file in the table root directory), as there is no value to
340+ /// assign for non-partitioned files.
341+ ///
342+ /// A partition directory named `null_partition_name` is read as NULL; see
343+ /// [`ExecutionOptions::listing_table_hive_default_partition_name`].
344+ ///
345+ /// [`ExecutionOptions::listing_table_hive_default_partition_name`]: datafusion_common::config::ExecutionOptions::listing_table_hive_default_partition_name
340346fn try_into_partitioned_file (
341347 object_meta : ObjectMeta ,
342348 partition_cols : & [ ( String , DataType ) ] ,
343349 table_path : & ListingTableUrl ,
350+ null_partition_name : & str ,
344351) -> Result < Option < PartitionedFile > > {
345352 let cols = partition_cols. iter ( ) . map ( |( name, _) | name. as_str ( ) ) ;
346353 let parsed = parse_partitions_for_path ( table_path, & object_meta. location , cols) ;
@@ -354,6 +361,9 @@ fn try_into_partitioned_file(
354361 . into_iter ( )
355362 . zip ( partition_cols)
356363 . map ( |( parsed, ( _, datatype) ) | {
364+ if !null_partition_name. is_empty ( ) && parsed == null_partition_name {
365+ return ScalarValue :: try_new_null ( datatype) ;
366+ }
357367 ScalarValue :: try_from_string ( parsed. into_owned ( ) , datatype)
358368 } )
359369 . collect :: < Result < Vec < _ > > > ( ) ?;
@@ -413,12 +423,19 @@ pub async fn pruned_partition_list<'a>(
413423 Metadata :: new ( ) ,
414424 ) ?;
415425
426+ let null_partition_name = ctx
427+ . config_options ( )
428+ . execution
429+ . listing_table_hive_default_partition_name
430+ . as_str ( ) ;
431+
416432 Ok ( objects
417- . try_filter_map ( |object_meta| {
433+ . try_filter_map ( move |object_meta| {
418434 futures:: future:: ready ( try_into_partitioned_file (
419435 object_meta,
420436 partition_cols,
421437 table_path,
438+ null_partition_name,
422439 ) )
423440 } )
424441 . try_filter_map ( move |pf| {
@@ -495,6 +512,7 @@ pub fn describe_partition(partition: &Partition) -> (&str, usize, Vec<&str>) {
495512
496513#[ cfg( test) ]
497514mod tests {
515+ use datafusion_common:: config:: HIVE_DEFAULT_PARTITION_NAME ;
498516 use datafusion_datasource:: file_groups:: FileGroup ;
499517 use std:: ops:: Not ;
500518
@@ -659,8 +677,13 @@ mod tests {
659677 version : None ,
660678 } ;
661679
662- let result =
663- try_into_partitioned_file ( meta, & partition_cols, & table_path) . unwrap ( ) ;
680+ let result = try_into_partitioned_file (
681+ meta,
682+ & partition_cols,
683+ & table_path,
684+ HIVE_DEFAULT_PARTITION_NAME ,
685+ )
686+ . unwrap ( ) ;
664687 assert ! ( result. is_some( ) ) ;
665688 let pf = result. unwrap ( ) ;
666689 assert_eq ! ( pf. partition_values. len( ) , 1 ) ;
@@ -685,8 +708,13 @@ mod tests {
685708 version : None ,
686709 } ;
687710
688- let result =
689- try_into_partitioned_file ( meta, & partition_cols, & table_path) . unwrap ( ) ;
711+ let result = try_into_partitioned_file (
712+ meta,
713+ & partition_cols,
714+ & table_path,
715+ HIVE_DEFAULT_PARTITION_NAME ,
716+ )
717+ . unwrap ( ) ;
690718 assert ! ( result. is_some( ) ) ;
691719 let pf = result. unwrap ( ) ;
692720 assert_eq ! ( pf. partition_values. len( ) , 1 ) ;
@@ -696,6 +724,60 @@ mod tests {
696724 ) ;
697725 }
698726
727+ #[ test]
728+ fn test_try_into_partitioned_file_null_partition ( ) {
729+ let table_path = ListingTableUrl :: parse ( "file:///bucket/mytable" ) . unwrap ( ) ;
730+ // A type the placeholder cannot be parsed as, which is how this used to fail.
731+ let partition_cols = vec ! [ ( "day" . to_string( ) , DataType :: Date32 ) ] ;
732+ let meta = ObjectMeta {
733+ location : Path :: from (
734+ "bucket/mytable/day=__HIVE_DEFAULT_PARTITION__/data.parquet" ,
735+ ) ,
736+ last_modified : chrono:: Utc :: now ( ) ,
737+ size : 100 ,
738+ e_tag : None ,
739+ version : None ,
740+ } ;
741+
742+ let pf = try_into_partitioned_file (
743+ meta,
744+ & partition_cols,
745+ & table_path,
746+ HIVE_DEFAULT_PARTITION_NAME ,
747+ )
748+ . unwrap ( )
749+ . expect ( "file should be listed" ) ;
750+
751+ assert_eq ! ( pf. partition_values, vec![ ScalarValue :: Date32 ( None ) ] ) ;
752+ }
753+
754+ #[ test]
755+ fn test_try_into_partitioned_file_null_partition_name_not_matched ( ) {
756+ let table_path = ListingTableUrl :: parse ( "file:///bucket/mytable" ) . unwrap ( ) ;
757+ let partition_cols = vec ! [ ( "category" . to_string( ) , DataType :: Utf8 ) ] ;
758+ let meta = ObjectMeta {
759+ location : Path :: from (
760+ "bucket/mytable/category=__HIVE_DEFAULT_PARTITION__/data.parquet" ,
761+ ) ,
762+ last_modified : chrono:: Utc :: now ( ) ,
763+ size : 100 ,
764+ e_tag : None ,
765+ version : None ,
766+ } ;
767+
768+ // The empty name turns the mapping off, leaving the directory name as the value.
769+ let pf = try_into_partitioned_file ( meta, & partition_cols, & table_path, "" )
770+ . unwrap ( )
771+ . expect ( "file should be listed" ) ;
772+
773+ assert_eq ! (
774+ pf. partition_values,
775+ vec![ ScalarValue :: Utf8 ( Some (
776+ "__HIVE_DEFAULT_PARTITION__" . to_string( )
777+ ) ) ]
778+ ) ;
779+ }
780+
699781 #[ test]
700782 fn test_try_into_partitioned_file_root_file_skipped ( ) {
701783 // File in root directory (not inside any partition path) should be
@@ -711,8 +793,13 @@ mod tests {
711793 version : None ,
712794 } ;
713795
714- let result =
715- try_into_partitioned_file ( meta, & partition_cols, & table_path) . unwrap ( ) ;
796+ let result = try_into_partitioned_file (
797+ meta,
798+ & partition_cols,
799+ & table_path,
800+ HIVE_DEFAULT_PARTITION_NAME ,
801+ )
802+ . unwrap ( ) ;
716803 assert ! (
717804 result. is_none( ) ,
718805 "Files outside partition structure should be skipped"
@@ -732,8 +819,13 @@ mod tests {
732819 version : None ,
733820 } ;
734821
735- let result =
736- try_into_partitioned_file ( meta, & partition_cols, & table_path) . unwrap ( ) ;
822+ let result = try_into_partitioned_file (
823+ meta,
824+ & partition_cols,
825+ & table_path,
826+ HIVE_DEFAULT_PARTITION_NAME ,
827+ )
828+ . unwrap ( ) ;
737829 assert ! (
738830 result. is_none( ) ,
739831 "Files with wrong partition column name should be skipped"
@@ -755,8 +847,13 @@ mod tests {
755847 version : None ,
756848 } ;
757849
758- let result =
759- try_into_partitioned_file ( meta, & partition_cols, & table_path) . unwrap ( ) ;
850+ let result = try_into_partitioned_file (
851+ meta,
852+ & partition_cols,
853+ & table_path,
854+ HIVE_DEFAULT_PARTITION_NAME ,
855+ )
856+ . unwrap ( ) ;
760857 assert ! ( result. is_some( ) ) ;
761858 let pf = result. unwrap ( ) ;
762859 assert_eq ! ( pf. partition_values. len( ) , 2 ) ;
@@ -786,8 +883,13 @@ mod tests {
786883 version : None ,
787884 } ;
788885
789- let result =
790- try_into_partitioned_file ( meta, & partition_cols, & table_path) . unwrap ( ) ;
886+ let result = try_into_partitioned_file (
887+ meta,
888+ & partition_cols,
889+ & table_path,
890+ HIVE_DEFAULT_PARTITION_NAME ,
891+ )
892+ . unwrap ( ) ;
791893 // File has year=2024 but no month= directory — parse_partitions_for_path
792894 // returns None because the path component "data.parquet" doesn't match
793895 // the expected "month=..." pattern.
0 commit comments