@@ -29,7 +29,7 @@ use datafusion_datasource::{FileExtensions, ListingTableUrl};
2929use datafusion_expr:: { BinaryExpr , Operator , lit, utils} ;
3030
3131use arrow:: {
32- array:: AsArray ,
32+ array:: { Array , AsArray } ,
3333 datatypes:: { DataType , Field } ,
3434 record_batch:: RecordBatch ,
3535} ;
@@ -50,6 +50,8 @@ use percent_encoding::{AsciiSet, CONTROLS, percent_decode_str, utf8_percent_enco
5050const PARTITION_VALUE_ENCODE_SET : & AsciiSet =
5151 & CONTROLS . add ( b' ' ) . add ( b'%' ) . add ( b'/' ) . add ( b'?' ) . add ( b'#' ) ;
5252
53+ const HIVE_DEFAULT_PARTITION_NAME : & str = "__HIVE_DEFAULT_PARTITION__" ;
54+
5355/// Check whether the given expression can be resolved using only the columns `col_names`.
5456/// This means that if this function returns true:
5557/// - the table provider can filter the table partition values with this expression
@@ -337,19 +339,22 @@ pub fn filter_partitioned_file(
337339 ) ?;
338340
339341 // Since we're only operating on a single file, our batch and resulting "array" holds only one
340- // value indicating if the input file matches the provided filters
342+ // value indicating if the input file matches the provided filters. A NULL is not a match:
343+ // a null partition value makes that reachable, as `part = 1` is NULL for a null partition.
341344 let matches = expr. evaluate ( & batch) ?. into_array ( 1 ) ?;
342- if matches. as_boolean ( ) . value ( 0 ) {
345+ let matches = matches. as_boolean ( ) ;
346+ if matches. is_valid ( 0 ) && matches. value ( 0 ) {
343347 return Ok ( Some ( pf) ) ;
344348 }
345349
346350 Ok ( None )
347351}
348352
349353/// Returns `Ok(None)` when the file is not inside a valid partition path
350- /// (e.g. a stale file in the table root directory). Such files are skipped
351- /// because hive-style partition values are never null and there is no valid
352- /// value to assign for non-partitioned files.
354+ /// (e.g. a stale file in the table root directory), as there is no value to
355+ /// assign for non-partitioned files.
356+ ///
357+ /// A partition value named `__HIVE_DEFAULT_PARTITION__` is read as NULL.
353358fn try_into_partitioned_file (
354359 object_meta : ObjectMeta ,
355360 partition_cols : & [ ( String , DataType ) ] ,
@@ -367,6 +372,9 @@ fn try_into_partitioned_file(
367372 . into_iter ( )
368373 . zip ( partition_cols)
369374 . map ( |( parsed, ( _, datatype) ) | {
375+ if parsed == HIVE_DEFAULT_PARTITION_NAME {
376+ return ScalarValue :: try_new_null ( datatype) ;
377+ }
370378 ScalarValue :: try_from_string ( parsed. into_owned ( ) , datatype)
371379 } )
372380 . collect :: < Result < Vec < _ > > > ( ) ?;
@@ -427,7 +435,7 @@ pub async fn pruned_partition_list<'a>(
427435 ) ?;
428436
429437 Ok ( objects
430- . try_filter_map ( |object_meta| {
438+ . try_filter_map ( move |object_meta| {
431439 futures:: future:: ready ( try_into_partitioned_file (
432440 object_meta,
433441 partition_cols,
@@ -709,6 +717,50 @@ mod tests {
709717 ) ;
710718 }
711719
720+ #[ test]
721+ fn test_try_into_partitioned_file_null_partition ( ) {
722+ let table_path = ListingTableUrl :: parse ( "file:///bucket/mytable" ) . unwrap ( ) ;
723+ // A type the placeholder cannot be parsed as, which is how this used to fail.
724+ let partition_cols = vec ! [ ( "day" . to_string( ) , DataType :: Date32 ) ] ;
725+ let meta = ObjectMeta {
726+ location : Path :: from (
727+ "bucket/mytable/day=__HIVE_DEFAULT_PARTITION__/data.parquet" ,
728+ ) ,
729+ last_modified : chrono:: Utc :: now ( ) ,
730+ size : 100 ,
731+ e_tag : None ,
732+ version : None ,
733+ } ;
734+
735+ let pf = try_into_partitioned_file ( meta, & partition_cols, & table_path)
736+ . unwrap ( )
737+ . expect ( "file should be listed" ) ;
738+
739+ assert_eq ! ( pf. partition_values, vec![ ScalarValue :: Date32 ( None ) ] ) ;
740+ }
741+
742+ #[ test]
743+ fn test_try_into_partitioned_file_other_partition_name_is_literal ( ) {
744+ let table_path = ListingTableUrl :: parse ( "file:///bucket/mytable" ) . unwrap ( ) ;
745+ let partition_cols = vec ! [ ( "category" . to_string( ) , DataType :: Utf8 ) ] ;
746+ let meta = ObjectMeta {
747+ location : Path :: from ( "bucket/mytable/category=MY_NULL/data.parquet" ) ,
748+ last_modified : chrono:: Utc :: now ( ) ,
749+ size : 100 ,
750+ e_tag : None ,
751+ version : None ,
752+ } ;
753+
754+ let pf = try_into_partitioned_file ( meta, & partition_cols, & table_path)
755+ . unwrap ( )
756+ . expect ( "file should be listed" ) ;
757+
758+ assert_eq ! (
759+ pf. partition_values,
760+ vec![ ScalarValue :: Utf8 ( Some ( "MY_NULL" . to_string( ) ) ) ]
761+ ) ;
762+ }
763+
712764 #[ test]
713765 fn test_try_into_partitioned_file_root_file_skipped ( ) {
714766 // File in root directory (not inside any partition path) should be
0 commit comments