Skip to content

Commit 9795e8c

Browse files
authored
[branch-56] fix: read Hive's null partition placeholder as NULL (backport #26310) (#27171)
## Which issue does this PR close? - part of #24461 - Backport of #26310 to `branch-56` - Related to #18083. 🤖 Generated with [Claude Code](https://claude.com/claude-code)
1 parent 899bd1f commit 9795e8c

13 files changed

Lines changed: 673 additions & 45 deletions

File tree

‎datafusion/catalog-listing/src/helpers.rs‎

Lines changed: 121 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ use datafusion_datasource::{FileExtensions, ListingTableUrl};
1616
use datafusion_expr::{BinaryExpr, Operator, lit, utils};
1717

1818
use arrow::{
19-
array::AsArray,
19+
array::{Array, AsArray},
2020
datatypes::{DataType, Field, Metadata},
2121
record_batch::RecordBatch,
2222
};
@@ -324,23 +324,30 @@ pub fn filter_partitioned_file(
324324
)?;
325325

326326
// Since we're only operating on a single file, our batch and resulting "array" holds only one
327-
// value indicating if the input file matches the provided filters
327+
// value indicating if the input file matches the provided filters. A NULL is not a match:
328+
// a null partition value makes that reachable, as `part = 1` is NULL for a null partition.
328329
let matches = expr.evaluate(&batch)?.into_array(1)?;
329-
if matches.as_boolean().value(0) {
330+
let matches = matches.as_boolean();
331+
if matches.is_valid(0) && matches.value(0) {
330332
return Ok(Some(pf));
331333
}
332334

333335
Ok(None)
334336
}
335337

336338
/// Returns `Ok(None)` when the file is not inside a valid partition path
337-
/// (e.g. a stale file in the table root directory). Such files are skipped
338-
/// because hive-style partition values are never null and there is no valid
339-
/// value to assign for non-partitioned files.
339+
/// (e.g. a stale file in the table root directory), as there is no value to
340+
/// assign for non-partitioned files.
341+
///
342+
/// A partition directory named `null_partition_name` is read as NULL; see
343+
/// [`ExecutionOptions::listing_table_hive_default_partition_name`].
344+
///
345+
/// [`ExecutionOptions::listing_table_hive_default_partition_name`]: datafusion_common::config::ExecutionOptions::listing_table_hive_default_partition_name
340346
fn try_into_partitioned_file(
341347
object_meta: ObjectMeta,
342348
partition_cols: &[(String, DataType)],
343349
table_path: &ListingTableUrl,
350+
null_partition_name: &str,
344351
) -> Result<Option<PartitionedFile>> {
345352
let cols = partition_cols.iter().map(|(name, _)| name.as_str());
346353
let parsed = parse_partitions_for_path(table_path, &object_meta.location, cols);
@@ -354,6 +361,9 @@ fn try_into_partitioned_file(
354361
.into_iter()
355362
.zip(partition_cols)
356363
.map(|(parsed, (_, datatype))| {
364+
if !null_partition_name.is_empty() && parsed == null_partition_name {
365+
return ScalarValue::try_new_null(datatype);
366+
}
357367
ScalarValue::try_from_string(parsed.into_owned(), datatype)
358368
})
359369
.collect::<Result<Vec<_>>>()?;
@@ -413,12 +423,19 @@ pub async fn pruned_partition_list<'a>(
413423
Metadata::new(),
414424
)?;
415425

426+
let null_partition_name = ctx
427+
.config_options()
428+
.execution
429+
.listing_table_hive_default_partition_name
430+
.as_str();
431+
416432
Ok(objects
417-
.try_filter_map(|object_meta| {
433+
.try_filter_map(move |object_meta| {
418434
futures::future::ready(try_into_partitioned_file(
419435
object_meta,
420436
partition_cols,
421437
table_path,
438+
null_partition_name,
422439
))
423440
})
424441
.try_filter_map(move |pf| {
@@ -495,6 +512,7 @@ pub fn describe_partition(partition: &Partition) -> (&str, usize, Vec<&str>) {
495512

496513
#[cfg(test)]
497514
mod tests {
515+
use datafusion_common::config::HIVE_DEFAULT_PARTITION_NAME;
498516
use datafusion_datasource::file_groups::FileGroup;
499517
use std::ops::Not;
500518

@@ -659,8 +677,13 @@ mod tests {
659677
version: None,
660678
};
661679

662-
let result =
663-
try_into_partitioned_file(meta, &partition_cols, &table_path).unwrap();
680+
let result = try_into_partitioned_file(
681+
meta,
682+
&partition_cols,
683+
&table_path,
684+
HIVE_DEFAULT_PARTITION_NAME,
685+
)
686+
.unwrap();
664687
assert!(result.is_some());
665688
let pf = result.unwrap();
666689
assert_eq!(pf.partition_values.len(), 1);
@@ -685,8 +708,13 @@ mod tests {
685708
version: None,
686709
};
687710

688-
let result =
689-
try_into_partitioned_file(meta, &partition_cols, &table_path).unwrap();
711+
let result = try_into_partitioned_file(
712+
meta,
713+
&partition_cols,
714+
&table_path,
715+
HIVE_DEFAULT_PARTITION_NAME,
716+
)
717+
.unwrap();
690718
assert!(result.is_some());
691719
let pf = result.unwrap();
692720
assert_eq!(pf.partition_values.len(), 1);
@@ -696,6 +724,60 @@ mod tests {
696724
);
697725
}
698726

727+
#[test]
728+
fn test_try_into_partitioned_file_null_partition() {
729+
let table_path = ListingTableUrl::parse("file:///bucket/mytable").unwrap();
730+
// A type the placeholder cannot be parsed as, which is how this used to fail.
731+
let partition_cols = vec![("day".to_string(), DataType::Date32)];
732+
let meta = ObjectMeta {
733+
location: Path::from(
734+
"bucket/mytable/day=__HIVE_DEFAULT_PARTITION__/data.parquet",
735+
),
736+
last_modified: chrono::Utc::now(),
737+
size: 100,
738+
e_tag: None,
739+
version: None,
740+
};
741+
742+
let pf = try_into_partitioned_file(
743+
meta,
744+
&partition_cols,
745+
&table_path,
746+
HIVE_DEFAULT_PARTITION_NAME,
747+
)
748+
.unwrap()
749+
.expect("file should be listed");
750+
751+
assert_eq!(pf.partition_values, vec![ScalarValue::Date32(None)]);
752+
}
753+
754+
#[test]
755+
fn test_try_into_partitioned_file_null_partition_name_not_matched() {
756+
let table_path = ListingTableUrl::parse("file:///bucket/mytable").unwrap();
757+
let partition_cols = vec![("category".to_string(), DataType::Utf8)];
758+
let meta = ObjectMeta {
759+
location: Path::from(
760+
"bucket/mytable/category=__HIVE_DEFAULT_PARTITION__/data.parquet",
761+
),
762+
last_modified: chrono::Utc::now(),
763+
size: 100,
764+
e_tag: None,
765+
version: None,
766+
};
767+
768+
// The empty name turns the mapping off, leaving the directory name as the value.
769+
let pf = try_into_partitioned_file(meta, &partition_cols, &table_path, "")
770+
.unwrap()
771+
.expect("file should be listed");
772+
773+
assert_eq!(
774+
pf.partition_values,
775+
vec![ScalarValue::Utf8(Some(
776+
"__HIVE_DEFAULT_PARTITION__".to_string()
777+
))]
778+
);
779+
}
780+
699781
#[test]
700782
fn test_try_into_partitioned_file_root_file_skipped() {
701783
// File in root directory (not inside any partition path) should be
@@ -711,8 +793,13 @@ mod tests {
711793
version: None,
712794
};
713795

714-
let result =
715-
try_into_partitioned_file(meta, &partition_cols, &table_path).unwrap();
796+
let result = try_into_partitioned_file(
797+
meta,
798+
&partition_cols,
799+
&table_path,
800+
HIVE_DEFAULT_PARTITION_NAME,
801+
)
802+
.unwrap();
716803
assert!(
717804
result.is_none(),
718805
"Files outside partition structure should be skipped"
@@ -732,8 +819,13 @@ mod tests {
732819
version: None,
733820
};
734821

735-
let result =
736-
try_into_partitioned_file(meta, &partition_cols, &table_path).unwrap();
822+
let result = try_into_partitioned_file(
823+
meta,
824+
&partition_cols,
825+
&table_path,
826+
HIVE_DEFAULT_PARTITION_NAME,
827+
)
828+
.unwrap();
737829
assert!(
738830
result.is_none(),
739831
"Files with wrong partition column name should be skipped"
@@ -755,8 +847,13 @@ mod tests {
755847
version: None,
756848
};
757849

758-
let result =
759-
try_into_partitioned_file(meta, &partition_cols, &table_path).unwrap();
850+
let result = try_into_partitioned_file(
851+
meta,
852+
&partition_cols,
853+
&table_path,
854+
HIVE_DEFAULT_PARTITION_NAME,
855+
)
856+
.unwrap();
760857
assert!(result.is_some());
761858
let pf = result.unwrap();
762859
assert_eq!(pf.partition_values.len(), 2);
@@ -786,8 +883,13 @@ mod tests {
786883
version: None,
787884
};
788885

789-
let result =
790-
try_into_partitioned_file(meta, &partition_cols, &table_path).unwrap();
886+
let result = try_into_partitioned_file(
887+
meta,
888+
&partition_cols,
889+
&table_path,
890+
HIVE_DEFAULT_PARTITION_NAME,
891+
)
892+
.unwrap();
791893
// File has year=2024 but no month= directory — parse_partitions_for_path
792894
// returns None because the path component "data.parquet" doesn't match
793895
// the expected "month=..." pattern.

‎datafusion/catalog-listing/src/table.rs‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -241,10 +241,12 @@ impl ListingTable {
241241
file_schema
242242
};
243243

244-
// Add the partition columns to the file schema
244+
// Add the partition columns to the file schema. They are nullable because a Hive
245+
// compliant writer spells a null partition value with a placeholder directory name
246+
// (see `ExecutionOptions::listing_table_hive_default_partition_name`).
245247
let mut builder = SchemaBuilder::from(file_schema.as_ref().to_owned());
246248
for (part_col_name, part_col_type) in &options.table_partition_cols {
247-
builder.push(Field::new(part_col_name, part_col_type.clone(), false));
249+
builder.push(Field::new(part_col_name, part_col_type.clone(), true));
248250
}
249251

250252
let table_schema = Arc::new(
@@ -360,7 +362,8 @@ impl ListingTable {
360362
self.options
361363
.table_partition_cols
362364
.iter()
363-
.map(|(col, field)| Arc::new(Field::new(col, field.clone(), false)))
365+
// Nullable to match the table schema built in `try_new`.
366+
.map(|(col, field)| Arc::new(Field::new(col, field.clone(), true)))
364367
.collect::<Vec<_>>(),
365368
)
366369
.build();
@@ -1120,7 +1123,7 @@ impl ListingTable {
11201123

11211124
// Note: Statistics already include both file columns and partition columns.
11221125
// PartitionedFile::with_statistics automatically appends exact partition column
1123-
// statistics (min=max=partition_value, null_count=0, distinct_count=1) computed
1126+
// statistics (e.g. min=max=partition_value, null_count=0, distinct_count=1) computed
11241127
// from partition_values.
11251128
Ok(ListFilesResult {
11261129
file_groups,

‎datafusion/common/src/config.rs‎

Lines changed: 26 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ use crate::{DataFusionError, Result};
2020
#[cfg(feature = "parquet_encryption")]
2121
use hex;
2222
use std::any::Any;
23+
use std::borrow::Cow;
2324
use std::collections::{BTreeMap, HashMap};
2425
use std::error::Error;
2526
use std::fmt::{self, Display};
@@ -886,6 +887,10 @@ impl From<SpillCompression> for Option<CompressionType> {
886887
}
887888
}
888889

890+
/// The directory name Hive compliant writers use in place of a null partition value, and the
891+
/// default for [`ExecutionOptions::listing_table_hive_default_partition_name`].
892+
pub const HIVE_DEFAULT_PARTITION_NAME: &str = "__HIVE_DEFAULT_PARTITION__";
893+
889894
config_namespace! {
890895
/// Options related to query execution
891896
///
@@ -1095,6 +1100,14 @@ config_namespace! {
10951100
/// inferred and will be represented in the table schema).
10961101
pub listing_table_factory_infer_partitions: bool, default = true
10971102

1103+
/// The directory name Hive compliant writers use in place of a null partition
1104+
/// value, e.g. `/table/date=__HIVE_DEFAULT_PARTITION__/data.parquet`. Partition
1105+
/// values equal to this name are read as NULL, as Hive, Spark and Impala do, and
1106+
/// partitioned writes reject them. Corresponds to Hive's
1107+
/// `hive.exec.default.partition.name`. Set this to the empty string to read and
1108+
/// write such values as the literal directory name instead.
1109+
pub listing_table_hive_default_partition_name: String, default = HIVE_DEFAULT_PARTITION_NAME.to_string()
1110+
10981111
/// Should DataFusion support recursive CTEs
10991112
pub enable_recursive_ctes: bool, default = true
11001113

@@ -2516,14 +2529,26 @@ impl ConfigOptions {
25162529
&mut docs,
25172530
"| {} | {} | {} |",
25182531
entry.key,
2519-
entry.value.as_deref().unwrap_or("NULL"),
2532+
display_default(entry.value.as_deref()),
25202533
entry.description
25212534
);
25222535
}
25232536
docs
25242537
}
25252538
}
25262539

2540+
/// Renders a default value for the generated documentation table.
2541+
///
2542+
/// Values containing `_` or `*` are shown as code, since markdown would otherwise read them
2543+
/// as formatting: a default of `__HIVE_DEFAULT_PARTITION__` renders as bold text.
2544+
fn display_default(value: Option<&str>) -> Cow<'_, str> {
2545+
match value {
2546+
None => Cow::Borrowed("NULL"),
2547+
Some(value) if value.contains(['_', '*']) => Cow::Owned(format!("`{value}`")),
2548+
Some(value) => Cow::Borrowed(value),
2549+
}
2550+
}
2551+
25272552
/// [`ConfigExtension`] provides a mechanism to store third-party configuration
25282553
/// within DataFusion [`ConfigOptions`]
25292554
///

0 commit comments

Comments
 (0)