Skip to content

Commit 05f79e3

Browse files
committed
fix: read Hive's null partition placeholder as NULL
Hive compliant writers put the rows whose partition value is null into a directory named `__HIVE_DEFAULT_PARTITION__`. `ListingTable` cast that name straight to the partition column's type, so a table with a null partition could not be queried at all unless the column happened to be a string: Cannot cast string '__HIVE_DEFAULT_PARTITION__' to value of Date32 type Read such values as a typed NULL instead, as Hive, Spark and Impala do, and declare partition columns nullable so that the NULL can be represented and `IS NULL` selects the null partition. The placeholder name follows the new `datafusion.execution.listing_table_hive_default_partition_name` option, so tables written with a different `hive.exec.default.partition.name` can be read; the empty name turns the mapping off. Closes apache#18083
1 parent 0d79252 commit 05f79e3

8 files changed

Lines changed: 495 additions & 184 deletions

File tree

‎datafusion/catalog-listing/src/helpers.rs‎

Lines changed: 123 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ use datafusion_datasource::{FileExtensions, ListingTableUrl};
2929
use datafusion_expr::{BinaryExpr, Operator, lit, utils};
3030

3131
use arrow::{
32-
array::AsArray,
32+
array::{Array, AsArray},
3333
datatypes::{DataType, Field, Metadata},
3434
record_batch::RecordBatch,
3535
};
@@ -337,23 +337,30 @@ pub fn filter_partitioned_file(
337337
)?;
338338

339339
// Since we're only operating on a single file, our batch and resulting "array" holds only one
340-
// value indicating if the input file matches the provided filters
340+
// value indicating if the input file matches the provided filters. A NULL is not a match:
341+
// a null partition value makes that reachable, as `part = 1` is NULL for a null partition.
341342
let matches = expr.evaluate(&batch)?.into_array(1)?;
342-
if matches.as_boolean().value(0) {
343+
let matches = matches.as_boolean();
344+
if matches.is_valid(0) && matches.value(0) {
343345
return Ok(Some(pf));
344346
}
345347

346348
Ok(None)
347349
}
348350

349351
/// Returns `Ok(None)` when the file is not inside a valid partition path
350-
/// (e.g. a stale file in the table root directory). Such files are skipped
351-
/// because hive-style partition values are never null and there is no valid
352-
/// value to assign for non-partitioned files.
352+
/// (e.g. a stale file in the table root directory), as there is no value to
353+
/// assign for non-partitioned files.
354+
///
355+
/// A partition directory named `null_partition_name` is read as NULL; see
356+
/// [`ExecutionOptions::listing_table_hive_default_partition_name`].
357+
///
358+
/// [`ExecutionOptions::listing_table_hive_default_partition_name`]: datafusion_common::config::ExecutionOptions::listing_table_hive_default_partition_name
353359
fn try_into_partitioned_file(
354360
object_meta: ObjectMeta,
355361
partition_cols: &[(String, DataType)],
356362
table_path: &ListingTableUrl,
363+
null_partition_name: &str,
357364
) -> Result<Option<PartitionedFile>> {
358365
let cols = partition_cols.iter().map(|(name, _)| name.as_str());
359366
let parsed = parse_partitions_for_path(table_path, &object_meta.location, cols);
@@ -367,6 +374,11 @@ fn try_into_partitioned_file(
367374
.into_iter()
368375
.zip(partition_cols)
369376
.map(|(parsed, (_, datatype))| {
377+
// Hive compliant writers spell a null partition value with a placeholder
378+
// directory name, so there is nothing to parse: the value is NULL.
379+
if !null_partition_name.is_empty() && parsed == null_partition_name {
380+
return ScalarValue::try_new_null(datatype);
381+
}
370382
ScalarValue::try_from_string(parsed.into_owned(), datatype)
371383
})
372384
.collect::<Result<Vec<_>>>()?;
@@ -426,12 +438,19 @@ pub async fn pruned_partition_list<'a>(
426438
Metadata::new(),
427439
)?;
428440

441+
let null_partition_name = ctx
442+
.config_options()
443+
.execution
444+
.listing_table_hive_default_partition_name
445+
.as_str();
446+
429447
Ok(objects
430-
.try_filter_map(|object_meta| {
448+
.try_filter_map(move |object_meta| {
431449
futures::future::ready(try_into_partitioned_file(
432450
object_meta,
433451
partition_cols,
434452
table_path,
453+
null_partition_name,
435454
))
436455
})
437456
.try_filter_map(move |pf| {
@@ -508,6 +527,7 @@ pub fn describe_partition(partition: &Partition) -> (&str, usize, Vec<&str>) {
508527

509528
#[cfg(test)]
510529
mod tests {
530+
use datafusion_common::config::HIVE_DEFAULT_PARTITION_NAME;
511531
use datafusion_datasource::file_groups::FileGroup;
512532
use std::ops::Not;
513533

@@ -672,8 +692,13 @@ mod tests {
672692
version: None,
673693
};
674694

675-
let result =
676-
try_into_partitioned_file(meta, &partition_cols, &table_path).unwrap();
695+
let result = try_into_partitioned_file(
696+
meta,
697+
&partition_cols,
698+
&table_path,
699+
HIVE_DEFAULT_PARTITION_NAME,
700+
)
701+
.unwrap();
677702
assert!(result.is_some());
678703
let pf = result.unwrap();
679704
assert_eq!(pf.partition_values.len(), 1);
@@ -698,8 +723,13 @@ mod tests {
698723
version: None,
699724
};
700725

701-
let result =
702-
try_into_partitioned_file(meta, &partition_cols, &table_path).unwrap();
726+
let result = try_into_partitioned_file(
727+
meta,
728+
&partition_cols,
729+
&table_path,
730+
HIVE_DEFAULT_PARTITION_NAME,
731+
)
732+
.unwrap();
703733
assert!(result.is_some());
704734
let pf = result.unwrap();
705735
assert_eq!(pf.partition_values.len(), 1);
@@ -709,6 +739,60 @@ mod tests {
709739
);
710740
}
711741

742+
#[test]
743+
fn test_try_into_partitioned_file_null_partition() {
744+
let table_path = ListingTableUrl::parse("file:///bucket/mytable").unwrap();
745+
// A type the placeholder cannot be parsed as, which is how this used to fail.
746+
let partition_cols = vec![("day".to_string(), DataType::Date32)];
747+
let meta = ObjectMeta {
748+
location: Path::from(
749+
"bucket/mytable/day=__HIVE_DEFAULT_PARTITION__/data.parquet",
750+
),
751+
last_modified: chrono::Utc::now(),
752+
size: 100,
753+
e_tag: None,
754+
version: None,
755+
};
756+
757+
let pf = try_into_partitioned_file(
758+
meta,
759+
&partition_cols,
760+
&table_path,
761+
HIVE_DEFAULT_PARTITION_NAME,
762+
)
763+
.unwrap()
764+
.expect("file should be listed");
765+
766+
assert_eq!(pf.partition_values, vec![ScalarValue::Date32(None)]);
767+
}
768+
769+
#[test]
770+
fn test_try_into_partitioned_file_null_partition_name_not_matched() {
771+
let table_path = ListingTableUrl::parse("file:///bucket/mytable").unwrap();
772+
let partition_cols = vec![("category".to_string(), DataType::Utf8)];
773+
let meta = ObjectMeta {
774+
location: Path::from(
775+
"bucket/mytable/category=__HIVE_DEFAULT_PARTITION__/data.parquet",
776+
),
777+
last_modified: chrono::Utc::now(),
778+
size: 100,
779+
e_tag: None,
780+
version: None,
781+
};
782+
783+
// The empty name turns the mapping off, leaving the directory name as the value.
784+
let pf = try_into_partitioned_file(meta, &partition_cols, &table_path, "")
785+
.unwrap()
786+
.expect("file should be listed");
787+
788+
assert_eq!(
789+
pf.partition_values,
790+
vec![ScalarValue::Utf8(Some(
791+
"__HIVE_DEFAULT_PARTITION__".to_string()
792+
))]
793+
);
794+
}
795+
712796
#[test]
713797
fn test_try_into_partitioned_file_root_file_skipped() {
714798
// File in root directory (not inside any partition path) should be
@@ -724,8 +808,13 @@ mod tests {
724808
version: None,
725809
};
726810

727-
let result =
728-
try_into_partitioned_file(meta, &partition_cols, &table_path).unwrap();
811+
let result = try_into_partitioned_file(
812+
meta,
813+
&partition_cols,
814+
&table_path,
815+
HIVE_DEFAULT_PARTITION_NAME,
816+
)
817+
.unwrap();
729818
assert!(
730819
result.is_none(),
731820
"Files outside partition structure should be skipped"
@@ -745,8 +834,13 @@ mod tests {
745834
version: None,
746835
};
747836

748-
let result =
749-
try_into_partitioned_file(meta, &partition_cols, &table_path).unwrap();
837+
let result = try_into_partitioned_file(
838+
meta,
839+
&partition_cols,
840+
&table_path,
841+
HIVE_DEFAULT_PARTITION_NAME,
842+
)
843+
.unwrap();
750844
assert!(
751845
result.is_none(),
752846
"Files with wrong partition column name should be skipped"
@@ -768,8 +862,13 @@ mod tests {
768862
version: None,
769863
};
770864

771-
let result =
772-
try_into_partitioned_file(meta, &partition_cols, &table_path).unwrap();
865+
let result = try_into_partitioned_file(
866+
meta,
867+
&partition_cols,
868+
&table_path,
869+
HIVE_DEFAULT_PARTITION_NAME,
870+
)
871+
.unwrap();
773872
assert!(result.is_some());
774873
let pf = result.unwrap();
775874
assert_eq!(pf.partition_values.len(), 2);
@@ -799,8 +898,13 @@ mod tests {
799898
version: None,
800899
};
801900

802-
let result =
803-
try_into_partitioned_file(meta, &partition_cols, &table_path).unwrap();
901+
let result = try_into_partitioned_file(
902+
meta,
903+
&partition_cols,
904+
&table_path,
905+
HIVE_DEFAULT_PARTITION_NAME,
906+
)
907+
.unwrap();
804908
// File has year=2024 but no month= directory — parse_partitions_for_path
805909
// returns None because the path component "data.parquet" doesn't match
806910
// the expected "month=..." pattern.

‎datafusion/catalog-listing/src/table.rs‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -252,10 +252,12 @@ impl ListingTable {
252252
file_schema
253253
};
254254

255-
// Add the partition columns to the file schema
255+
// Add the partition columns to the file schema. They are nullable because a Hive
256+
// compliant writer spells a null partition value with a placeholder directory name
257+
// (see `ExecutionOptions::listing_table_hive_default_partition_name`).
256258
let mut builder = SchemaBuilder::from(file_schema.as_ref().to_owned());
257259
for (part_col_name, part_col_type) in &options.table_partition_cols {
258-
builder.push(Field::new(part_col_name, part_col_type.clone(), false));
260+
builder.push(Field::new(part_col_name, part_col_type.clone(), true));
259261
}
260262

261263
let table_schema = Arc::new(
@@ -371,7 +373,8 @@ impl ListingTable {
371373
self.options
372374
.table_partition_cols
373375
.iter()
374-
.map(|(col, field)| Arc::new(Field::new(col, field.clone(), false)))
376+
// Nullable to match the table schema built in `try_new`.
377+
.map(|(col, field)| Arc::new(Field::new(col, field.clone(), true)))
375378
.collect::<Vec<_>>(),
376379
)
377380
.build();

‎datafusion/common/src/config.rs‎

Lines changed: 25 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@ use crate::{DataFusionError, Result};
3333
#[cfg(feature = "parquet_encryption")]
3434
use hex;
3535
use std::any::Any;
36+
use std::borrow::Cow;
3637
use std::collections::{BTreeMap, HashMap};
3738
use std::error::Error;
3839
use std::fmt::{self, Display};
@@ -894,6 +895,10 @@ impl From<SpillCompression> for Option<CompressionType> {
894895
}
895896
}
896897

898+
/// The directory name Hive compliant writers use in place of a null partition value, and the
899+
/// default for [`ExecutionOptions::listing_table_hive_default_partition_name`].
900+
pub const HIVE_DEFAULT_PARTITION_NAME: &str = "__HIVE_DEFAULT_PARTITION__";
901+
897902
config_namespace! {
898903
/// Options related to query execution
899904
///
@@ -1126,6 +1131,13 @@ config_namespace! {
11261131
/// inferred and will be represented in the table schema).
11271132
pub listing_table_factory_infer_partitions: bool, default = true
11281133

1134+
/// The directory name Hive compliant writers use in place of a null partition
1135+
/// value, e.g. `/table/date=__HIVE_DEFAULT_PARTITION__/data.parquet`. Partition
1136+
/// values equal to this name are read as NULL, as Hive, Spark and Impala do.
1137+
/// Corresponds to Hive's `hive.exec.default.partition.name`. Set this to the empty
1138+
/// string to read such values as the literal directory name instead.
1139+
pub listing_table_hive_default_partition_name: String, default = HIVE_DEFAULT_PARTITION_NAME.to_string()
1140+
11291141
/// Should DataFusion support recursive CTEs
11301142
pub enable_recursive_ctes: bool, default = true
11311143

@@ -2489,14 +2501,26 @@ impl ConfigOptions {
24892501
&mut docs,
24902502
"| {} | {} | {} |",
24912503
entry.key,
2492-
entry.value.as_deref().unwrap_or("NULL"),
2504+
display_default(entry.value.as_deref()),
24932505
entry.description
24942506
);
24952507
}
24962508
docs
24972509
}
24982510
}
24992511

2512+
/// Renders a default value for the generated documentation table.
2513+
///
2514+
/// Values containing `_` or `*` are shown as code, since markdown would otherwise read them
2515+
/// as formatting: a default of `__HIVE_DEFAULT_PARTITION__` renders as bold text.
2516+
fn display_default(value: Option<&str>) -> Cow<'_, str> {
2517+
match value {
2518+
None => Cow::Borrowed("NULL"),
2519+
Some(value) if value.contains(['_', '*']) => Cow::Owned(format!("`{value}`")),
2520+
Some(value) => Cow::Borrowed(value),
2521+
}
2522+
}
2523+
25002524
/// [`ConfigExtension`] provides a mechanism to store third-party configuration
25012525
/// within DataFusion [`ConfigOptions`]
25022526
///

0 commit comments

Comments
 (0)