Skip to content

Commit 31cb93f

Browse files
[branch-54] fix: read Hive's null partition placeholder as NULL (backport #26310) (#27308)
## Which issue does this PR close? - Backport of #26310 to `branch-54` - Part of #23805 ## Rationale for this change Hive writers use `__HIVE_DEFAULT_PARTITION__` for NULL partition values. `ListingTable` currently parses this marker as the partition column's type, causing cast errors for non-string columns and returning the literal marker for string columns. `IS NULL` consequently misses those rows. ## What changes are included in this PR? - Read the Hive marker as a typed NULL and declare partition columns nullable. - Treat NULL partition-predicate results as non-matches and account for NULL partition values in file statistics. - Reject partitioned writes of the literal marker, which would otherwise read back as NULL. This backport uses a fixed marker, without the new execution option or public constant. Ordering tests reflect branch-54's existing explicit-sort fallback. ## Are these changes tested? Yes ## Are there any user-facing changes? Yes. Hive marker directories yield NULL values, and partition columns are nullable. Writing the literal marker returns an error. The marker cannot be configured or disabled in this backport. AI assistance: Codex. --------- Co-authored-by: Xander <[email protected]>
1 parent 483a48b commit 31cb93f

9 files changed

Lines changed: 456 additions & 23 deletions

File tree

‎datafusion/catalog-listing/src/helpers.rs‎

Lines changed: 59 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ use datafusion_datasource::{FileExtensions, ListingTableUrl};
2929
use datafusion_expr::{BinaryExpr, Operator, lit, utils};
3030

3131
use arrow::{
32-
array::AsArray,
32+
array::{Array, AsArray},
3333
datatypes::{DataType, Field},
3434
record_batch::RecordBatch,
3535
};
@@ -45,6 +45,8 @@ use datafusion_physical_expr::create_physical_expr;
4545
use object_store::path::Path;
4646
use object_store::{ObjectMeta, ObjectStore};
4747

48+
const HIVE_DEFAULT_PARTITION_NAME: &str = "__HIVE_DEFAULT_PARTITION__";
49+
4850
/// Check whether the given expression can be resolved using only the columns `col_names`.
4951
/// This means that if this function returns true:
5052
/// - the table provider can filter the table partition values with this expression
@@ -349,19 +351,22 @@ fn filter_partitions(
349351
let expr = create_physical_expr(&filter, df_schema, &props)?;
350352

351353
// Since we're only operating on a single file, our batch and resulting "array" holds only one
352-
// value indicating if the input file matches the provided filters
354+
// value indicating if the input file matches the provided filters. A NULL is not a match:
355+
// a null partition value makes that reachable, as `part = 1` is NULL for a null partition.
353356
let matches = expr.evaluate(&batch)?.into_array(1)?;
354-
if matches.as_boolean().value(0) {
357+
let matches = matches.as_boolean();
358+
if matches.is_valid(0) && matches.value(0) {
355359
return Ok(Some(pf));
356360
}
357361

358362
Ok(None)
359363
}
360364

361365
/// Returns `Ok(None)` when the file is not inside a valid partition path
362-
/// (e.g. a stale file in the table root directory). Such files are skipped
363-
/// because hive-style partition values are never null and there is no valid
364-
/// value to assign for non-partitioned files.
366+
/// (e.g. a stale file in the table root directory), as there is no value to
367+
/// assign for non-partitioned files.
368+
///
369+
/// A partition value named `__HIVE_DEFAULT_PARTITION__` is read as NULL.
365370
fn try_into_partitioned_file(
366371
object_meta: ObjectMeta,
367372
partition_cols: &[(String, DataType)],
@@ -379,6 +384,9 @@ fn try_into_partitioned_file(
379384
.into_iter()
380385
.zip(partition_cols)
381386
.map(|(parsed, (_, datatype))| {
387+
if parsed == HIVE_DEFAULT_PARTITION_NAME {
388+
return ScalarValue::try_new_null(datatype);
389+
}
382390
ScalarValue::try_from_string(parsed.to_string(), datatype)
383391
})
384392
.collect::<Result<Vec<_>>>()?;
@@ -439,7 +447,7 @@ pub async fn pruned_partition_list<'a>(
439447
)?;
440448

441449
Ok(objects
442-
.try_filter_map(|object_meta| {
450+
.try_filter_map(move |object_meta| {
443451
futures::future::ready(try_into_partitioned_file(
444452
object_meta,
445453
partition_cols,
@@ -649,6 +657,50 @@ mod tests {
649657
);
650658
}
651659

660+
#[test]
661+
fn test_try_into_partitioned_file_null_partition() {
662+
let table_path = ListingTableUrl::parse("file:///bucket/mytable").unwrap();
663+
// A type the placeholder cannot be parsed as, which is how this used to fail.
664+
let partition_cols = vec![("day".to_string(), DataType::Date32)];
665+
let meta = ObjectMeta {
666+
location: Path::from(
667+
"bucket/mytable/day=__HIVE_DEFAULT_PARTITION__/data.parquet",
668+
),
669+
last_modified: chrono::Utc::now(),
670+
size: 100,
671+
e_tag: None,
672+
version: None,
673+
};
674+
675+
let pf = try_into_partitioned_file(meta, &partition_cols, &table_path)
676+
.unwrap()
677+
.expect("file should be listed");
678+
679+
assert_eq!(pf.partition_values, vec![ScalarValue::Date32(None)]);
680+
}
681+
682+
#[test]
683+
fn test_try_into_partitioned_file_other_partition_name_is_literal() {
684+
let table_path = ListingTableUrl::parse("file:///bucket/mytable").unwrap();
685+
let partition_cols = vec![("category".to_string(), DataType::Utf8)];
686+
let meta = ObjectMeta {
687+
location: Path::from("bucket/mytable/category=MY_NULL/data.parquet"),
688+
last_modified: chrono::Utc::now(),
689+
size: 100,
690+
e_tag: None,
691+
version: None,
692+
};
693+
694+
let pf = try_into_partitioned_file(meta, &partition_cols, &table_path)
695+
.unwrap()
696+
.expect("file should be listed");
697+
698+
assert_eq!(
699+
pf.partition_values,
700+
vec![ScalarValue::Utf8(Some("MY_NULL".to_string()))]
701+
);
702+
}
703+
652704
#[test]
653705
fn test_try_into_partitioned_file_root_file_skipped() {
654706
// File in root directory (not inside any partition path) should be

‎datafusion/catalog-listing/src/table.rs‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -211,10 +211,11 @@ impl ListingTable {
211211
.options
212212
.ok_or_else(|| internal_datafusion_err!("No ListingOptions provided"))?;
213213

214-
// Add the partition columns to the file schema
214+
// Add the partition columns to the file schema. They are nullable because a Hive
215+
// compliant writer uses `__HIVE_DEFAULT_PARTITION__` for a null partition value.
215216
let mut builder = SchemaBuilder::from(file_schema.as_ref().to_owned());
216217
for (part_col_name, part_col_type) in &options.table_partition_cols {
217-
builder.push(Field::new(part_col_name, part_col_type.clone(), false));
218+
builder.push(Field::new(part_col_name, part_col_type.clone(), true));
218219
}
219220

220221
let table_schema = Arc::new(
@@ -341,7 +342,8 @@ impl ListingTable {
341342
self.options
342343
.table_partition_cols
343344
.iter()
344-
.map(|(col, field)| Arc::new(Field::new(col, field.clone(), false)))
345+
// Nullable to match the table schema built in `try_new`.
346+
.map(|(col, field)| Arc::new(Field::new(col, field.clone(), true)))
345347
.collect(),
346348
);
347349

@@ -792,7 +794,7 @@ impl ListingTable {
792794

793795
// Note: Statistics already include both file columns and partition columns.
794796
// PartitionedFile::with_statistics automatically appends exact partition column
795-
// statistics (min=max=partition_value, null_count=0, distinct_count=1) computed
797+
// statistics (e.g. min=max=partition_value, null_count=0, distinct_count=1) computed
796798
// from partition_values.
797799
Ok(ListFilesResult {
798800
file_groups,

‎datafusion/core/tests/catalog_listing/pruned_partition_list.rs‎

Lines changed: 105 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -233,6 +233,111 @@ async fn test_list_partition() {
233233
);
234234
}
235235

236+
/// A partition directory named `__HIVE_DEFAULT_PARTITION__` holds the rows whose partition
237+
/// value is null, so it is read as NULL rather than parsed as the column's type.
238+
#[tokio::test]
239+
async fn test_pruned_partition_list_null_partition() {
240+
let (store, state) = make_test_store_and_state(&[
241+
("tablepath/mypartition=3/file.parquet", 100),
242+
(
243+
"tablepath/mypartition=__HIVE_DEFAULT_PARTITION__/file.parquet",
244+
100,
245+
),
246+
]);
247+
248+
let pruned = pruned_partition_list(
249+
state.as_ref(),
250+
store.as_ref(),
251+
&ListingTableUrl::parse("file:///tablepath/").unwrap(),
252+
&[],
253+
".parquet",
254+
&[(String::from("mypartition"), DataType::Int32)],
255+
)
256+
.await
257+
.expect("partition pruning failed")
258+
.try_collect::<Vec<_>>()
259+
.await
260+
.unwrap();
261+
262+
assert_eq!(pruned.len(), 2);
263+
assert_eq!(&pruned[0].partition_values, &[ScalarValue::Int32(Some(3))]);
264+
assert_eq!(&pruned[1].partition_values, &[ScalarValue::Int32(None)]);
265+
}
266+
267+
#[tokio::test]
268+
async fn test_pruned_partition_list_prunes_null_partition() {
269+
let files: &[(&str, u64)] = &[
270+
("tablepath/mypartition=3/file.parquet", 100),
271+
(
272+
"tablepath/mypartition=__HIVE_DEFAULT_PARTITION__/file.parquet",
273+
100,
274+
),
275+
];
276+
let partition_cols = &[(String::from("mypartition"), DataType::Int32)];
277+
278+
let pruned_with = |filter: Expr| async move {
279+
let (store, state) = make_test_store_and_state(files);
280+
pruned_partition_list(
281+
state.as_ref(),
282+
store.as_ref(),
283+
&ListingTableUrl::parse("file:///tablepath/").unwrap(),
284+
&[filter],
285+
".parquet",
286+
partition_cols,
287+
)
288+
.await
289+
.expect("partition pruning failed")
290+
.try_collect::<Vec<_>>()
291+
.await
292+
.unwrap()
293+
};
294+
295+
// The null partition does not match an equality filter: `mypartition = 3` is NULL there.
296+
let pruned = pruned_with(Expr::eq(col("mypartition"), lit(3i32))).await;
297+
assert_eq!(pruned.len(), 1);
298+
assert_eq!(
299+
pruned[0].object_meta.location.as_ref(),
300+
"tablepath/mypartition=3/file.parquet"
301+
);
302+
303+
// ... and only the null partition matches `IS NULL`.
304+
let pruned = pruned_with(col("mypartition").is_null()).await;
305+
assert_eq!(pruned.len(), 1);
306+
assert_eq!(
307+
pruned[0].object_meta.location.as_ref(),
308+
"tablepath/mypartition=__HIVE_DEFAULT_PARTITION__/file.parquet"
309+
);
310+
}
311+
312+
#[tokio::test]
313+
async fn test_pruned_partition_list_hive_null_partition_name() {
314+
let (store, state) = make_test_store_and_state(&[
315+
("tablepath/mypartition=MY_NULL/file.parquet", 100),
316+
(
317+
"tablepath/mypartition=__HIVE_DEFAULT_PARTITION__/file.parquet",
318+
100,
319+
),
320+
]);
321+
let pruned = pruned_partition_list(
322+
state.as_ref(),
323+
store.as_ref(),
324+
&ListingTableUrl::parse("file:///tablepath/").unwrap(),
325+
&[],
326+
".parquet",
327+
&[(String::from("mypartition"), DataType::Utf8)],
328+
)
329+
.await
330+
.expect("partition pruning failed")
331+
.try_collect::<Vec<_>>()
332+
.await
333+
.unwrap();
334+
335+
// Only the fixed Hive marker is read as NULL; other names remain literal values.
336+
assert_eq!(pruned.len(), 2);
337+
assert_eq!(&pruned[0].partition_values, &[ScalarValue::from("MY_NULL")]);
338+
assert_eq!(&pruned[1].partition_values, &[ScalarValue::Utf8(None)]);
339+
}
340+
236341
pub fn make_test_store_and_state(
237342
files: &[(&str, u64)],
238343
) -> (Arc<InMemory>, Arc<dyn Session>) {

‎datafusion/datasource/src/file_groups.rs‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -371,9 +371,11 @@ impl FileGroupPartitioner {
371371
/// in the group for the **full table schema** (file columns + partition columns).
372372
///
373373
/// Partition column statistics are derived from the individual file partition values:
374-
/// - `min` = minimum partition value across all files in the group
375-
/// - `max` = maximum partition value across all files in the group
376-
/// - `null_count` = 0 (partition values are never null)
374+
/// - `min` = minimum partition value across all files in the group, or absent if any
375+
/// file in the group has a null partition value (see #26748)
376+
/// - `max` = maximum partition value across all files in the group, or absent if any
377+
/// file in the group has a null partition value
378+
/// - `null_count` = the number of rows in files with a null partition value
377379
///
378380
/// This allows query optimizers to prune entire file groups based on partition bounds.
379381
#[derive(Debug, Clone)]

‎datafusion/datasource/src/mod.rs‎

Lines changed: 44 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -117,9 +117,12 @@ impl FileRange {
117117
/// computed from [`Self::partition_values`]:
118118
///
119119
/// - `min = max = partition_value` (all rows in a file share the same partition value)
120-
/// - `null_count = 0` (partition values extracted from paths are never null)
120+
/// - `null_count = 0` (every row has the file's partition value)
121121
/// - `distinct_count = 1` (single distinct value per file for each partition column)
122122
///
123+
/// A null partition value instead gives `null_count = num_rows`, `distinct_count = 0`
124+
/// and absent `min`/`max`.
125+
///
123126
/// This enables query optimizers to use partition column bounds for pruning and planning.
124127
pub struct PartitionedFile {
125128
/// Path for the file (e.g. URL, filesystem path, etc)
@@ -316,8 +319,11 @@ impl PartitionedFile {
316319
/// This method will automatically append exact statistics for partition columns
317320
/// based on `partition_values`:
318321
/// - `min = max = partition_value` (all rows have the same value)
319-
/// - `null_count = 0` (partition values from paths are never null)
322+
/// - `null_count = 0` (all rows have a non-null value)
320323
/// - `distinct_count = 1` (all rows have the same partition value)
324+
///
325+
/// A null partition value (e.g. a Hive default partition) instead gives
326+
/// `null_count = num_rows`, `distinct_count = 0` and absent `min`/`max`.
321327
pub fn with_statistics(mut self, file_statistics: Arc<Statistics>) -> Self {
322328
if self.partition_values.is_empty() {
323329
// No partition columns, use stats as-is
@@ -326,11 +332,20 @@ impl PartitionedFile {
326332
// Extend stats with exact partition column statistics
327333
let mut stats = Arc::unwrap_or_clone(file_statistics);
328334
for partition_value in &self.partition_values {
335+
let (null_count, min_max, distinct_count) = if partition_value.is_null() {
336+
(stats.num_rows, Precision::Absent, Precision::Exact(0))
337+
} else {
338+
(
339+
Precision::Exact(0),
340+
Precision::Exact(partition_value.clone()),
341+
Precision::Exact(1),
342+
)
343+
};
329344
let col_stats = ColumnStatistics {
330-
null_count: Precision::Exact(0),
331-
max_value: Precision::Exact(partition_value.clone()),
332-
min_value: Precision::Exact(partition_value.clone()),
333-
distinct_count: Precision::Exact(1),
345+
null_count,
346+
max_value: min_max.clone(),
347+
min_value: min_max,
348+
distinct_count,
334349
sum_value: Precision::Absent,
335350
byte_size: partition_value
336351
.data_type()
@@ -767,6 +782,29 @@ mod tests {
767782
);
768783
}
769784

785+
#[test]
786+
fn test_with_statistics_null_partition_value() {
787+
use crate::PartitionedFile;
788+
use datafusion_common::stats::Precision;
789+
use datafusion_common::{ScalarValue, Statistics};
790+
791+
let mut pf = PartitionedFile::new("test.parquet", 100);
792+
pf.partition_values = vec![ScalarValue::Date32(None)];
793+
794+
let file_stats = Arc::new(Statistics {
795+
num_rows: Precision::Exact(2),
796+
total_byte_size: Precision::Exact(16),
797+
column_statistics: vec![],
798+
});
799+
800+
let stats = pf.with_statistics(file_stats).statistics.unwrap();
801+
let partition_col_stats = &stats.column_statistics[0];
802+
assert_eq!(partition_col_stats.null_count, Precision::Exact(2));
803+
assert_eq!(partition_col_stats.min_value, Precision::Absent);
804+
assert_eq!(partition_col_stats.max_value, Precision::Absent);
805+
assert_eq!(partition_col_stats.distinct_count, Precision::Exact(0));
806+
}
807+
770808
#[test]
771809
fn test_url_contains() {
772810
let url = ListingTableUrl::parse("file:///var/data/mytable/").unwrap();

0 commit comments

Comments
 (0)