Skip to content

Commit d9ac63e

Browse files
kamilk-palhareshkhclaude
authored
[branch-55] fix: read Hive's null partition placeholder as NULL (#26310) (#27392)
## Which issue does this PR close? - Backport of #26310 to `branch-55` - Part of #26868 --- ## Differences from the branch-54 backport (#27308) This backport is the same as #27308. Comparing the two diffs with `git range-diff` shows only these two changed lines that differ: 1. **`table.rs`: indentation only.** The same `false` → `true` nullability change is one level deeper, because branch-55 builds the schema with `TableSchemaBuilder` instead of `TableSchema::new`. - branch-55: [table.rs#L347-L348](https://github.com/kamilk-pal/datafusion/blob/a14dd16c2fa438d6ef90631b7e118383c2c916c5/datafusion/catalog-listing/src/table.rs#L347-L348) - branch-54: [table.rs#L345-L346](https://github.com/apache/datafusion/blob/31cb93f88404daa7d64048aaec7fa802aaaa0079/datafusion/catalog-listing/src/table.rs#L345-L346) 2. **`listing_table_partitions.slt`: comment only.** It says "branch-55" instead of "branch-54". branch-55 keeps the explicit sort just like branch-54, so the expected plan is unchanged. - branch-55: [listing_table_partitions.slt#L217](https://github.com/kamilk-pal/datafusion/blob/a14dd16c2fa438d6ef90631b7e118383c2c916c5/datafusion/sqllogictest/test_files/listing_table_partitions.slt#L217) - branch-54: [listing_table_partitions.slt#L217](https://github.com/apache/datafusion/blob/31cb93f88404daa7d64048aaec7fa802aaaa0079/datafusion/sqllogictest/test_files/listing_table_partitions.slt#L217) All other added and removed lines are identical in both backports. Some surrounding code differs because the base branches differ (for example, branch-55 percent-decodes partition values), but neither backport changes it. Co-authored-by: Haresh Khanna <[email protected]> Co-authored-by: Claude Opus 5.5 <[email protected]>
1 parent 44a258e commit d9ac63e

9 files changed

Lines changed: 456 additions & 23 deletions

File tree

‎datafusion/catalog-listing/src/helpers.rs‎

Lines changed: 59 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ use datafusion_datasource::{FileExtensions, ListingTableUrl};
2929
use datafusion_expr::{BinaryExpr, Operator, lit, utils};
3030

3131
use arrow::{
32-
array::AsArray,
32+
array::{Array, AsArray},
3333
datatypes::{DataType, Field},
3434
record_batch::RecordBatch,
3535
};
@@ -50,6 +50,8 @@ use percent_encoding::{AsciiSet, CONTROLS, percent_decode_str, utf8_percent_enco
5050
const PARTITION_VALUE_ENCODE_SET: &AsciiSet =
5151
&CONTROLS.add(b' ').add(b'%').add(b'/').add(b'?').add(b'#');
5252

53+
const HIVE_DEFAULT_PARTITION_NAME: &str = "__HIVE_DEFAULT_PARTITION__";
54+
5355
/// Check whether the given expression can be resolved using only the columns `col_names`.
5456
/// This means that if this function returns true:
5557
/// - the table provider can filter the table partition values with this expression
@@ -337,19 +339,22 @@ pub fn filter_partitioned_file(
337339
)?;
338340

339341
// Since we're only operating on a single file, our batch and resulting "array" holds only one
340-
// value indicating if the input file matches the provided filters
342+
// value indicating if the input file matches the provided filters. A NULL is not a match:
343+
// a null partition value makes that reachable, as `part = 1` is NULL for a null partition.
341344
let matches = expr.evaluate(&batch)?.into_array(1)?;
342-
if matches.as_boolean().value(0) {
345+
let matches = matches.as_boolean();
346+
if matches.is_valid(0) && matches.value(0) {
343347
return Ok(Some(pf));
344348
}
345349

346350
Ok(None)
347351
}
348352

349353
/// Returns `Ok(None)` when the file is not inside a valid partition path
350-
/// (e.g. a stale file in the table root directory). Such files are skipped
351-
/// because hive-style partition values are never null and there is no valid
352-
/// value to assign for non-partitioned files.
354+
/// (e.g. a stale file in the table root directory), as there is no value to
355+
/// assign for non-partitioned files.
356+
///
357+
/// A partition value named `__HIVE_DEFAULT_PARTITION__` is read as NULL.
353358
fn try_into_partitioned_file(
354359
object_meta: ObjectMeta,
355360
partition_cols: &[(String, DataType)],
@@ -367,6 +372,9 @@ fn try_into_partitioned_file(
367372
.into_iter()
368373
.zip(partition_cols)
369374
.map(|(parsed, (_, datatype))| {
375+
if parsed == HIVE_DEFAULT_PARTITION_NAME {
376+
return ScalarValue::try_new_null(datatype);
377+
}
370378
ScalarValue::try_from_string(parsed.into_owned(), datatype)
371379
})
372380
.collect::<Result<Vec<_>>>()?;
@@ -427,7 +435,7 @@ pub async fn pruned_partition_list<'a>(
427435
)?;
428436

429437
Ok(objects
430-
.try_filter_map(|object_meta| {
438+
.try_filter_map(move |object_meta| {
431439
futures::future::ready(try_into_partitioned_file(
432440
object_meta,
433441
partition_cols,
@@ -709,6 +717,50 @@ mod tests {
709717
);
710718
}
711719

720+
#[test]
721+
fn test_try_into_partitioned_file_null_partition() {
722+
let table_path = ListingTableUrl::parse("file:///bucket/mytable").unwrap();
723+
// A type the placeholder cannot be parsed as, which is how this used to fail.
724+
let partition_cols = vec![("day".to_string(), DataType::Date32)];
725+
let meta = ObjectMeta {
726+
location: Path::from(
727+
"bucket/mytable/day=__HIVE_DEFAULT_PARTITION__/data.parquet",
728+
),
729+
last_modified: chrono::Utc::now(),
730+
size: 100,
731+
e_tag: None,
732+
version: None,
733+
};
734+
735+
let pf = try_into_partitioned_file(meta, &partition_cols, &table_path)
736+
.unwrap()
737+
.expect("file should be listed");
738+
739+
assert_eq!(pf.partition_values, vec![ScalarValue::Date32(None)]);
740+
}
741+
742+
#[test]
743+
fn test_try_into_partitioned_file_other_partition_name_is_literal() {
744+
let table_path = ListingTableUrl::parse("file:///bucket/mytable").unwrap();
745+
let partition_cols = vec![("category".to_string(), DataType::Utf8)];
746+
let meta = ObjectMeta {
747+
location: Path::from("bucket/mytable/category=MY_NULL/data.parquet"),
748+
last_modified: chrono::Utc::now(),
749+
size: 100,
750+
e_tag: None,
751+
version: None,
752+
};
753+
754+
let pf = try_into_partitioned_file(meta, &partition_cols, &table_path)
755+
.unwrap()
756+
.expect("file should be listed");
757+
758+
assert_eq!(
759+
pf.partition_values,
760+
vec![ScalarValue::Utf8(Some("MY_NULL".to_string()))]
761+
);
762+
}
763+
712764
#[test]
713765
fn test_try_into_partitioned_file_root_file_skipped() {
714766
// File in root directory (not inside any partition path) should be

‎datafusion/catalog-listing/src/table.rs‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -224,10 +224,11 @@ impl ListingTable {
224224
.options
225225
.ok_or_else(|| internal_datafusion_err!("No ListingOptions provided"))?;
226226

227-
// Add the partition columns to the file schema
227+
// Add the partition columns to the file schema. They are nullable because a Hive
228+
// compliant writer uses `__HIVE_DEFAULT_PARTITION__` for a null partition value.
228229
let mut builder = SchemaBuilder::from(file_schema.as_ref().to_owned());
229230
for (part_col_name, part_col_type) in &options.table_partition_cols {
230-
builder.push(Field::new(part_col_name, part_col_type.clone(), false));
231+
builder.push(Field::new(part_col_name, part_col_type.clone(), true));
231232
}
232233

233234
let table_schema = Arc::new(
@@ -343,7 +344,8 @@ impl ListingTable {
343344
self.options
344345
.table_partition_cols
345346
.iter()
346-
.map(|(col, field)| Arc::new(Field::new(col, field.clone(), false)))
347+
// Nullable to match the table schema built in `try_new`.
348+
.map(|(col, field)| Arc::new(Field::new(col, field.clone(), true)))
347349
.collect::<Vec<_>>(),
348350
)
349351
.build();
@@ -987,7 +989,7 @@ impl ListingTable {
987989

988990
// Note: Statistics already include both file columns and partition columns.
989991
// PartitionedFile::with_statistics automatically appends exact partition column
990-
// statistics (min=max=partition_value, null_count=0, distinct_count=1) computed
992+
// statistics (e.g. min=max=partition_value, null_count=0, distinct_count=1) computed
991993
// from partition_values.
992994
Ok(ListFilesResult {
993995
file_groups,

‎datafusion/core/tests/catalog_listing/pruned_partition_list.rs‎

Lines changed: 105 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -233,6 +233,111 @@ async fn test_list_partition() {
233233
);
234234
}
235235

236+
/// A partition directory named `__HIVE_DEFAULT_PARTITION__` holds the rows whose partition
237+
/// value is null, so it is read as NULL rather than parsed as the column's type.
238+
#[tokio::test]
239+
async fn test_pruned_partition_list_null_partition() {
240+
let (store, state) = make_test_store_and_state(&[
241+
("tablepath/mypartition=3/file.parquet", 100),
242+
(
243+
"tablepath/mypartition=__HIVE_DEFAULT_PARTITION__/file.parquet",
244+
100,
245+
),
246+
]);
247+
248+
let pruned = pruned_partition_list(
249+
state.as_ref(),
250+
store.as_ref(),
251+
&ListingTableUrl::parse("file:///tablepath/").unwrap(),
252+
&[],
253+
".parquet",
254+
&[(String::from("mypartition"), DataType::Int32)],
255+
)
256+
.await
257+
.expect("partition pruning failed")
258+
.try_collect::<Vec<_>>()
259+
.await
260+
.unwrap();
261+
262+
assert_eq!(pruned.len(), 2);
263+
assert_eq!(&pruned[0].partition_values, &[ScalarValue::Int32(Some(3))]);
264+
assert_eq!(&pruned[1].partition_values, &[ScalarValue::Int32(None)]);
265+
}
266+
267+
#[tokio::test]
268+
async fn test_pruned_partition_list_prunes_null_partition() {
269+
let files: &[(&str, u64)] = &[
270+
("tablepath/mypartition=3/file.parquet", 100),
271+
(
272+
"tablepath/mypartition=__HIVE_DEFAULT_PARTITION__/file.parquet",
273+
100,
274+
),
275+
];
276+
let partition_cols = &[(String::from("mypartition"), DataType::Int32)];
277+
278+
let pruned_with = |filter: Expr| async move {
279+
let (store, state) = make_test_store_and_state(files);
280+
pruned_partition_list(
281+
state.as_ref(),
282+
store.as_ref(),
283+
&ListingTableUrl::parse("file:///tablepath/").unwrap(),
284+
&[filter],
285+
".parquet",
286+
partition_cols,
287+
)
288+
.await
289+
.expect("partition pruning failed")
290+
.try_collect::<Vec<_>>()
291+
.await
292+
.unwrap()
293+
};
294+
295+
// The null partition does not match an equality filter: `mypartition = 3` is NULL there.
296+
let pruned = pruned_with(Expr::eq(col("mypartition"), lit(3i32))).await;
297+
assert_eq!(pruned.len(), 1);
298+
assert_eq!(
299+
pruned[0].object_meta.location.as_ref(),
300+
"tablepath/mypartition=3/file.parquet"
301+
);
302+
303+
// ... and only the null partition matches `IS NULL`.
304+
let pruned = pruned_with(col("mypartition").is_null()).await;
305+
assert_eq!(pruned.len(), 1);
306+
assert_eq!(
307+
pruned[0].object_meta.location.as_ref(),
308+
"tablepath/mypartition=__HIVE_DEFAULT_PARTITION__/file.parquet"
309+
);
310+
}
311+
312+
#[tokio::test]
313+
async fn test_pruned_partition_list_hive_null_partition_name() {
314+
let (store, state) = make_test_store_and_state(&[
315+
("tablepath/mypartition=MY_NULL/file.parquet", 100),
316+
(
317+
"tablepath/mypartition=__HIVE_DEFAULT_PARTITION__/file.parquet",
318+
100,
319+
),
320+
]);
321+
let pruned = pruned_partition_list(
322+
state.as_ref(),
323+
store.as_ref(),
324+
&ListingTableUrl::parse("file:///tablepath/").unwrap(),
325+
&[],
326+
".parquet",
327+
&[(String::from("mypartition"), DataType::Utf8)],
328+
)
329+
.await
330+
.expect("partition pruning failed")
331+
.try_collect::<Vec<_>>()
332+
.await
333+
.unwrap();
334+
335+
// Only the fixed Hive marker is read as NULL; other names remain literal values.
336+
assert_eq!(pruned.len(), 2);
337+
assert_eq!(&pruned[0].partition_values, &[ScalarValue::from("MY_NULL")]);
338+
assert_eq!(&pruned[1].partition_values, &[ScalarValue::Utf8(None)]);
339+
}
340+
236341
pub fn make_test_store_and_state(
237342
files: &[(&str, u64)],
238343
) -> (Arc<InMemory>, Arc<dyn Session>) {

‎datafusion/datasource/src/file_groups.rs‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -371,9 +371,11 @@ impl FileGroupPartitioner {
371371
/// in the group for the **full table schema** (file columns + partition columns).
372372
///
373373
/// Partition column statistics are derived from the individual file partition values:
374-
/// - `min` = minimum partition value across all files in the group
375-
/// - `max` = maximum partition value across all files in the group
376-
/// - `null_count` = 0 (partition values are never null)
374+
/// - `min` = minimum partition value across all files in the group, or absent if any
375+
/// file in the group has a null partition value (see #26748)
376+
/// - `max` = maximum partition value across all files in the group, or absent if any
377+
/// file in the group has a null partition value
378+
/// - `null_count` = the number of rows in files with a null partition value
377379
///
378380
/// This allows query optimizers to prune entire file groups based on partition bounds.
379381
#[derive(Debug, Clone)]

‎datafusion/datasource/src/mod.rs‎

Lines changed: 44 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -118,9 +118,12 @@ impl FileRange {
118118
/// computed from [`Self::partition_values`]:
119119
///
120120
/// - `min = max = partition_value` (all rows in a file share the same partition value)
121-
/// - `null_count = 0` (partition values extracted from paths are never null)
121+
/// - `null_count = 0` (every row has the file's partition value)
122122
/// - `distinct_count = 1` (single distinct value per file for each partition column)
123123
///
124+
/// A null partition value instead gives `null_count = num_rows`, `distinct_count = 0`
125+
/// and absent `min`/`max`.
126+
///
124127
/// This enables query optimizers to use partition column bounds for pruning and planning.
125128
pub struct PartitionedFile {
126129
/// Path for the file (e.g. URL, filesystem path, etc)
@@ -339,8 +342,11 @@ impl PartitionedFile {
339342
/// This method will automatically append exact statistics for partition columns
340343
/// based on `partition_values`:
341344
/// - `min = max = partition_value` (all rows have the same value)
342-
/// - `null_count = 0` (partition values from paths are never null)
345+
/// - `null_count = 0` (all rows have a non-null value)
343346
/// - `distinct_count = 1` (all rows have the same partition value)
347+
///
348+
/// A null partition value (e.g. a Hive default partition) instead gives
349+
/// `null_count = num_rows`, `distinct_count = 0` and absent `min`/`max`.
344350
pub fn with_statistics(mut self, file_statistics: Arc<Statistics>) -> Self {
345351
if self.partition_values.is_empty() {
346352
// No partition columns, use stats as-is
@@ -349,11 +355,20 @@ impl PartitionedFile {
349355
// Extend stats with exact partition column statistics
350356
let mut stats = Arc::unwrap_or_clone(file_statistics);
351357
for partition_value in &self.partition_values {
358+
let (null_count, min_max, distinct_count) = if partition_value.is_null() {
359+
(stats.num_rows, Precision::Absent, Precision::Exact(0))
360+
} else {
361+
(
362+
Precision::Exact(0),
363+
Precision::Exact(partition_value.clone()),
364+
Precision::Exact(1),
365+
)
366+
};
352367
let col_stats = ColumnStatistics {
353-
null_count: Precision::Exact(0),
354-
max_value: Precision::Exact(partition_value.clone()),
355-
min_value: Precision::Exact(partition_value.clone()),
356-
distinct_count: Precision::Exact(1),
368+
null_count,
369+
max_value: min_max.clone(),
370+
min_value: min_max,
371+
distinct_count,
357372
sum_value: Precision::Absent,
358373
byte_size: partition_value
359374
.data_type()
@@ -679,6 +694,29 @@ mod tests {
679694
);
680695
}
681696

697+
#[test]
698+
fn test_with_statistics_null_partition_value() {
699+
use crate::PartitionedFile;
700+
use datafusion_common::stats::Precision;
701+
use datafusion_common::{ScalarValue, Statistics};
702+
703+
let mut pf = PartitionedFile::new("test.parquet", 100);
704+
pf.partition_values = vec![ScalarValue::Date32(None)];
705+
706+
let file_stats = Arc::new(Statistics {
707+
num_rows: Precision::Exact(2),
708+
total_byte_size: Precision::Exact(16),
709+
column_statistics: vec![],
710+
});
711+
712+
let stats = pf.with_statistics(file_stats).statistics.unwrap();
713+
let partition_col_stats = &stats.column_statistics[0];
714+
assert_eq!(partition_col_stats.null_count, Precision::Exact(2));
715+
assert_eq!(partition_col_stats.min_value, Precision::Absent);
716+
assert_eq!(partition_col_stats.max_value, Precision::Absent);
717+
assert_eq!(partition_col_stats.distinct_count, Precision::Exact(0));
718+
}
719+
682720
#[test]
683721
fn test_url_contains() {
684722
let url = ListingTableUrl::parse("file:///var/data/mytable/").unwrap();

0 commit comments

Comments
 (0)