Skip to content

Commit 99a6af8

Browse files
authored
fix: reject NULL partition keys during partitioned writes (apache#24951)
## Which issue does this PR close? Related to apache#18083. Full support for NULL partition values remains covered by that issue. ## Rationale for this change Partitioned writes read partition values without checking their validity. A NULL integer partition key can be written under `p=0`, so reading the output turns NULL into zero and silently changes the data. For example: ```sql CREATE TABLE src(p INT, v INT) AS VALUES (NULL, 1), (0, 2); COPY src TO '/tmp/null_partition/' STORED AS PARQUET PARTITIONED BY (p); ``` Both rows can end up in the zero partition. This PR rejects the write with `NULL values are not supported for partition column 'p'`. ## What changes are included in this PR? - Check logical nulls before extracting partition keys, including NULL dictionary values referenced by valid dictionary indices. - Continue accepting nullable columns when the selected rows contain no NULL values. - Add focused unit tests and a `copy.slt` regression covering rejection and a filtered successful write. ## What is the testing strategy for this PR? - The rejection regressions fail without the production fix and pass with it. - Focused partition-key tests: 5 passed with `RUST_BACKTRACE=1`. - `cargo clippy --all-targets --all-features -- -D warnings` passed. - The extended workspace Rust suites passed after excluding four host memory-measurement runners that also fail on current main in this environment. - The SQL logic harness passed all 513 files, including encrypted Parquet coverage. - `cargo fmt --all -- --check` and `git diff --check` passed. ## Are there any user-facing changes? Partitioned writes containing NULL partition keys now return an execution error. Callers can filter or replace these values before writing. Validation happens per batch, so files written by earlier batches can remain after an error.
1 parent 17c6d74 commit 99a6af8

2 files changed

Lines changed: 112 additions & 1 deletion

File tree

‎datafusion/datasource/src/write/demux.rs‎

Lines changed: 90 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,9 @@ use datafusion_common::cast::{
3838
as_int64_array, as_large_string_array, as_string_array, as_string_view_array,
3939
as_uint8_array, as_uint16_array, as_uint32_array, as_uint64_array,
4040
};
41-
use datafusion_common::{exec_datafusion_err, internal_datafusion_err, not_impl_err};
41+
use datafusion_common::{
42+
exec_datafusion_err, exec_err, internal_datafusion_err, not_impl_err,
43+
};
4244
use datafusion_common_runtime::SpawnedTask;
4345

4446
use chrono::NaiveDate;
@@ -392,6 +394,11 @@ fn compute_partition_keys_by_row<'a>(
392394
"PartitionBy Column {} does not exist in source data! Got schema {schema}.",
393395
col
394396
))?;
397+
if col_array.logical_null_count() > 0 {
398+
return exec_err!(
399+
"NULL values are not supported for partition column '{col}'"
400+
);
401+
}
395402

396403
match dtype {
397404
DataType::Utf8 => {
@@ -595,3 +602,85 @@ fn compute_hive_style_file_path(
595602

596603
file_path.join(format!("{write_id}.{file_extension}"))
597604
}
605+
606+
#[cfg(test)]
607+
mod tests {
608+
use super::*;
609+
use arrow::array::{Array, ArrayRef, DictionaryArray, Int32Array};
610+
use arrow::datatypes::Int32Type;
611+
612+
fn partition_batch(column: ArrayRef) -> RecordBatch {
613+
RecordBatch::try_from_iter_with_nullable([("p", column, true)]).unwrap()
614+
}
615+
616+
#[test]
617+
fn partition_keys_reject_null_values() {
618+
let columns: Vec<ArrayRef> = vec![
619+
Arc::new(Int32Array::from(vec![Some(0), None])),
620+
Arc::new(StringArray::from(vec![Some(""), None])),
621+
];
622+
for column in columns {
623+
let partition_by = vec![("p".to_string(), column.data_type().clone())];
624+
let batch = partition_batch(column);
625+
let err = compute_partition_keys_by_row(&batch, &partition_by).unwrap_err();
626+
assert_eq!(
627+
err.strip_backtrace(),
628+
"Execution error: NULL values are not supported for partition column 'p'"
629+
);
630+
}
631+
}
632+
633+
#[test]
634+
fn partition_keys_accept_slice_without_nulls() {
635+
let column = Int32Array::from(vec![None, Some(0), Some(1)]).slice(1, 2);
636+
let batch = partition_batch(Arc::new(column));
637+
let partition_by = vec![("p".to_string(), DataType::Int32)];
638+
let keys = compute_partition_keys_by_row(&batch, &partition_by).unwrap();
639+
assert_eq!(keys, vec![vec![Cow::Borrowed("0"), Cow::Borrowed("1")]]);
640+
}
641+
642+
fn dictionary_with_null_value() -> DictionaryArray<Int32Type> {
643+
DictionaryArray::try_new(
644+
Int32Array::from(vec![0, 1]),
645+
Arc::new(StringArray::from(vec![Some("a"), None])),
646+
)
647+
.unwrap()
648+
}
649+
650+
#[test]
651+
fn partition_keys_reject_dictionary_null_values() {
652+
let column = dictionary_with_null_value();
653+
let partition_by = vec![("p".to_string(), column.data_type().clone())];
654+
let batch = partition_batch(Arc::new(column));
655+
let err = compute_partition_keys_by_row(&batch, &partition_by).unwrap_err();
656+
assert_eq!(
657+
err.strip_backtrace(),
658+
"Execution error: NULL values are not supported for partition column 'p'"
659+
);
660+
}
661+
662+
#[test]
663+
fn partition_keys_reject_dictionary_null_keys() {
664+
let column = DictionaryArray::<Int32Type>::try_new(
665+
Int32Array::from(vec![Some(0), None]),
666+
Arc::new(StringArray::from(vec!["a"])),
667+
)
668+
.unwrap();
669+
let partition_by = vec![("p".to_string(), column.data_type().clone())];
670+
let batch = partition_batch(Arc::new(column));
671+
let err = compute_partition_keys_by_row(&batch, &partition_by).unwrap_err();
672+
assert_eq!(
673+
err.strip_backtrace(),
674+
"Execution error: NULL values are not supported for partition column 'p'"
675+
);
676+
}
677+
678+
#[test]
679+
fn partition_keys_accept_unused_dictionary_null_values() {
680+
let column = dictionary_with_null_value().slice(0, 1);
681+
let partition_by = vec![("p".to_string(), column.data_type().clone())];
682+
let batch = partition_batch(Arc::new(column));
683+
let keys = compute_partition_keys_by_row(&batch, &partition_by).unwrap();
684+
assert_eq!(keys, vec![vec![Cow::Borrowed("a")]]);
685+
}
686+
}

‎datafusion/sqllogictest/test_files/copy.slt‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,28 @@ select * from validate_partitioned_parquet order by col1, col2;
4242
1 Foo
4343
2 Bar
4444

45+
# Reject NULL partition values instead of writing them into the zero partition.
46+
statement ok
47+
CREATE TABLE nullable_partition_source (id INT, p INT) AS VALUES (1, NULL), (2, 0);
48+
49+
statement error DataFusion error: Execution error: NULL values are not supported for partition column 'p'
50+
COPY nullable_partition_source TO 'test_files/scratch/copy/null_partition/' STORED AS PARQUET PARTITIONED BY (p);
51+
52+
# Nullable partition columns still work when the copied rows contain no NULLs.
53+
query I
54+
COPY (SELECT * FROM nullable_partition_source WHERE p IS NOT NULL) TO 'test_files/scratch/copy/non_null_partition/' STORED AS PARQUET PARTITIONED BY (p);
55+
----
56+
1
57+
58+
statement ok
59+
CREATE EXTERNAL TABLE validate_non_null_partition (id INT, p INT)
60+
STORED AS PARQUET LOCATION 'test_files/scratch/copy/non_null_partition/' PARTITIONED BY (p);
61+
62+
query II
63+
SELECT id, p FROM validate_non_null_partition;
64+
----
65+
2 0
66+
4567
# validate partition paths were actually generated
4668
statement ok
4769
CREATE EXTERNAL TABLE validate_partitioned_parquet_bar STORED AS PARQUET

0 commit comments

Comments
 (0)