Skip to content

Commit ad518d5

Browse files
authored
Merge branch 'main' into eliminate-aggregate-distinct
2 parents 0f5e908 + 7b00b63 commit ad518d5

10 files changed

Lines changed: 788 additions & 172 deletions

File tree

‎datafusion/datasource/src/write/demux.rs‎

Lines changed: 90 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,9 @@ use datafusion_common::cast::{
3838
as_int64_array, as_large_string_array, as_string_array, as_string_view_array,
3939
as_uint8_array, as_uint16_array, as_uint32_array, as_uint64_array,
4040
};
41-
use datafusion_common::{exec_datafusion_err, internal_datafusion_err, not_impl_err};
41+
use datafusion_common::{
42+
exec_datafusion_err, exec_err, internal_datafusion_err, not_impl_err,
43+
};
4244
use datafusion_common_runtime::SpawnedTask;
4345

4446
use chrono::NaiveDate;
@@ -392,6 +394,11 @@ fn compute_partition_keys_by_row<'a>(
392394
"PartitionBy Column {} does not exist in source data! Got schema {schema}.",
393395
col
394396
))?;
397+
if col_array.logical_null_count() > 0 {
398+
return exec_err!(
399+
"NULL values are not supported for partition column '{col}'"
400+
);
401+
}
395402

396403
match dtype {
397404
DataType::Utf8 => {
@@ -595,3 +602,85 @@ fn compute_hive_style_file_path(
595602

596603
file_path.join(format!("{write_id}.{file_extension}"))
597604
}
605+
606+
#[cfg(test)]
607+
mod tests {
608+
use super::*;
609+
use arrow::array::{Array, ArrayRef, DictionaryArray, Int32Array};
610+
use arrow::datatypes::Int32Type;
611+
612+
fn partition_batch(column: ArrayRef) -> RecordBatch {
613+
RecordBatch::try_from_iter_with_nullable([("p", column, true)]).unwrap()
614+
}
615+
616+
#[test]
617+
fn partition_keys_reject_null_values() {
618+
let columns: Vec<ArrayRef> = vec![
619+
Arc::new(Int32Array::from(vec![Some(0), None])),
620+
Arc::new(StringArray::from(vec![Some(""), None])),
621+
];
622+
for column in columns {
623+
let partition_by = vec![("p".to_string(), column.data_type().clone())];
624+
let batch = partition_batch(column);
625+
let err = compute_partition_keys_by_row(&batch, &partition_by).unwrap_err();
626+
assert_eq!(
627+
err.strip_backtrace(),
628+
"Execution error: NULL values are not supported for partition column 'p'"
629+
);
630+
}
631+
}
632+
633+
#[test]
634+
fn partition_keys_accept_slice_without_nulls() {
635+
let column = Int32Array::from(vec![None, Some(0), Some(1)]).slice(1, 2);
636+
let batch = partition_batch(Arc::new(column));
637+
let partition_by = vec![("p".to_string(), DataType::Int32)];
638+
let keys = compute_partition_keys_by_row(&batch, &partition_by).unwrap();
639+
assert_eq!(keys, vec![vec![Cow::Borrowed("0"), Cow::Borrowed("1")]]);
640+
}
641+
642+
fn dictionary_with_null_value() -> DictionaryArray<Int32Type> {
643+
DictionaryArray::try_new(
644+
Int32Array::from(vec![0, 1]),
645+
Arc::new(StringArray::from(vec![Some("a"), None])),
646+
)
647+
.unwrap()
648+
}
649+
650+
#[test]
651+
fn partition_keys_reject_dictionary_null_values() {
652+
let column = dictionary_with_null_value();
653+
let partition_by = vec![("p".to_string(), column.data_type().clone())];
654+
let batch = partition_batch(Arc::new(column));
655+
let err = compute_partition_keys_by_row(&batch, &partition_by).unwrap_err();
656+
assert_eq!(
657+
err.strip_backtrace(),
658+
"Execution error: NULL values are not supported for partition column 'p'"
659+
);
660+
}
661+
662+
#[test]
663+
fn partition_keys_reject_dictionary_null_keys() {
664+
let column = DictionaryArray::<Int32Type>::try_new(
665+
Int32Array::from(vec![Some(0), None]),
666+
Arc::new(StringArray::from(vec!["a"])),
667+
)
668+
.unwrap();
669+
let partition_by = vec![("p".to_string(), column.data_type().clone())];
670+
let batch = partition_batch(Arc::new(column));
671+
let err = compute_partition_keys_by_row(&batch, &partition_by).unwrap_err();
672+
assert_eq!(
673+
err.strip_backtrace(),
674+
"Execution error: NULL values are not supported for partition column 'p'"
675+
);
676+
}
677+
678+
#[test]
679+
fn partition_keys_accept_unused_dictionary_null_values() {
680+
let column = dictionary_with_null_value().slice(0, 1);
681+
let partition_by = vec![("p".to_string(), column.data_type().clone())];
682+
let batch = partition_batch(Arc::new(column));
683+
let keys = compute_partition_keys_by_row(&batch, &partition_by).unwrap();
684+
assert_eq!(keys, vec![vec![Cow::Borrowed("a")]]);
685+
}
686+
}

0 commit comments

Comments
 (0)