Skip to content

Commit c6f7cd9

Browse files
committed
Fix min-max stats
1 parent 4b6bdae commit c6f7cd9

2 files changed

Lines changed: 103 additions & 12 deletions

File tree

‎datafusion/datasource/src/statistics.rs‎

Lines changed: 75 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -27,9 +27,9 @@ use crate::file_groups::FileGroup;
2727

2828
use arrow::array::RecordBatch;
2929
use arrow::compute::SortColumn;
30-
use arrow::datatypes::SchemaRef;
30+
use arrow::datatypes::{DataType, SchemaRef};
3131
use arrow::row::{Row, Rows};
32-
use datafusion_common::stats::NdvFallback;
32+
use datafusion_common::stats::{NdvFallback, Precision};
3333
use datafusion_common::{
3434
DataFusionError, Result, ScalarValue, plan_datafusion_err, plan_err,
3535
};
@@ -83,17 +83,28 @@ impl MinMaxStatistics {
8383
};
8484

8585
// Helper function to get min/max statistics for a given column of projected_schema
86-
let get_min_max = |i: usize| -> Result<(Vec<ScalarValue>, Vec<ScalarValue>)> {
86+
let get_min_max = |i: usize,
87+
data_type: &DataType|
88+
-> Result<(Vec<ScalarValue>, Vec<ScalarValue>)> {
8789
Ok(statistics_and_partition_values
8890
.iter()
8991
.map(|(s, pv)| {
9092
if i < s.column_statistics.len() {
91-
s.column_statistics[i]
92-
.min_value
93-
.get_value()
94-
.cloned()
95-
.zip(s.column_statistics[i].max_value.get_value().cloned())
96-
.ok_or_else(|| plan_datafusion_err!("statistics not found"))
93+
let col_stats = &s.column_statistics[i];
94+
match (
95+
col_stats.min_value.get_value(),
96+
col_stats.max_value.get_value(),
97+
) {
98+
(Some(min), Some(max)) => Ok((min.clone(), max.clone())),
99+
// An all-null column, e.g. a null partition value, has a null min and max.
100+
_ if matches!(s.num_rows, Precision::Exact(_))
101+
&& col_stats.null_count == s.num_rows =>
102+
{
103+
let null = ScalarValue::try_new_null(data_type)?;
104+
Ok((null.clone(), null))
105+
}
106+
_ => Err(plan_datafusion_err!("statistics not found")),
107+
}
97108
} else if let Some(partition_value) =
98109
pv.get(i - s.column_statistics.len())
99110
{
@@ -143,7 +154,8 @@ impl MinMaxStatistics {
143154
.map(|p| p[c.index()])
144155
.unwrap_or_else(|| c.index());
145156

146-
let (min, max) = get_min_max(i).map_err(|e| {
157+
let data_type = projected_schema.field(c.index()).data_type();
158+
let (min, max) = get_min_max(i, data_type).map_err(|e| {
147159
e.context(format!("get min/max for column: '{}'", c.name()))
148160
})?;
149161
Ok((
@@ -384,8 +396,7 @@ mod tests {
384396
use super::*;
385397
use crate::PartitionedFile;
386398
use crate::file_groups::FileGroup;
387-
use arrow::datatypes::{DataType, Field, Schema};
388-
use datafusion_common::stats::Precision;
399+
use arrow::datatypes::{Field, Schema};
389400
use datafusion_physical_plan::ColumnStatistics;
390401

391402
fn test_schema() -> SchemaRef {
@@ -502,4 +513,56 @@ mod tests {
502513
"unexpected error: {err:?}"
503514
);
504515
}
516+
517+
#[test]
518+
fn min_max_statistics_null_partition_value() -> Result<()> {
519+
let schema = Arc::new(Schema::new(vec![
520+
Field::new("a", DataType::Int64, true),
521+
Field::new("d", DataType::Date32, true),
522+
]));
523+
let sort_order =
524+
[PhysicalSortExpr::new_default(Arc::new(Column::new("d", 1)))].into();
525+
let file = |path: &str, d: Option<i32>| {
526+
let mut pf = PartitionedFile::new(path, 1);
527+
pf.partition_values = vec![ScalarValue::Date32(d)];
528+
pf.with_statistics(Arc::new(
529+
Statistics::new_unknown(&test_schema())
530+
.with_num_rows(Precision::Exact(1)),
531+
))
532+
};
533+
let files = [file("d=1/f.parquet", Some(1)), file("null/f.parquet", None)];
534+
535+
let stats =
536+
MinMaxStatistics::new_from_files(&sort_order, &schema, None, files.iter())?;
537+
538+
let sorted = stats.min_values_sorted();
539+
assert_eq!(sorted.iter().map(|(i, _)| *i).collect::<Vec<_>>(), [1, 0]);
540+
assert!(!stats.is_sorted());
541+
542+
Ok(())
543+
}
544+
545+
#[test]
546+
fn min_max_statistics_null_partition_value_unknown_num_rows() {
547+
let schema = Arc::new(Schema::new(vec![
548+
Field::new("a", DataType::Int64, true),
549+
Field::new("d", DataType::Date32, true),
550+
]));
551+
let sort_order =
552+
[PhysicalSortExpr::new_default(Arc::new(Column::new("d", 1)))].into();
553+
let mut pf = PartitionedFile::new("null/f.parquet", 1);
554+
pf.partition_values = vec![ScalarValue::Date32(None)];
555+
let files =
556+
[pf.with_statistics(Arc::new(Statistics::new_unknown(&test_schema())))];
557+
558+
let Err(err) =
559+
MinMaxStatistics::new_from_files(&sort_order, &schema, None, files.iter())
560+
else {
561+
panic!("expected missing statistics error")
562+
};
563+
assert!(
564+
err.to_string().contains("statistics not found"),
565+
"unexpected error: {err:?}"
566+
);
567+
}
505568
}

‎datafusion/sqllogictest/test_files/listing_table_partitions.slt‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -189,3 +189,31 @@ query DD
189189
select min(d), max(d) from null_part;
190190
----
191191
2024-01-01 2024-01-01
192+
193+
# A null partition still lets files be ordered by their partition value.
194+
statement ok
195+
set datafusion.execution.target_partitions = 1;
196+
197+
statement count 0
198+
create external table null_part_ordered (column1 int, column2 varchar, d date)
199+
stored as parquet
200+
partitioned by (d)
201+
with order (d)
202+
location 'test_files/scratch/listing_table_partitions/null_part/';
203+
204+
query TT
205+
explain select column1, d from null_part_ordered order by d;
206+
----
207+
logical_plan
208+
01)Sort: null_part_ordered.d ASC NULLS LAST
209+
02)--TableScan: null_part_ordered projection=[column1, d]
210+
physical_plan DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/listing_table_partitions/null_part/d=2024-01-01/data.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/listing_table_partitions/null_part/d=__HIVE_DEFAULT_PARTITION__/data.parquet]]}, projection=[column1, d], output_ordering=[d@1 ASC NULLS LAST], file_type=parquet
211+
212+
query ID
213+
select column1, d from null_part_ordered order by d;
214+
----
215+
1 2024-01-01
216+
2 NULL
217+
218+
statement ok
219+
set datafusion.execution.target_partitions = 4;

0 commit comments

Comments
 (0)