Skip to content

Commit 5a5df7d

Browse files
committed
fix: include nulls in file ordering bounds
1 parent 9551051 commit 5a5df7d

2 files changed

Lines changed: 104 additions & 13 deletions

File tree

‎datafusion/datasource/src/file_scan_config/mod.rs‎

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2188,6 +2188,9 @@ mod tests {
21882188
.map(|stats| {
21892189
stats
21902190
.map(|(min, max)| ColumnStatistics {
2191+
null_count: Precision::Exact(usize::from(
2192+
min.is_none() || max.is_none(),
2193+
)),
21912194
min_value: Precision::Exact(ScalarValue::Float64(
21922195
min,
21932196
)),
@@ -3797,6 +3800,61 @@ mod tests {
37973800
))
37983801
}
37993802

3803+
#[test]
3804+
fn file_group_ordering_includes_nulls() -> Result<()> {
3805+
// Arrange
3806+
let schema =
3807+
Arc::new(Schema::new(vec![Field::new("a", DataType::Float64, true)]));
3808+
for (descending, nulls_first) in
3809+
[(false, false), (false, true), (true, false), (true, true)]
3810+
{
3811+
let sort = PhysicalSortExpr::new(
3812+
Arc::new(Column::new("a", 0)),
3813+
arrow::compute::SortOptions {
3814+
descending,
3815+
nulls_first,
3816+
},
3817+
);
3818+
for null_count in [
3819+
Precision::Exact(0),
3820+
Precision::Exact(1),
3821+
Precision::Absent,
3822+
Precision::Inexact(0),
3823+
] {
3824+
let mut files = vec![
3825+
make_file_with_null_stats("low", 0.0, 9.0, 0),
3826+
make_file_with_null_stats("high", 10.0, 19.0, 0),
3827+
];
3828+
for file in &mut files {
3829+
Arc::make_mut(file.statistics.as_mut().unwrap()).column_statistics
3830+
[0]
3831+
.null_count = null_count;
3832+
}
3833+
if descending {
3834+
files.reverse();
3835+
}
3836+
let config = FileScanConfigBuilder::new(
3837+
ObjectStoreUrl::local_filesystem(),
3838+
Arc::new(MockSource::new(TableSchema::from(&schema))),
3839+
)
3840+
.with_file_groups(vec![FileGroup::new(files)])
3841+
.with_output_ordering(vec![[sort.clone()].into()])
3842+
.build();
3843+
3844+
// Act
3845+
let ordered = config.eq_properties().ordering_satisfy([sort.clone()])?;
3846+
3847+
// Assert
3848+
assert_eq!(
3849+
ordered,
3850+
null_count == Precision::Exact(0),
3851+
"descending={descending}, nulls_first={nulls_first}, null_count={null_count:?}"
3852+
);
3853+
}
3854+
}
3855+
Ok(())
3856+
}
3857+
38003858
#[test]
38013859
fn sort_pushdown_unsupported_with_nulls_does_not_upgrade_to_exact() -> Result<()> {
38023860
// Files are non-overlapping but one has NULLs.

‎datafusion/datasource/src/statistics.rs‎

Lines changed: 46 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -26,10 +26,10 @@ use crate::PartitionedFile;
2626
use crate::file_groups::FileGroup;
2727

2828
use arrow::array::RecordBatch;
29-
use arrow::compute::SortColumn;
30-
use arrow::datatypes::SchemaRef;
29+
use arrow::compute::{SortColumn, SortOptions};
30+
use arrow::datatypes::{Field, SchemaRef};
3131
use arrow::row::{Row, Rows};
32-
use datafusion_common::stats::NdvFallback;
32+
use datafusion_common::stats::{NdvFallback, Precision};
3333
use datafusion_common::{
3434
DataFusionError, Result, ScalarValue, plan_datafusion_err, plan_err,
3535
};
@@ -83,16 +83,46 @@ impl MinMaxStatistics {
8383
};
8484

8585
// Helper function to get min/max statistics for a given column of projected_schema
86-
let get_min_max = |i: usize| -> Result<(Vec<ScalarValue>, Vec<ScalarValue>)> {
86+
let get_min_max = |i: usize,
87+
field: &Field,
88+
options: SortOptions|
89+
-> Result<(Vec<ScalarValue>, Vec<ScalarValue>)> {
90+
let null = ScalarValue::try_new_null(field.data_type())?;
8791
Ok(statistics_and_partition_values
8892
.iter()
8993
.map(|(s, pv)| {
9094
if i < s.column_statistics.len() {
91-
s.column_statistics[i]
92-
.min_value
93-
.get_value()
94-
.cloned()
95-
.zip(s.column_statistics[i].max_value.get_value().cloned())
95+
let column_statistics = &s.column_statistics[i];
96+
let all_null = matches!(
97+
(s.num_rows, column_statistics.null_count),
98+
(Precision::Exact(rows), Precision::Exact(nulls))
99+
if rows > 0 && rows == nulls
100+
);
101+
let may_have_nulls = match column_statistics.null_count {
102+
Precision::Exact(nulls) => nulls > 0,
103+
_ => field.is_nullable(),
104+
};
105+
// Self::new swaps the bounds for descending sort keys.
106+
let nulls_on_min = options.nulls_first != options.descending;
107+
let min = if all_null || (may_have_nulls && nulls_on_min) {
108+
Some(null.clone())
109+
} else {
110+
column_statistics
111+
.min_value
112+
.get_value()
113+
.filter(|value| !value.is_null())
114+
.cloned()
115+
};
116+
let max = if all_null || (may_have_nulls && !nulls_on_min) {
117+
Some(null.clone())
118+
} else {
119+
column_statistics
120+
.max_value
121+
.get_value()
122+
.filter(|value| !value.is_null())
123+
.cloned()
124+
};
125+
min.zip(max)
96126
.ok_or_else(|| plan_datafusion_err!("statistics not found"))
97127
} else if let Some(partition_value) =
98128
pv.get(i - s.column_statistics.len())
@@ -135,17 +165,20 @@ impl MinMaxStatistics {
135165

136166
let (min_values, max_values): (Vec<_>, Vec<_>) = sort_columns
137167
.iter()
138-
.map(|c| {
168+
.zip(projected_sort_order.iter())
169+
.map(|(c, sort_expr)| {
139170
// Reverse the projection to get the index of the column in the full statistics
140171
// The file statistics contains _every_ column , but the sort column's index()
141172
// refers to the index in projected_schema
142173
let i = projection
143174
.map(|p| p[c.index()])
144175
.unwrap_or_else(|| c.index());
145176

146-
let (min, max) = get_min_max(i).map_err(|e| {
147-
e.context(format!("get min/max for column: '{}'", c.name()))
148-
})?;
177+
let (min, max) =
178+
get_min_max(i, projected_schema.field(c.index()), sort_expr.options)
179+
.map_err(|e| {
180+
e.context(format!("get min/max for column: '{}'", c.name()))
181+
})?;
149182
Ok((
150183
ScalarValue::iter_to_array(min)?,
151184
ScalarValue::iter_to_array(max)?,

0 commit comments

Comments
 (0)