Skip to content

Commit ad59f94

Browse files
committed
fix: keep a per-partition fetch exact only for an exact row count
An input estimate no larger than the fetch does not prove that no partition loses rows, so passing the input through unchanged could keep exact column statistics that the fetch may have changed. Take that shortcut only for an exact count, and let an estimate become inexact.
1 parent 81b7550 commit ad59f94

1 file changed

Lines changed: 31 additions & 3 deletions

File tree

‎datafusion/physical-plan/src/statistics.rs‎

Lines changed: 31 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ use crate::operator_statistics::{
2626
ExtendedStatistics, StatisticsRegistry, StatisticsResult,
2727
};
2828
use datafusion_common::extensions::Extensions;
29+
use datafusion_common::stats::Precision;
2930
use datafusion_common::{
3031
Result, Statistics, assert_eq_or_internal_err, assert_or_internal_err,
3132
};
@@ -113,7 +114,7 @@ impl StatisticsArgs {
113114
/// every partition can emit up to `fetch` rows, so the output can reach
114115
/// `fetch * n_partitions` rows, and never more than the input. How the input
115116
/// rows are spread over the partitions is unknown, so the result is inexact
116-
/// unless `fetch` cannot drop any row.
117+
/// unless an exact input count shows that `fetch` cannot drop any row.
117118
pub(crate) fn with_per_partition_fetch(
118119
input: Statistics,
119120
fetch: Option<usize>,
@@ -126,8 +127,9 @@ pub(crate) fn with_per_partition_fetch(
126127
if args.partition().is_some() || n_partitions <= 1 {
127128
return input.with_fetch(Some(fetch), 0, 1);
128129
}
129-
// No partition can hold more than `fetch` rows, so none is dropped.
130-
if matches!(input.num_rows.get_value(), Some(&num_rows) if num_rows <= fetch) {
130+
// No partition can hold more than `fetch` rows, so none is dropped. Only
131+
// an exact count proves that: an estimate below `fetch` may be wrong.
132+
if matches!(input.num_rows, Precision::Exact(num_rows) if num_rows <= fetch) {
131133
return Ok(input);
132134
}
133135
Ok(input
@@ -697,6 +699,32 @@ mod tests {
697699
assert_eq!(overall.num_rows, Precision::Exact(999));
698700
}
699701

702+
/// Only an exact row count can show that a fetch on each of four partitions
703+
/// drops no row. An estimate cannot, so the result becomes inexact, column
704+
/// statistics included.
705+
#[test]
706+
fn per_partition_fetch_keeps_exactness_only_for_exact_counts() -> Result<()> {
707+
let schema = Schema::new(vec![Field::new("a", DataType::Int32, true)]);
708+
let fetch_10_of_4 = |num_rows| {
709+
let mut input = Statistics::new_unknown(&schema);
710+
input.num_rows = num_rows;
711+
input.column_statistics[0].null_count = Precision::Exact(3);
712+
with_per_partition_fetch(input, Some(10), 4, &StatisticsArgs::new())
713+
};
714+
715+
let exact = fetch_10_of_4(Precision::Exact(8))?;
716+
assert_eq!(exact.num_rows, Precision::Exact(8));
717+
assert_eq!(exact.column_statistics[0].null_count, Precision::Exact(3));
718+
719+
let estimate = fetch_10_of_4(Precision::Inexact(8))?;
720+
assert_eq!(estimate.num_rows, Precision::Inexact(8));
721+
assert_eq!(
722+
estimate.column_statistics[0].null_count,
723+
Precision::Inexact(3)
724+
);
725+
Ok(())
726+
}
727+
700728
/// Row counts under a fetch that applies to each of four partitions.
701729
#[test]
702730
fn per_partition_fetch_counts_every_partition() -> Result<()> {

0 commit comments

Comments
 (0)