Skip to content

Commit e126357

Browse files
author
Sergey Zhukov
committed
Statistics: Migrate to Distribution from Precision (apache#14896)
1 parent efa5edb commit e126357

10 files changed

Lines changed: 56 additions & 91 deletions

File tree

‎datafusion/core/src/datasource/statistics.rs‎

Lines changed: 4 additions & 48 deletions
Original file line numberDiff line numberDiff line change
@@ -147,30 +147,8 @@ fn set_max_if_greater(
147147
max_nominee: &ProbabilityDistribution,
148148
max_value: &mut ProbabilityDistribution,
149149
) {
150-
// match (&max_value, max_nominee) {
151-
// (Precision::Exact(val1), Precision::Exact(val2)) if val1 < val2 => {
152-
// *max_value = max_nominee.clone();
153-
// }
154-
// (Precision::Exact(val1), Precision::Inexact(val2))
155-
// | (Precision::Inexact(val1), Precision::Inexact(val2))
156-
// | (Precision::Inexact(val1), Precision::Exact(val2))
157-
// if val1 < val2 =>
158-
// {
159-
// *max_value = max_nominee.clone().to_inexact();
160-
// }
161-
// (Precision::Exact(_), Precision::Absent) => {
162-
// let exact_max = mem::take(max_value);
163-
// *max_value = exact_max.to_inexact();
164-
// }
165-
// (Precision::Absent, Precision::Exact(_)) => {
166-
// *max_value = max_nominee.clone().to_inexact();
167-
// }
168-
// (Precision::Absent, Precision::Inexact(_)) => {
169-
// *max_value = max_nominee.clone();
170-
// }
171-
// _ => {}
172-
// }
173-
if max_nominee.get_value().unwrap_or(&ScalarValue::Null) < max_value.get_value().unwrap_or(&ScalarValue::Null) {
150+
let scalar_null = ScalarValue::try_new_null(&DataType::UInt64).unwrap();
151+
if max_nominee.get_value().unwrap_or(&scalar_null) < max_value.get_value().unwrap_or(&scalar_null) {
174152
*max_value = max_nominee.clone();
175153
}
176154
}
@@ -181,30 +159,8 @@ fn set_min_if_lesser(
181159
min_nominee: &ProbabilityDistribution,
182160
min_value: &mut ProbabilityDistribution,
183161
) {
184-
// match (&min_value, min_nominee) {
185-
// (Precision::Exact(val1), Precision::Exact(val2)) if val1 > val2 => {
186-
// *min_value = min_nominee.clone();
187-
// }
188-
// (Precision::Exact(val1), Precision::Inexact(val2))
189-
// | (Precision::Inexact(val1), Precision::Inexact(val2))
190-
// | (Precision::Inexact(val1), Precision::Exact(val2))
191-
// if val1 > val2 =>
192-
// {
193-
// *min_value = min_nominee.clone().to_inexact();
194-
// }
195-
// (Precision::Exact(_), Precision::Absent) => {
196-
// let exact_min = mem::take(min_value);
197-
// *min_value = exact_min.to_inexact();
198-
// }
199-
// (Precision::Absent, Precision::Exact(_)) => {
200-
// *min_value = min_nominee.clone().to_inexact();
201-
// }
202-
// (Precision::Absent, Precision::Inexact(_)) => {
203-
// *min_value = min_nominee.clone();
204-
// }
205-
// _ => {}
206-
// }
207-
if min_nominee.get_value().unwrap_or(&ScalarValue::Null) > min_value.get_value().unwrap_or(&ScalarValue::Null) {
162+
let scalar_null = ScalarValue::try_new_null(&DataType::UInt64).unwrap();
163+
if min_nominee.get_value().unwrap_or(&scalar_null) > min_value.get_value().unwrap_or(&scalar_null) {
208164
*min_value = min_nominee.clone();
209165
}
210166
}

‎datafusion/datasource/src/statistics.rs‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ use std::sync::Arc;
2525
use crate::PartitionedFile;
2626

2727
use arrow::array::RecordBatch;
28-
use arrow::datatypes::SchemaRef;
28+
use arrow::datatypes::{DataType, SchemaRef};
2929
use arrow::{
3030
compute::SortColumn,
3131
row::{Row, Rows},
@@ -87,8 +87,9 @@ impl MinMaxStatistics {
8787
.iter()
8888
.map(|(s, pv)| {
8989
if i < s.column_statistics.len() {
90-
let min_value = s.column_statistics[i].min_value.get_value().unwrap_or(&ScalarValue::Null).clone();
91-
let max_value = s.column_statistics[i].max_value.get_value().unwrap_or(&ScalarValue::Null).clone();
90+
let scalar_null = ScalarValue::try_new_null(&DataType::UInt64).unwrap();
91+
let min_value = s.column_statistics[i].min_value.get_value().unwrap_or(&scalar_null).clone();
92+
let max_value = s.column_statistics[i].max_value.get_value().unwrap_or(&scalar_null).clone();
9293
Ok((min_value, max_value))
9394
} else {
9495
let partition_value = &pv[i - s.column_statistics.len()];

‎datafusion/expr-common/src/statistics.rs‎

Lines changed: 18 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -88,17 +88,18 @@ impl ProbabilityDistribution {
8888
/// Constructs a new [`Generic`] distribution from the given range. Other
8989
/// parameters (mean, median and variance) are initialized with null values.
9090
pub fn new_from_interval(range: Interval) -> Result<Self> {
91-
let null = ScalarValue::try_from(range.data_type())?;
92-
Self::new_generic(null.clone(), null.clone(), null, range)
91+
let scalar_null = ScalarValue::try_new_null(&range.data_type())?;
92+
Self::new_generic(scalar_null.clone(), scalar_null.clone(), scalar_null, range)
9393
}
9494

9595
/// Constructs a new [`Generic`] distribution from the given type and infinity range. Other
9696
/// parameters (mean, median and variance) are initialized with null values.
9797
pub fn new_unknown(data_type: &DataType) -> Result<Self> {
98+
let scalar_null = ScalarValue::try_new_null(data_type)?;
9899
Self::new_generic(
99-
ScalarValue::Null,
100-
ScalarValue::Null,
101-
ScalarValue::Null,
100+
scalar_null.clone(),
101+
scalar_null.clone(),
102+
scalar_null,
102103
Interval::make_non_negative_infinity_interval(data_type)?,
103104
)
104105
}
@@ -289,9 +290,10 @@ impl ProbabilityDistribution {
289290
}
290291

291292
pub fn with_estimated_selectivity(self, selectivity: f64) -> Self {
292-
let value = self.get_value().unwrap_or(&ScalarValue::Null);
293+
let scalar_null = ScalarValue::try_new_null(&self.data_type()).unwrap();
294+
let value = self.get_value().unwrap_or(&scalar_null);
293295
let selectivity = ScalarValue::Float64(Some(selectivity));
294-
let res = value.mul(selectivity).unwrap_or(ScalarValue::Null);
296+
let res = value.mul(selectivity).unwrap_or(scalar_null);
295297
ProbabilityDistribution::new_exact(res).unwrap_or_default()
296298
}
297299

@@ -370,8 +372,9 @@ pub struct GaussianDistribution {
370372

371373
impl GaussianDistribution {
372374
pub fn get_value(&self) -> Option<&ScalarValue> {
373-
if self.variance() == &ScalarValue::new_zero(&DataType::UInt64).unwrap_or(ScalarValue::Null) &&
374-
self.mean() == &ScalarValue::new_zero(&DataType::UInt64).unwrap_or(ScalarValue::Null) {
375+
let scalar_null = ScalarValue::try_from(self.data_type()).unwrap();
376+
if self.variance() == &ScalarValue::new_zero(&DataType::UInt64).unwrap_or(scalar_null.clone()) &&
377+
self.mean() == &ScalarValue::new_zero(&DataType::UInt64).unwrap_or(scalar_null) {
375378
return Some(self.variance());
376379
}
377380
None
@@ -389,7 +392,9 @@ pub struct BernoulliDistribution {
389392

390393
impl BernoulliDistribution {
391394
pub fn get_value(&self) -> Option<&ScalarValue> {
392-
if self.p_value() == &ScalarValue::new_zero(&DataType::UInt64).unwrap_or(ScalarValue::Null) || self.p_value() == &ScalarValue::new_one(&DataType::UInt64).unwrap_or(ScalarValue::Null) {
395+
let scalar_null = ScalarValue::try_from(self.data_type()).unwrap();
396+
if self.p_value() == &ScalarValue::new_zero(&DataType::UInt64).unwrap_or(scalar_null.clone())
397+
|| self.p_value() == &ScalarValue::new_one(&DataType::UInt64).unwrap_or(scalar_null) {
393398
return Some(self.p_value());
394399
}
395400
None
@@ -410,7 +415,8 @@ pub struct GenericDistribution {
410415

411416
impl GenericDistribution {
412417
pub fn get_value(&self) -> Option<&ScalarValue> {
413-
if self.variance() == &ScalarValue::new_zero(&DataType::UInt64).unwrap_or(ScalarValue::Null) &&
418+
let scalar_null = ScalarValue::try_from(self.data_type()).unwrap();
419+
if self.variance() == &ScalarValue::new_zero(&DataType::UInt64).unwrap_or(scalar_null) &&
414420
(self.range().lower() == self.range().upper()) {
415421
return Some(self.range().lower());
416422
}
@@ -1126,7 +1132,7 @@ impl TableStatistics {
11261132
let skip = ScalarValue::UInt64(Some(skip as u64));
11271133
let fetch = fetch.map(|v| v as u64);
11281134
let fetch = ScalarValue::UInt64(fetch);
1129-
let num_rows = self.num_rows.get_value().unwrap_or(&ScalarValue::Null).clone();
1135+
let num_rows = self.num_rows.get_value().unwrap_or(&ScalarValue::UInt64(None)).clone();
11301136

11311137
self.num_rows = if num_rows <= fetch && skip == ScalarValue::new_one(&DataType::UInt64)? {
11321138
self.num_rows

‎datafusion/physical-expr/src/analysis.rs‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -105,8 +105,7 @@ impl ExprBoundaries {
105105
schema.fields.len()
106106
)
107107
})?;
108-
let empty_field =
109-
ScalarValue::try_from(field.data_type()).unwrap_or(ScalarValue::Null);
108+
let empty_field = ScalarValue::try_from(field.data_type())?;
110109
let min_value = col_stats.min_value.get_value().unwrap_or(&empty_field).clone();
111110
let max_value = col_stats.max_value.get_value().unwrap_or(&empty_field).clone();
112111
let interval = Interval::try_new(min_value, max_value)?;

‎datafusion/physical-optimizer/src/enforce_distribution.rs‎

Lines changed: 3 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ use crate::utils::{
3232
};
3333

3434
use arrow::compute::SortOptions;
35+
use arrow::datatypes::DataType;
3536
use datafusion_common::config::ConfigOptions;
3637
use datafusion_common::error::Result;
3738
use datafusion_common::tree_node::{Transformed, TransformedResult, TreeNode};
@@ -1112,14 +1113,10 @@ fn get_repartition_requirement_status(
11121113
{
11131114
// Decide whether adding a round robin is beneficial depending on
11141115
// the statistical information we have on the number of rows:
1115-
// let roundrobin_beneficial_stats = match child.statistics()?.num_rows {
1116-
// Precision::Exact(n_rows) => n_rows > batch_size,
1117-
// Precision::Inexact(n_rows) => !should_use_estimates || (n_rows > batch_size),
1118-
// Precision::Absent => true,
1119-
// };
1116+
let scalar_null = ScalarValue::try_new_null(&DataType::UInt64).unwrap();
11201117
let roundrobin_beneficial_stats = match child.statistics()?.num_rows.get_value() {
11211118
None => true,
1122-
_ => !should_use_estimates || (child.statistics()?.num_rows.get_value().unwrap_or(&ScalarValue::Null) > &ScalarValue::from(batch_size as u64))
1119+
_ => !should_use_estimates || (child.statistics()?.num_rows.get_value().unwrap_or(&scalar_null) > &ScalarValue::from(batch_size as u64))
11231120
};
11241121
let is_hash = matches!(requirement, Distribution::HashPartitioned(_));
11251122
// Hash re-partitioning is necessary when the input has more than one

‎datafusion/physical-optimizer/src/join_selection.rs‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ use std::sync::Arc;
2727

2828
use crate::PhysicalOptimizerRule;
2929

30+
use arrow::datatypes::DataType;
3031
use datafusion_common::config::ConfigOptions;
3132
use datafusion_common::error::Result;
3233
use datafusion_common::tree_node::{Transformed, TransformedResult, TreeNode};
@@ -89,10 +90,11 @@ fn supports_collect_by_thresholds(
8990
return false;
9091
};
9192

92-
let byte_size = stats.total_byte_size.get_value().unwrap_or(&ScalarValue::Null);
93-
let threshold_byte_size= ScalarValue::try_from(threshold_byte_size as u64).unwrap_or(ScalarValue::Null);
94-
let num_rows = stats.num_rows.get_value().unwrap_or(&ScalarValue::Null);
95-
let threshold_num_rows = ScalarValue::try_from(threshold_num_rows as u64).unwrap_or(ScalarValue::Null);
93+
let scalar_null = ScalarValue::try_new_null(&DataType::UInt64).unwrap();
94+
let byte_size = stats.total_byte_size.get_value().unwrap_or(&scalar_null);
95+
let threshold_byte_size= ScalarValue::try_from(threshold_byte_size as u64).unwrap_or(scalar_null.clone());
96+
let num_rows = stats.num_rows.get_value().unwrap_or(&scalar_null);
97+
let threshold_num_rows = ScalarValue::try_from(threshold_num_rows as u64).unwrap_or(scalar_null.clone());
9698
if byte_size.is_null() {
9799
*byte_size < threshold_byte_size
98100
} else if num_rows.is_null() {

‎datafusion/physical-plan/src/aggregates/mod.rs‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -950,10 +950,11 @@ impl ExecutionPlan for AggregateExec {
950950
_ => {
951951
// When the input row count is 0 or 1, we can adopt that statistic keeping its reliability.
952952
// When it is larger than 1, we degrade the precision since it may decrease after aggregation.
953+
let scalar_null = ScalarValue::try_new_null(&DataType::UInt64)?;
953954
let num_rows = self.input().statistics()?.num_rows;
954-
let num_rows = if num_rows.get_value().unwrap_or(&ScalarValue::Null) > &ScalarValue::new_one(&DataType::UInt64)? {
955+
let num_rows = if num_rows.get_value().unwrap_or(&scalar_null) > &ScalarValue::new_one(&DataType::UInt64)? {
955956
self.input().statistics()?.num_rows.to_inexact()?
956-
} else if num_rows.get_value().unwrap_or(&ScalarValue::Null) == &ScalarValue::new_zero(&DataType::UInt64)? {
957+
} else if num_rows.get_value().unwrap_or(&scalar_null) == &ScalarValue::new_zero(&DataType::UInt64)? {
957958
let n_rows = self.input().statistics()?.num_rows;
958959
let to_add = ProbabilityDistribution::new_exact(ScalarValue::new_one(&DataType::UInt64)?)?;
959960
self.input().statistics()?.num_rows = new_generic_from_binary_op(&Operator::Plus, &n_rows, &to_add)?;

‎datafusion/physical-plan/src/joins/cross_join.rs‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -412,7 +412,6 @@ fn stats_cartesian_product(
412412
.into_iter()
413413
.map(|s| {
414414
let null_count = new_generic_from_binary_op(&Operator::Multiply, &s.null_count, &right_row_count).unwrap_or_default();
415-
let sum_value = new_generic_from_binary_op(&Operator::Multiply, &s.sum_value, &right).unwrap_or_default();
416415

417416
ColumnStatistics {
418417
null_count,

‎datafusion/physical-plan/src/joins/utils.rs‎

Lines changed: 15 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -776,7 +776,8 @@ fn estimate_join_cardinality(
776776
}
777777
})
778778
.unzip::<_, _, Vec<_>, Vec<_>>();
779-
779+
780+
let scalar_null = ScalarValue::try_new_null(&DataType::UInt64).unwrap();
780781
match join_type {
781782
JoinType::Inner | JoinType::Left | JoinType::Right | JoinType::Full => {
782783
let total_byte_size = ProbabilityDistribution::new_unknown(&DataType::UInt64).unwrap_or_default();
@@ -809,7 +810,7 @@ fn estimate_join_cardinality(
809810
};
810811

811812
Some(PartialJoinStatistics {
812-
num_rows: cardinality.get_value().unwrap_or(&ScalarValue::Null).clone(),
813+
num_rows: cardinality.get_value().unwrap_or(&scalar_null).clone(),
813814
// We don't do anything specific here, just combine the existing
814815
// statistics which might yield subpar results (although it is
815816
// true, esp regarding min/max). For a better estimation, we need
@@ -836,7 +837,7 @@ fn estimate_join_cardinality(
836837
};
837838

838839
Some(PartialJoinStatistics {
839-
num_rows: cardinality.unwrap_or(ScalarValue::Null),
840+
num_rows: cardinality.unwrap_or(scalar_null),
840841
column_statistics: outer_stats.column_statistics,
841842
})
842843
}
@@ -850,7 +851,7 @@ fn estimate_join_cardinality(
850851
};
851852

852853
Some(PartialJoinStatistics {
853-
num_rows: outer_stats.num_rows.get_value().unwrap_or(&ScalarValue::Null).clone(),
854+
num_rows: outer_stats.num_rows.get_value().unwrap_or(&scalar_null).clone(),
854855
column_statistics: outer_stats.column_statistics,
855856
})
856857
}
@@ -859,7 +860,7 @@ fn estimate_join_cardinality(
859860
let mut column_statistics = left_stats.column_statistics;
860861
column_statistics.push(ColumnStatistics::new_unknown().unwrap_or_default());
861862
Some(PartialJoinStatistics {
862-
num_rows: left_stats.num_rows.get_value().unwrap_or(&ScalarValue::Null).clone(),
863+
num_rows: left_stats.num_rows.get_value().unwrap_or(&scalar_null).clone(),
863864
column_statistics,
864865
})
865866
}
@@ -910,7 +911,8 @@ fn estimate_inner_join_cardinality(
910911
// With the assumption that the smaller input's domain is generally represented in the bigger
911912
// input's domain, we can estimate the inner join's cardinality by taking the cartesian product
912913
// of the two inputs and normalizing it by the selectivity factor.
913-
let value = join_selectivity.get_value().unwrap_or(&ScalarValue::Null);
914+
let null = ScalarValue::try_new_null(&DataType::UInt64).unwrap();
915+
let value = join_selectivity.get_value().unwrap_or(&null);
914916
if value > &ScalarValue::new_one(&DataType::UInt64).unwrap() {
915917
let res = new_generic_from_binary_op(&Operator::Multiply, &left_stats.num_rows, &right_stats.num_rows).unwrap();
916918
Some(new_generic_from_binary_op(&Operator::Divide, &res, &join_selectivity).unwrap())
@@ -938,6 +940,7 @@ fn estimate_disjoint_inputs(
938940
left_stats: &TableStatistics,
939941
right_stats: &TableStatistics,
940942
) -> Option<ProbabilityDistribution> {
943+
let null = ScalarValue::try_new_null(&DataType::UInt64).unwrap();
941944
for (left_stat, right_stat) in left_stats
942945
.column_statistics
943946
.iter()
@@ -946,16 +949,16 @@ fn estimate_disjoint_inputs(
946949
// If there is no overlap in any of the join columns, this means the join
947950
// itself is disjoint and the cardinality is 0. Though we can only assume
948951
// this when the statistics are exact (since it is a very strong assumption).
949-
let left_min_val = left_stat.min_value.get_value().unwrap_or(&ScalarValue::Null);
950-
let right_max_val = right_stat.max_value.get_value().unwrap_or(&ScalarValue::Null);
952+
let left_min_val = left_stat.min_value.get_value().unwrap_or(&null);
953+
let right_max_val = right_stat.max_value.get_value().unwrap_or(&null);
951954
if left_min_val > right_max_val {
952955
return Some(
953956
ProbabilityDistribution::new_zero(&DataType::UInt64).unwrap_or_default()
954957
);
955958
}
956959

957-
let left_max_val = left_stat.max_value.get_value().unwrap_or(&ScalarValue::Null);
958-
let right_min_val = right_stat.min_value.get_value().unwrap_or(&ScalarValue::Null);
960+
let left_max_val = left_stat.max_value.get_value().unwrap_or(&null);
961+
let right_min_val = right_stat.min_value.get_value().unwrap_or(&null);
959962
if left_max_val < right_min_val
960963
{
961964
return Some(
@@ -992,7 +995,8 @@ fn max_distinct_count(
992995
.ok()
993996
.and_then(|e| e.cardinality())
994997
{
995-
return if &ScalarValue::UInt64(Some(range_dc)) < &result.get_value().unwrap_or(&ScalarValue::Null)
998+
let null = ScalarValue::try_new_null(&DataType::UInt64).unwrap();
999+
return if &ScalarValue::UInt64(Some(range_dc)) < &result.get_value().unwrap_or(&null)
9961000
{
9971001
if stats.min_value.is_exact().unwrap()
9981002
&& stats.max_value.is_exact().unwrap()

‎datafusion/physical-plan/src/projection.rs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@ use arrow::record_batch::{RecordBatch, RecordBatchOptions};
4141
use datafusion_common::tree_node::{
4242
Transformed, TransformedResult, TreeNode, TreeNodeRecursion,
4343
};
44-
use datafusion_common::{internal_err, JoinSide, Result};
44+
use datafusion_common::{internal_err, JoinSide, Result, ScalarValue};
4545
use datafusion_execution::TaskContext;
4646
use datafusion_expr::interval_arithmetic::Interval;
4747
use datafusion_expr::statistics::{new_generic_from_binary_op, ColumnStatistics, ProbabilityDistribution, TableStatistics};
@@ -318,7 +318,7 @@ fn stats_projection(
318318
}
319319

320320
if primitive_row_size_possible {
321-
let prim_row_size = ProbabilityDistribution::new_uniform(Interval::make(Some(primitive_row_size as u64), Some(primitive_row_size as u64))?)?;
321+
let prim_row_size = ProbabilityDistribution::new_exact(ScalarValue::UInt64(Some(primitive_row_size as u64)))?;
322322
stats.total_byte_size = new_generic_from_binary_op(&Operator::Multiply, &prim_row_size, &stats.num_rows)?;
323323
}
324324
stats.column_statistics = column_statistics;

0 commit comments

Comments
 (0)