Skip to content

Commit efa5edb

Browse files
author
Sergey Zhukov
committed
Statistics: Migrate to Distribution from Precision (apache#14896)
1 parent 7ac3675 commit efa5edb

1 file changed

Lines changed: 32 additions & 53 deletions

File tree

  • datafusion/physical-plan/src/joins

‎datafusion/physical-plan/src/joins/utils.rs‎

Lines changed: 32 additions & 53 deletions
Original file line numberDiff line numberDiff line change
@@ -975,60 +975,39 @@ fn max_distinct_count(
975975
num_rows: &ProbabilityDistribution,
976976
stats: &ColumnStatistics,
977977
) -> ProbabilityDistribution {
978-
// match &stats.distinct_count {
979-
// &dc @ (Precision::Exact(_) | Precision::Inexact(_)) => dc,
980-
// _ => {
981-
// // The number can never be greater than the number of rows we have
982-
// // minus the nulls (since they don't count as distinct values).
983-
// let result = match num_rows {
984-
// Precision::Absent => Precision::Absent,
985-
// Precision::Inexact(count) => {
986-
// // To safeguard against inexact number of rows (e.g. 0) being smaller than
987-
// // an exact null count we need to do a checked subtraction.
988-
// match count.checked_sub(*stats.null_count.get_value().unwrap_or(&0)) {
989-
// None => Precision::Inexact(0),
990-
// Some(non_null_count) => Precision::Inexact(non_null_count),
991-
// }
992-
// }
993-
// Precision::Exact(count) => {
994-
// let count = count - stats.null_count.get_value().unwrap_or(&0);
995-
// if stats.null_count.is_exact().unwrap_or(false) {
996-
// Precision::Exact(count)
997-
// } else {
998-
// Precision::Inexact(count)
999-
// }
1000-
// }
1001-
// };
1002-
// // Cap the estimate using the number of possible values:
1003-
// if let (Some(min), Some(max)) =
1004-
// (stats.min_value.get_value(), stats.max_value.get_value())
1005-
// {
1006-
// if let Some(range_dc) = Interval::try_new(min.clone(), max.clone())
1007-
// .ok()
1008-
// .and_then(|e| e.cardinality())
1009-
// {
1010-
// let range_dc = range_dc as usize;
1011-
// // Note that the `unwrap` calls in the below statement are safe.
1012-
// return if matches!(result, Precision::Absent)
1013-
// || &range_dc < result.get_value().unwrap()
1014-
// {
1015-
// if stats.min_value.is_exact().unwrap()
1016-
// && stats.max_value.is_exact().unwrap()
1017-
// {
1018-
// Precision::Exact(range_dc)
1019-
// } else {
1020-
// Precision::Inexact(range_dc)
1021-
// }
1022-
// } else {
1023-
// result
1024-
// };
1025-
// }
1026-
// }
978+
let result = match num_rows.get_value() {
979+
None => ProbabilityDistribution::new_unknown(&DataType::UInt64).unwrap_or_default(),
980+
Some(val) => {
981+
let zero = ScalarValue::new_zero(&DataType::UInt64).unwrap();
982+
let null_count = stats.null_count.get_value().unwrap_or(&zero);
983+
let count = val.sub(null_count).unwrap();
984+
ProbabilityDistribution::new_exact(count).unwrap_or_default()
985+
},
986+
};
1027987

1028-
// result
1029-
// }
1030-
// }
1031-
todo!()
988+
if let (Some(min), Some(max)) =
989+
(stats.min_value.get_value(), stats.max_value.get_value())
990+
{
991+
if let Some(range_dc) = Interval::try_new(min.clone(), max.clone())
992+
.ok()
993+
.and_then(|e| e.cardinality())
994+
{
995+
return if &ScalarValue::UInt64(Some(range_dc)) < &result.get_value().unwrap_or(&ScalarValue::Null)
996+
{
997+
if stats.min_value.is_exact().unwrap()
998+
&& stats.max_value.is_exact().unwrap()
999+
{
1000+
ProbabilityDistribution::new_exact(ScalarValue::UInt64(Some(range_dc))).unwrap_or_default()
1001+
} else {
1002+
ProbabilityDistribution::new_unknown(&DataType::UInt64).unwrap_or_default()
1003+
}
1004+
} else {
1005+
result
1006+
};
1007+
}
1008+
}
1009+
1010+
result
10321011
}
10331012

10341013
enum OnceFutState<T> {

0 commit comments

Comments
 (0)