Skip to content

Commit 006ac9b

Browse files
author
Sergey Zhukov
committed
Statistics: Migrate to Distribution from Precision (apache#14896)
1 parent 14ce193 commit 006ac9b

10 files changed

Lines changed: 286 additions & 122 deletions

File tree

‎datafusion/common/src/stats.rs‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,9 @@ use arrow::datatypes::{DataType, Schema, SchemaRef};
2525

2626
/// Represents a value with a degree of certainty. `Precision` is used to
2727
/// propagate information the precision of statistical values.
28+
/// // #[deprecated(
29+
// note = "Precision will be replaced by Distribution in future updates"
30+
// )]
2831
#[derive(Clone, PartialEq, Eq, Default, Copy)]
2932
pub enum Precision<T: Debug + Clone + PartialEq + Eq + PartialOrd> {
3033
/// The exact value is known

‎datafusion/core/src/datasource/listing/table.rs‎

Lines changed: 32 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@ use datafusion_catalog::TableProvider;
3636
use datafusion_common::{config_err, DataFusionError, Result};
3737
use datafusion_datasource::file_scan_config::FileScanConfig;
3838
use datafusion_expr::dml::InsertOp;
39+
use datafusion_expr::statistics::StatisticsNew;
3940
use datafusion_expr::{utils::conjunction, Expr, TableProviderFilterPushDown};
4041
use datafusion_expr::{SortExpr, TableType};
4142
use datafusion_physical_plan::empty::EmptyExec;
@@ -948,7 +949,7 @@ impl TableProvider for ListingTable {
948949
)
949950
.with_file_groups(partitioned_file_lists)
950951
.with_constraints(self.constraints.clone())
951-
.with_statistics(statistics)
952+
// .with_statistics(statistics)
952953
.with_projection(projection.cloned())
953954
.with_limit(limit)
954955
.with_output_ordering(output_ordering)
@@ -1087,11 +1088,11 @@ impl ListingTable {
10871088
ctx: &'a dyn Session,
10881089
filters: &'a [Expr],
10891090
limit: Option<usize>,
1090-
) -> Result<(Vec<Vec<PartitionedFile>>, Statistics)> {
1091+
) -> Result<(Vec<Vec<PartitionedFile>>, StatisticsNew)> {
10911092
let store = if let Some(url) = self.table_paths.first() {
10921093
ctx.runtime_env().object_store(url)?
10931094
} else {
1094-
return Ok((vec![], Statistics::new_unknown(&self.file_schema)));
1095+
return Ok((vec![], StatisticsNew::new_unknown(&self.file_schema)?));
10951096
};
10961097
// list files (with partitions)
10971098
let file_list = future::try_join_all(self.table_paths.iter().map(|table_path| {
@@ -1119,7 +1120,7 @@ impl ListingTable {
11191120
} else {
11201121
Ok((
11211122
part_file,
1122-
Arc::new(Statistics::new_unknown(&self.file_schema)),
1123+
Arc::new(StatisticsNew::new_unknown(&self.file_schema)?),
11231124
))
11241125
}
11251126
})
@@ -1150,32 +1151,33 @@ impl ListingTable {
11501151
ctx: &dyn Session,
11511152
store: &Arc<dyn ObjectStore>,
11521153
part_file: &PartitionedFile,
1153-
) -> Result<Arc<Statistics>> {
1154-
match self
1155-
.collected_statistics
1156-
.get_with_extra(&part_file.object_meta.location, &part_file.object_meta)
1157-
{
1158-
Some(statistics) => Ok(statistics),
1159-
None => {
1160-
let statistics = self
1161-
.options
1162-
.format
1163-
.infer_stats(
1164-
ctx,
1165-
store,
1166-
Arc::clone(&self.file_schema),
1167-
&part_file.object_meta,
1168-
)
1169-
.await?;
1170-
let statistics = Arc::new(statistics);
1171-
self.collected_statistics.put_with_extra(
1172-
&part_file.object_meta.location,
1173-
Arc::clone(&statistics),
1174-
&part_file.object_meta,
1175-
);
1176-
Ok(statistics)
1177-
}
1178-
}
1154+
) -> Result<Arc<StatisticsNew>> {
1155+
// match self
1156+
// .collected_statistics
1157+
// .get_with_extra(&part_file.object_meta.location, &part_file.object_meta)
1158+
// {
1159+
// Some(statistics) => Ok(statistics),
1160+
// None => {
1161+
// let statistics = self
1162+
// .options
1163+
// .format
1164+
// .infer_stats(
1165+
// ctx,
1166+
// store,
1167+
// Arc::clone(&self.file_schema),
1168+
// &part_file.object_meta,
1169+
// )
1170+
// .await?;
1171+
// let statistics = Arc::new(statistics);
1172+
// self.collected_statistics.put_with_extra(
1173+
// &part_file.object_meta.location,
1174+
// Arc::clone(&statistics),
1175+
// &part_file.object_meta,
1176+
// );
1177+
// Ok(statistics)
1178+
// }
1179+
// }
1180+
todo!()
11791181
}
11801182
}
11811183

‎datafusion/core/src/datasource/statistics.rs‎

Lines changed: 92 additions & 73 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,10 @@
1818
use std::mem;
1919
use std::sync::Arc;
2020

21+
use arrow_schema::DataType;
22+
use datafusion_expr::interval_arithmetic::Interval;
23+
use datafusion_expr::statistics::{new_generic_from_binary_op, ColumnStatisticsNew, Distribution, StatisticsNew};
24+
use datafusion_expr::Operator;
2125
use futures::{Stream, StreamExt};
2226

2327
use datafusion_common::stats::Precision;
@@ -35,11 +39,11 @@ use super::listing::PartitionedFile;
3539
/// `ListingTable`. If it is false we only construct bare statistics and skip a potentially expensive
3640
/// call to `multiunzip` for constructing file level summary statistics.
3741
pub async fn get_statistics_with_limit(
38-
all_files: impl Stream<Item = Result<(PartitionedFile, Arc<Statistics>)>>,
42+
all_files: impl Stream<Item = Result<(PartitionedFile, Arc<StatisticsNew>)>>,
3943
file_schema: SchemaRef,
4044
limit: Option<usize>,
4145
collect_stats: bool,
42-
) -> Result<(Vec<PartitionedFile>, Statistics)> {
46+
) -> Result<(Vec<PartitionedFile>, StatisticsNew)> {
4347
let mut result_files = vec![];
4448
// These statistics can be calculated as long as at least one file provides
4549
// useful information. If none of the files provides any information, then
@@ -48,9 +52,23 @@ pub async fn get_statistics_with_limit(
4852
// - zero for summations, and
4953
// - neutral element for extreme points.
5054
let size = file_schema.fields().len();
51-
let mut col_stats_set = vec![ColumnStatistics::default(); size];
52-
let mut num_rows = Precision::<usize>::Absent;
53-
let mut total_byte_size = Precision::<usize>::Absent;
55+
let mut col_stats_set = vec![ColumnStatisticsNew::new_unknown()?; size];
56+
let mut num_rows = Distribution::new_generic(
57+
ScalarValue::Null,
58+
ScalarValue::Null,
59+
ScalarValue::Null,
60+
Interval::make_zero(&DataType::UInt64)?,
61+
)?;
62+
let mut total_byte_size = Distribution::new_generic(
63+
ScalarValue::Null,
64+
ScalarValue::Null,
65+
ScalarValue::Null,
66+
Interval::make_zero(&DataType::UInt64)?,
67+
)?;
68+
let limit = match limit {
69+
Some(v) => ScalarValue::UInt64(Some(v as u64)),
70+
None => ScalarValue::UInt64(Some(u64::MAX)),
71+
};
5472

5573
// Fusing the stream allows us to call next safely even once it is finished.
5674
let mut all_files = Box::pin(all_files.fuse());
@@ -77,10 +95,10 @@ pub async fn get_statistics_with_limit(
7795
// currently ignores tables that have no statistics regarding the
7896
// number of rows.
7997
let conservative_num_rows = match num_rows {
80-
Precision::Exact(nr) => nr,
81-
_ => usize::MIN,
98+
Distribution::Uniform(ref dist) => dist.get_value().unwrap_or(&ScalarValue::UInt64(Some(u64::MIN))).clone(),
99+
_ => ScalarValue::UInt64(Some(u64::MIN)),
82100
};
83-
if conservative_num_rows <= limit.unwrap_or(usize::MAX) {
101+
if conservative_num_rows <= limit {
84102
while let Some(current) = all_files.next().await {
85103
let (mut file, file_stats) = current?;
86104
file.statistics = Some(file_stats.as_ref().clone());
@@ -103,7 +121,7 @@ pub async fn get_statistics_with_limit(
103121
.iter()
104122
.zip(col_stats_set.iter_mut())
105123
{
106-
let ColumnStatistics {
124+
let ColumnStatisticsNew {
107125
null_count: file_nc,
108126
max_value: file_max,
109127
min_value: file_min,
@@ -114,23 +132,21 @@ pub async fn get_statistics_with_limit(
114132
col_stats.null_count = add_row_stats(*file_nc, col_stats.null_count);
115133
set_max_if_greater(file_max, &mut col_stats.max_value);
116134
set_min_if_lesser(file_min, &mut col_stats.min_value);
117-
col_stats.sum_value = file_sum.add(&col_stats.sum_value);
135+
col_stats.sum_value = new_generic_from_binary_op(&Operator::Plus, file_sum, &col_stats.sum_value)?;
118136
}
119137

120138
// If the number of rows exceeds the limit, we can stop processing
121139
// files. This only applies when we know the number of rows. It also
122140
// currently ignores tables that have no statistics regarding the
123141
// number of rows.
124-
if num_rows.get_value().unwrap_or(&usize::MIN)
125-
> &limit.unwrap_or(usize::MAX)
126-
{
142+
if num_rows.get_value().unwrap_or(&ScalarValue::UInt64(Some(u64::MIN))) > &limit {
127143
break;
128144
}
129145
}
130146
}
131147
};
132148

133-
let mut statistics = Statistics {
149+
let mut statistics = StatisticsNew {
134150
num_rows,
135151
total_byte_size,
136152
column_statistics: col_stats_set,
@@ -139,81 +155,84 @@ pub async fn get_statistics_with_limit(
139155
// If we still have files in the stream, it means that the limit kicked
140156
// in, and the statistic could have been different had we processed the
141157
// files in a different order.
142-
statistics = statistics.to_inexact()
158+
statistics = statistics.to_inexact()?
143159
}
144160

145161
Ok((result_files, statistics))
146162
}
147163

148164
fn add_row_stats(
149-
file_num_rows: Precision<usize>,
150-
num_rows: Precision<usize>,
151-
) -> Precision<usize> {
152-
match (file_num_rows, &num_rows) {
153-
(Precision::Absent, _) => num_rows.to_inexact(),
154-
(lhs, Precision::Absent) => lhs.to_inexact(),
155-
(lhs, rhs) => lhs.add(rhs),
156-
}
165+
file_num_rows: Distribution,
166+
num_row: Distribution,
167+
) -> Distribution {
168+
// match (file_num_rows, &num_rows) {
169+
// (Precision::Absent, _) => num_rows.to_inexact(),
170+
// (lhs, Precision::Absent) => lhs.to_inexact(),
171+
// (lhs, rhs) => lhs.add(rhs),
172+
// }
173+
todo!()
157174
}
158175

159176
/// If the given value is numerically greater than the original maximum value,
160177
/// return the new maximum value with appropriate exactness information.
161178
fn set_max_if_greater(
162-
max_nominee: &Precision<ScalarValue>,
163-
max_value: &mut Precision<ScalarValue>,
179+
max_nominee: &Distribution,
180+
max_value: &mut Distribution,
164181
) {
165-
match (&max_value, max_nominee) {
166-
(Precision::Exact(val1), Precision::Exact(val2)) if val1 < val2 => {
167-
*max_value = max_nominee.clone();
168-
}
169-
(Precision::Exact(val1), Precision::Inexact(val2))
170-
| (Precision::Inexact(val1), Precision::Inexact(val2))
171-
| (Precision::Inexact(val1), Precision::Exact(val2))
172-
if val1 < val2 =>
173-
{
174-
*max_value = max_nominee.clone().to_inexact();
175-
}
176-
(Precision::Exact(_), Precision::Absent) => {
177-
let exact_max = mem::take(max_value);
178-
*max_value = exact_max.to_inexact();
179-
}
180-
(Precision::Absent, Precision::Exact(_)) => {
181-
*max_value = max_nominee.clone().to_inexact();
182-
}
183-
(Precision::Absent, Precision::Inexact(_)) => {
184-
*max_value = max_nominee.clone();
185-
}
186-
_ => {}
187-
}
182+
// match (&max_value, max_nominee) {
183+
// (Precision::Exact(val1), Precision::Exact(val2)) if val1 < val2 => {
184+
// *max_value = max_nominee.clone();
185+
// }
186+
// (Precision::Exact(val1), Precision::Inexact(val2))
187+
// | (Precision::Inexact(val1), Precision::Inexact(val2))
188+
// | (Precision::Inexact(val1), Precision::Exact(val2))
189+
// if val1 < val2 =>
190+
// {
191+
// *max_value = max_nominee.clone().to_inexact();
192+
// }
193+
// (Precision::Exact(_), Precision::Absent) => {
194+
// let exact_max = mem::take(max_value);
195+
// *max_value = exact_max.to_inexact();
196+
// }
197+
// (Precision::Absent, Precision::Exact(_)) => {
198+
// *max_value = max_nominee.clone().to_inexact();
199+
// }
200+
// (Precision::Absent, Precision::Inexact(_)) => {
201+
// *max_value = max_nominee.clone();
202+
// }
203+
// _ => {}
204+
// }
205+
todo!()
188206
}
189207

190208
/// If the given value is numerically lesser than the original minimum value,
191209
/// return the new minimum value with appropriate exactness information.
192210
fn set_min_if_lesser(
193-
min_nominee: &Precision<ScalarValue>,
194-
min_value: &mut Precision<ScalarValue>,
211+
min_nominee: &Distribution,
212+
min_value: &mut Distribution,
195213
) {
196-
match (&min_value, min_nominee) {
197-
(Precision::Exact(val1), Precision::Exact(val2)) if val1 > val2 => {
198-
*min_value = min_nominee.clone();
199-
}
200-
(Precision::Exact(val1), Precision::Inexact(val2))
201-
| (Precision::Inexact(val1), Precision::Inexact(val2))
202-
| (Precision::Inexact(val1), Precision::Exact(val2))
203-
if val1 > val2 =>
204-
{
205-
*min_value = min_nominee.clone().to_inexact();
206-
}
207-
(Precision::Exact(_), Precision::Absent) => {
208-
let exact_min = mem::take(min_value);
209-
*min_value = exact_min.to_inexact();
210-
}
211-
(Precision::Absent, Precision::Exact(_)) => {
212-
*min_value = min_nominee.clone().to_inexact();
213-
}
214-
(Precision::Absent, Precision::Inexact(_)) => {
215-
*min_value = min_nominee.clone();
216-
}
217-
_ => {}
218-
}
214+
// match (&min_value, min_nominee) {
215+
// (Precision::Exact(val1), Precision::Exact(val2)) if val1 > val2 => {
216+
// *min_value = min_nominee.clone();
217+
// }
218+
// (Precision::Exact(val1), Precision::Inexact(val2))
219+
// | (Precision::Inexact(val1), Precision::Inexact(val2))
220+
// | (Precision::Inexact(val1), Precision::Exact(val2))
221+
// if val1 > val2 =>
222+
// {
223+
// *min_value = min_nominee.clone().to_inexact();
224+
// }
225+
// (Precision::Exact(_), Precision::Absent) => {
226+
// let exact_min = mem::take(min_value);
227+
// *min_value = exact_min.to_inexact();
228+
// }
229+
// (Precision::Absent, Precision::Exact(_)) => {
230+
// *min_value = min_nominee.clone().to_inexact();
231+
// }
232+
// (Precision::Absent, Precision::Inexact(_)) => {
233+
// *min_value = min_nominee.clone();
234+
// }
235+
// _ => {}
236+
// }
237+
todo!()
219238
}

‎datafusion/datasource/src/file.rs‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ use crate::file_scan_config::FileScanConfig;
2727
use crate::file_stream::FileOpener;
2828
use arrow::datatypes::SchemaRef;
2929
use datafusion_common::Statistics;
30+
use datafusion_expr::statistics::StatisticsNew;
3031
use datafusion_physical_expr::LexOrdering;
3132
use datafusion_physical_plan::metrics::ExecutionPlanMetricsSet;
3233
use datafusion_physical_plan::DisplayFormatType;
@@ -53,11 +54,11 @@ pub trait FileSource: Send + Sync {
5354
/// Initialize new instance with projection information
5455
fn with_projection(&self, config: &FileScanConfig) -> Arc<dyn FileSource>;
5556
/// Initialize new instance with projected statistics
56-
fn with_statistics(&self, statistics: Statistics) -> Arc<dyn FileSource>;
57+
fn with_statistics(&self, statistics: StatisticsNew) -> Arc<dyn FileSource>;
5758
/// Return execution plan metrics
5859
fn metrics(&self) -> &ExecutionPlanMetricsSet;
5960
/// Return projected statistics
60-
fn statistics(&self) -> datafusion_common::Result<Statistics>;
61+
fn statistics(&self) -> datafusion_common::Result<StatisticsNew>;
6162
/// String representation of file source such as "csv", "json", "parquet"
6263
fn file_type(&self) -> &str;
6364
/// Format FileType specific information

0 commit comments

Comments
 (0)