Skip to content

Commit 9d9e912

Browse files
author
Sergey Zhukov
committed
Statistics: Migrate to Distribution from Precision (apache#14896)
1 parent 819eb9d commit 9d9e912

61 files changed

Lines changed: 526 additions & 507 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎datafusion/core/src/datasource/file_format/csv.rs‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,6 @@ mod tests {
3434
use datafusion_catalog::Session;
3535
use datafusion_common::cast::as_string_array;
3636
use datafusion_common::internal_err;
37-
use datafusion_common::stats::Precision;
3837
use datafusion_common::test_util::arrow_test_data;
3938
use datafusion_common::{assert_batches_eq, Result};
4039
use datafusion_datasource::decoder::{

‎datafusion/core/src/datasource/file_format/json.rs‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,6 @@ mod tests {
4242
use arrow::json::ReaderBuilder;
4343
use arrow::util::pretty;
4444
use datafusion_common::cast::as_int64_array;
45-
use datafusion_common::stats::Precision;
4645
use datafusion_common::{assert_batches_eq, internal_err};
4746

4847
use datafusion_common::Result;

‎datafusion/core/src/datasource/listing/table.rs‎

Lines changed: 6 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -40,16 +40,14 @@ use datafusion_expr::statistics::TableStatistics;
4040
use datafusion_expr::{utils::conjunction, Expr, TableProviderFilterPushDown};
4141
use datafusion_expr::{SortExpr, TableType};
4242
use datafusion_physical_plan::empty::EmptyExec;
43-
use datafusion_physical_plan::{ExecutionPlan, Statistics};
43+
use datafusion_physical_plan::ExecutionPlan;
4444

4545
use arrow::datatypes::{DataType, Field, Schema, SchemaBuilder, SchemaRef};
4646
use datafusion_common::{
4747
config_datafusion_err, internal_err, plan_err, project_schema, Constraints,
4848
SchemaExt, ToDFSchema,
4949
};
50-
use datafusion_execution::cache::{
51-
cache_manager::FileStatisticsCache, cache_unit::DefaultFileStatisticsCache,
52-
};
50+
use datafusion_execution::cache::cache_manager::FileStatisticsCache;
5351
use datafusion_physical_expr::{
5452
create_physical_expr, LexOrdering, PhysicalSortRequirement,
5553
};
@@ -1227,10 +1225,10 @@ mod tests {
12271225
assert_eq!(exec.output_partitioning().partition_count(), 1);
12281226

12291227
// test metadata
1230-
let expected_num_rows = ProbabilityDistribution::new_uniform_exact(ScalarValue::UInt64(Some(8)))?;
1231-
let expected_total_byte_size = ProbabilityDistribution::new_uniform_exact(ScalarValue::UInt64(Some(671)))?;
1232-
assert_eq!(exec.statistics()?.num_rows.as_ref(), expected_num_rows.as_ref());
1233-
assert_eq!(exec.statistics()?.total_byte_size.as_ref(), expected_total_byte_size.as_ref());
1228+
let expected_num_rows = ProbabilityDistribution::new_exact(ScalarValue::UInt64(Some(8)))?;
1229+
let expected_total_byte_size = ProbabilityDistribution::new_exact(ScalarValue::UInt64(Some(671)))?;
1230+
assert_eq!(exec.statistics()?.num_rows.get_value(), expected_num_rows.get_value());
1231+
assert_eq!(exec.statistics()?.total_byte_size.get_value(), expected_total_byte_size.get_value());
12341232

12351233
Ok(())
12361234
}

‎datafusion/core/src/datasource/physical_plan/arrow_file.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ use arrow::buffer::Buffer;
2828
use arrow::datatypes::SchemaRef;
2929
use arrow_ipc::reader::FileDecoder;
3030
use datafusion_common::config::ConfigOptions;
31-
use datafusion_common::{Constraints, Statistics};
31+
use datafusion_common::Constraints;
3232
use datafusion_datasource::file::FileSource;
3333
use datafusion_datasource::file_scan_config::FileScanConfig;
3434
use datafusion_datasource::source::DataSourceExec;

‎datafusion/core/src/datasource/statistics.rs‎

Lines changed: 17 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@
1717
use std::sync::Arc;
1818

1919
use arrow_schema::DataType;
20-
use datafusion_expr::statistics::{new_generic_from_binary_op, ColumnStatisticsNew, ProbabilityDistribution, TableStatistics};
20+
use datafusion_expr::statistics::{new_generic_from_binary_op, ColumnStatistics, ProbabilityDistribution, TableStatistics};
2121
use datafusion_expr::Operator;
2222
use futures::{Stream, StreamExt};
2323
use datafusion_common::ScalarValue;
@@ -46,9 +46,9 @@ pub async fn get_statistics_with_limit(
4646
// - zero for summations, and
4747
// - neutral element for extreme points.
4848
let size = file_schema.fields().len();
49-
let mut col_stats_set = vec![ColumnStatisticsNew::new_unknown()?; size];
50-
let mut num_rows = ProbabilityDistribution::new_generic_unknown(&DataType::UInt64)?;
51-
let mut total_byte_size = ProbabilityDistribution::new_generic_unknown(&DataType::UInt64)?;
49+
let mut col_stats_set = vec![ColumnStatistics::new_unknown()?; size];
50+
let mut num_rows = ProbabilityDistribution::new_unknown(&DataType::UInt64)?;
51+
let mut total_byte_size = ProbabilityDistribution::new_unknown(&DataType::UInt64)?;
5252
let limit_value = limit.map(|v| v as _).unwrap_or(u64::MAX);
5353
let limit = ScalarValue::UInt64(Some(limit_value));
5454

@@ -76,12 +76,8 @@ pub async fn get_statistics_with_limit(
7676
// files. This only applies when we know the number of rows. It also
7777
// currently ignores tables that have no statistics regarding the
7878
// number of rows.
79-
let conservative_num_rows = if num_rows.as_ref().is_null() {
80-
ScalarValue::UInt64(Some(u64::MIN))
81-
} else {
82-
num_rows.as_ref().clone()
83-
};
84-
if conservative_num_rows <= limit {
79+
let conservative_num_rows = num_rows.get_value().unwrap_or(&ScalarValue::UInt64(Some(u64::MIN)));
80+
if conservative_num_rows <= &limit {
8581
while let Some(current) = all_files.next().await {
8682
let (mut file, file_stats) = current?;
8783
file.statistics = Some(file_stats.as_ref().clone());
@@ -94,25 +90,25 @@ pub async fn get_statistics_with_limit(
9490
// counts across all the files in question. If any file does not
9591
// provide any information or provides an inexact value, we demote
9692
// the statistic precision to inexact.
97-
num_rows = add_row_stats(file_stats.num_rows.clone(), num_rows)?;
93+
num_rows = new_generic_from_binary_op(&Operator::Plus, &file_stats.num_rows, &num_rows)?;
9894

9995
total_byte_size =
100-
add_row_stats(file_stats.total_byte_size.clone(), total_byte_size)?;
96+
new_generic_from_binary_op(&Operator::Plus, &file_stats.total_byte_size, &total_byte_size)?;
10197

10298
for (file_col_stats, col_stats) in file_stats
10399
.column_statistics
104100
.iter()
105101
.zip(col_stats_set.iter_mut())
106102
{
107-
let ColumnStatisticsNew {
103+
let ColumnStatistics {
108104
null_count: file_nc,
109105
max_value: file_max,
110106
min_value: file_min,
111107
sum_value: file_sum,
112108
distinct_count: _,
113109
} = file_col_stats;
114110

115-
col_stats.null_count = add_row_stats(file_nc.clone(), col_stats.null_count.clone())?;
111+
col_stats.null_count = new_generic_from_binary_op(&Operator::Plus, &file_nc, &col_stats.null_count)?;
116112
set_max_if_greater(file_max, &mut col_stats.max_value);
117113
set_min_if_lesser(file_min, &mut col_stats.min_value);
118114
col_stats.sum_value = new_generic_from_binary_op(&Operator::Plus, file_sum, &col_stats.sum_value)?;
@@ -122,11 +118,7 @@ pub async fn get_statistics_with_limit(
122118
// files. This only applies when we know the number of rows. It also
123119
// currently ignores tables that have no statistics regarding the
124120
// number of rows.
125-
let num_rows = if num_rows.as_ref().is_null() {
126-
&ScalarValue::UInt64(Some(u64::MIN))
127-
} else {
128-
num_rows.as_ref()
129-
};
121+
let num_rows = num_rows.get_value().unwrap_or(&ScalarValue::UInt64(Some(u64::MIN)));
130122
if num_rows > &limit {
131123
break;
132124
}
@@ -149,16 +141,6 @@ pub async fn get_statistics_with_limit(
149141
Ok((result_files, statistics))
150142
}
151143

152-
fn add_row_stats(
153-
file_num_rows: ProbabilityDistribution,
154-
num_row: ProbabilityDistribution,
155-
) -> Result<ProbabilityDistribution> {
156-
match (file_num_rows, &num_row) {
157-
(lhs, rhs) => new_generic_from_binary_op(&Operator::Plus, &lhs, &rhs),
158-
_ => unimplemented!(),
159-
}
160-
}
161-
162144
/// If the given value is numerically greater than the original maximum value,
163145
/// return the new maximum value with appropriate exactness information.
164146
fn set_max_if_greater(
@@ -188,7 +170,9 @@ fn set_max_if_greater(
188170
// }
189171
// _ => {}
190172
// }
191-
todo!()
173+
if max_nominee.get_value().unwrap_or(&ScalarValue::Null) < max_value.get_value().unwrap_or(&ScalarValue::Null) {
174+
*max_value = max_nominee.clone();
175+
}
192176
}
193177

194178
/// If the given value is numerically lesser than the original minimum value,
@@ -220,5 +204,7 @@ fn set_min_if_lesser(
220204
// }
221205
// _ => {}
222206
// }
223-
todo!()
207+
if min_nominee.get_value().unwrap_or(&ScalarValue::Null) > min_value.get_value().unwrap_or(&ScalarValue::Null) {
208+
*min_value = min_nominee.clone();
209+
}
224210
}

‎datafusion/core/tests/custom_sources_cases/mod.rs‎

Lines changed: 33 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -31,15 +31,13 @@ use datafusion::logical_expr::{
3131
col, Expr, LogicalPlan, LogicalPlanBuilder, TableScan, UNNAMED_TABLE,
3232
};
3333
use datafusion::physical_plan::{
34-
collect, ColumnStatistics, DisplayAs, DisplayFormatType, ExecutionPlan, Partitioning,
35-
RecordBatchStream, SendableRecordBatchStream, Statistics,
34+
collect, DisplayAs, DisplayFormatType, ExecutionPlan, Partitioning,
35+
RecordBatchStream, SendableRecordBatchStream,
3636
};
37-
use datafusion::scalar::ScalarValue;
3837
use datafusion_catalog::Session;
3938
use datafusion_common::cast::as_primitive_array;
40-
use datafusion_common::project_schema;
41-
use datafusion_common::stats::Precision;
42-
use datafusion_expr::statistics::TableStatistics;
39+
use datafusion_common::{project_schema, ScalarValue};
40+
use datafusion_expr::statistics::{ColumnStatistics, ProbabilityDistribution, TableStatistics};
4341
use datafusion_physical_expr::EquivalenceProperties;
4442
use datafusion_physical_plan::execution_plan::{Boundedness, EmissionType};
4543
use datafusion_physical_plan::placeholder_row::PlaceholderRowExec;
@@ -181,28 +179,35 @@ impl ExecutionPlan for CustomExecutionPlan {
181179
}
182180

183181
fn statistics(&self) -> Result<TableStatistics> {
184-
// let batch = TEST_CUSTOM_RECORD_BATCH!().unwrap();
185-
// Ok(Statistics {
186-
// num_rows: Precision::Exact(batch.num_rows()),
187-
// total_byte_size: Precision::Absent,
188-
// column_statistics: self
189-
// .projection
190-
// .clone()
191-
// .unwrap_or_else(|| (0..batch.columns().len()).collect())
192-
// .iter()
193-
// .map(|i| ColumnStatistics {
194-
// null_count: Precision::Exact(batch.column(*i).null_count()),
195-
// min_value: Precision::Exact(ScalarValue::Int32(aggregate::min(
196-
// as_primitive_array::<Int32Type>(batch.column(*i)).unwrap(),
197-
// ))),
198-
// max_value: Precision::Exact(ScalarValue::Int32(aggregate::max(
199-
// as_primitive_array::<Int32Type>(batch.column(*i)).unwrap(),
200-
// ))),
201-
// ..Default::default()
202-
// })
203-
// .collect(),
204-
// })
205-
todo!()
182+
let batch = TEST_CUSTOM_RECORD_BATCH!().unwrap();
183+
let n_rows = ScalarValue::UInt64(Some(batch.num_rows() as u64));
184+
let total_byte_size = ProbabilityDistribution::new_unknown(&DataType::UInt64)?;
185+
let num_rows = ProbabilityDistribution::new_exact(n_rows)?;
186+
let column_statistics = self.projection
187+
.clone()
188+
.unwrap_or_else(|| (0..batch.columns().len())
189+
.collect())
190+
.iter()
191+
.map(|i| {
192+
let null_count = ProbabilityDistribution::new_exact(ScalarValue::UInt64(Some(batch.column(*i).null_count() as u64))).unwrap_or_default();
193+
let min_val = aggregate::min(as_primitive_array::<Int32Type>(batch.column(*i)).unwrap());
194+
let min_value = ProbabilityDistribution::new_exact(ScalarValue::Int32(min_val)).unwrap_or_default();
195+
let max_val = ScalarValue::Int32(aggregate::max(as_primitive_array::<Int32Type>(batch.column(*i)).unwrap()));
196+
let max_value = ProbabilityDistribution::new_exact(max_val).unwrap_or_default();
197+
198+
ColumnStatistics {
199+
null_count,
200+
max_value,
201+
min_value,
202+
..Default::default()
203+
}
204+
}).collect();
205+
206+
Ok(TableStatistics {
207+
num_rows,
208+
total_byte_size,
209+
column_statistics,
210+
})
206211
}
207212
}
208213

‎datafusion/core/tests/custom_sources_cases/provider_filter_pushdown.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ use datafusion::logical_expr::TableProviderFilterPushDown;
2929
use datafusion::physical_plan::stream::RecordBatchStreamAdapter;
3030
use datafusion::physical_plan::{
3131
DisplayAs, DisplayFormatType, ExecutionPlan, Partitioning, PlanProperties,
32-
SendableRecordBatchStream, Statistics,
32+
SendableRecordBatchStream,
3333
};
3434
use datafusion::prelude::*;
3535
use datafusion::scalar::ScalarValue;

‎datafusion/core/tests/custom_sources_cases/statistics.rs‎

Lines changed: 20 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -26,15 +26,15 @@ use datafusion::{
2626
error::Result,
2727
logical_expr::Expr,
2828
physical_plan::{
29-
ColumnStatistics, DisplayAs, DisplayFormatType, ExecutionPlan, Partitioning,
29+
DisplayAs, DisplayFormatType, ExecutionPlan, Partitioning,
3030
PlanProperties, SendableRecordBatchStream, Statistics,
3131
},
3232
prelude::SessionContext,
3333
scalar::ScalarValue,
3434
};
3535
use datafusion_catalog::Session;
3636
use datafusion_common::{project_schema, stats::Precision};
37-
use datafusion_expr::statistics::{ColumnStatisticsNew, ProbabilityDistribution, TableStatistics};
37+
use datafusion_expr::statistics::{ColumnStatistics, ProbabilityDistribution, TableStatistics};
3838
use datafusion_physical_expr::EquivalenceProperties;
3939
use datafusion_physical_plan::execution_plan::{Boundedness, EmissionType};
4040

@@ -114,8 +114,8 @@ impl TableProvider for StatisticsValidation {
114114
let proj_col_stats = projection
115115
.iter()
116116
.map(|i| current_stat.column_statistics[*i].clone())
117-
.collect::<Vec<ColumnStatisticsNew>>();
118-
let total_byte_size = ProbabilityDistribution::new_generic_unknown(&DataType::UInt64)?;
117+
.collect::<Vec<ColumnStatistics>>();
118+
let total_byte_size = ProbabilityDistribution::new_unknown(&DataType::UInt64)?;
119119
// Ok(Arc::new(Self::new(
120120
// TableStatistics {
121121
// num_rows: current_stat.num_rows,
@@ -197,25 +197,25 @@ fn init_ctx(stats: TableStatistics, schema: Schema) -> Result<SessionContext> {
197197
Ok(ctx)
198198
}
199199

200-
fn fully_defined() -> (Statistics, Schema) {
200+
fn fully_defined() -> (TableStatistics, Schema) {
201201
(
202-
Statistics {
203-
num_rows: Precision::Exact(13),
204-
total_byte_size: Precision::Absent, // ignore byte size for now
202+
TableStatistics {
203+
num_rows: ProbabilityDistribution::new_exact(ScalarValue::UInt64(Some(13))).unwrap_or_default(),
204+
total_byte_size: ProbabilityDistribution::new_unknown(&DataType::UInt64).unwrap_or_default(),
205205
column_statistics: vec![
206-
ColumnStatistics {
207-
distinct_count: Precision::Exact(2),
208-
max_value: Precision::Exact(ScalarValue::Int32(Some(1023))),
209-
min_value: Precision::Exact(ScalarValue::Int32(Some(-24))),
210-
sum_value: Precision::Exact(ScalarValue::Int64(Some(10))),
211-
null_count: Precision::Exact(0),
206+
ColumnStatistics {
207+
null_count: ProbabilityDistribution::new_exact(ScalarValue::UInt64(Some(2))).unwrap_or_default(),
208+
max_value: ProbabilityDistribution::new_exact(ScalarValue::Int32(Some(1023))).unwrap_or_default(),
209+
min_value: ProbabilityDistribution::new_exact(ScalarValue::Int32(Some(-24))).unwrap_or_default(),
210+
sum_value: ProbabilityDistribution::new_exact(ScalarValue::Int64(Some(10))).unwrap_or_default(),
211+
distinct_count: ProbabilityDistribution::new_zero(&DataType::UInt64).unwrap_or_default(),
212212
},
213-
ColumnStatistics {
214-
distinct_count: Precision::Exact(13),
215-
max_value: Precision::Exact(ScalarValue::Int64(Some(5486))),
216-
min_value: Precision::Exact(ScalarValue::Int64(Some(-6783))),
217-
sum_value: Precision::Exact(ScalarValue::Int64(Some(10))),
218-
null_count: Precision::Exact(5),
213+
ColumnStatistics {
214+
null_count: ProbabilityDistribution::new_exact(ScalarValue::UInt64(Some(13))).unwrap_or_default(),
215+
max_value: ProbabilityDistribution::new_exact(ScalarValue::Int64(Some(5486))).unwrap_or_default(),
216+
min_value: ProbabilityDistribution::new_exact(ScalarValue::Int64(Some(-6783))).unwrap_or_default(),
217+
sum_value: ProbabilityDistribution::new_exact(ScalarValue::Int64(Some(10))).unwrap_or_default(),
218+
distinct_count: ProbabilityDistribution::new_exact(ScalarValue::UInt64(Some(5))).unwrap_or_default(),
219219
},
220220
],
221221
},

‎datafusion/core/tests/parquet/file_statistics.rs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -52,14 +52,14 @@ async fn check_stats_precision_with_filter_pushdown() {
5252
let (_, _, state) = get_cache_runtime_state();
5353
// Scan without filter, stats are exact
5454
let exec = table.scan(&state, None, &[], None).await.unwrap();
55-
let expected = ProbabilityDistribution::new_uniform_exact(ScalarValue::UInt64(Some(8))).unwrap();
55+
let expected = ProbabilityDistribution::new_exact(ScalarValue::UInt64(Some(8))).unwrap();
5656
assert_eq!(exec.statistics().unwrap().num_rows, expected);
5757

5858
// Scan with filter pushdown, stats are inexact
5959
let filter = Expr::gt(col("id"), lit(1));
6060

6161
let exec = table.scan(&state, None, &[filter], None).await.unwrap();
62-
let expected = ProbabilityDistribution::new_uniform_exact(ScalarValue::UInt64(Some(8))).unwrap();
62+
let expected = ProbabilityDistribution::new_exact(ScalarValue::UInt64(Some(8))).unwrap();
6363
assert_eq!(exec.statistics().unwrap().num_rows, expected);
6464
}
6565

0 commit comments

Comments
 (0)