Skip to content

Commit 75d1219

Browse files
author
Sergey Zhukov
committed
Statistics: Migrate to Distribution from Precision (apache#14896)
1 parent ecc720d commit 75d1219

56 files changed

Lines changed: 1208 additions & 602 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎datafusion-examples/examples/advanced_parquet_index.rs‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -498,10 +498,11 @@ impl TableProvider for IndexTableProvider {
498498
// provide the factory to create parquet reader without re-reading metadata
499499
.with_parquet_file_reader_factory(Arc::new(reader_factory)),
500500
);
501-
let file_scan_config = FileScanConfig::new(object_store_url, schema, file_source)?
502-
.with_limit(limit)
503-
.with_projection(projection.cloned())
504-
.with_file(partitioned_file);
501+
let file_scan_config =
502+
FileScanConfig::new(object_store_url, schema, file_source)?
503+
.with_limit(limit)
504+
.with_projection(projection.cloned())
505+
.with_file(partitioned_file);
505506

506507
// Finally, put it all together into a DataSourceExec
507508
Ok(file_scan_config.build())

‎datafusion-examples/examples/custom_file_format.rs‎

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -21,12 +21,8 @@ use arrow::{
2121
array::{AsArray, RecordBatch, StringArray, UInt8Array},
2222
datatypes::{DataType, Field, Schema, SchemaRef, UInt64Type},
2323
};
24-
use datafusion::{logical_expr::statistics::TableStatistics, physical_expr::LexRequirement};
2524
use datafusion::physical_expr::PhysicalExpr;
26-
use datafusion::{
27-
catalog::Session,
28-
common::GetExt,
29-
};
25+
use datafusion::{catalog::Session, common::GetExt};
3026
use datafusion::{
3127
datasource::physical_plan::FileSource, execution::session_state::SessionStateBuilder,
3228
};
@@ -43,6 +39,9 @@ use datafusion::{
4339
physical_plan::ExecutionPlan,
4440
prelude::SessionContext,
4541
};
42+
use datafusion::{
43+
logical_expr::statistics::TableStatistics, physical_expr::LexRequirement,
44+
};
4645
use object_store::{ObjectMeta, ObjectStore};
4746
use tempfile::tempdir;
4847

‎datafusion-examples/examples/expr_api.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,12 +26,12 @@ use datafusion::common::tree_node::{Transformed, TreeNode};
2626
use datafusion::common::DFSchema;
2727
use datafusion::common::{ScalarValue, ToDFSchema};
2828
use datafusion::error::Result;
29-
use datafusion::logical_expr::statistics::{ColumnStatistics, ProbabilityDistribution};
3029
use datafusion::functions_aggregate::first_last::first_value_udaf;
3130
use datafusion::logical_expr::execution_props::ExecutionProps;
3231
use datafusion::logical_expr::expr::BinaryExpr;
3332
use datafusion::logical_expr::interval_arithmetic::Interval;
3433
use datafusion::logical_expr::simplify::SimplifyContext;
34+
use datafusion::logical_expr::statistics::{ColumnStatistics, ProbabilityDistribution};
3535
use datafusion::logical_expr::{ColumnarValue, ExprFunctionExt, ExprSchemable, Operator};
3636
use datafusion::optimizer::analyzer::type_coercion::TypeCoercionRewriter;
3737
use datafusion::optimizer::simplify_expressions::ExprSimplifier;

‎datafusion/core/src/datasource/listing/table.rs‎

Lines changed: 30 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -1222,10 +1222,18 @@ mod tests {
12221222
assert_eq!(exec.output_partitioning().partition_count(), 1);
12231223

12241224
// test metadata
1225-
let expected_num_rows = ProbabilityDistribution::new_exact(ScalarValue::UInt64(Some(8)))?;
1226-
let expected_total_byte_size = ProbabilityDistribution::new_exact(ScalarValue::UInt64(Some(671)))?;
1227-
assert_eq!(exec.statistics()?.num_rows.get_value(), expected_num_rows.get_value());
1228-
assert_eq!(exec.statistics()?.total_byte_size.get_value(), expected_total_byte_size.get_value());
1225+
let expected_num_rows =
1226+
ProbabilityDistribution::new_exact(ScalarValue::UInt64(Some(8)))?;
1227+
let expected_total_byte_size =
1228+
ProbabilityDistribution::new_exact(ScalarValue::UInt64(Some(671)))?;
1229+
assert_eq!(
1230+
exec.statistics()?.num_rows.get_value(),
1231+
expected_num_rows.get_value()
1232+
);
1233+
assert_eq!(
1234+
exec.statistics()?.total_byte_size.get_value(),
1235+
expected_total_byte_size.get_value()
1236+
);
12291237

12301238
Ok(())
12311239
}
@@ -1862,24 +1870,24 @@ mod tests {
18621870
Ok(())
18631871
}
18641872

1865-
// #[cfg(feature = "parquet")]
1866-
// #[tokio::test]
1867-
// async fn test_insert_into_append_1_new_parquet_files_defaults() -> Result<()> {
1868-
// let mut config_map: HashMap<String, String> = HashMap::new();
1869-
// config_map.insert("datafusion.execution.batch_size".into(), "20".into());
1870-
// config_map.insert(
1871-
// "datafusion.execution.soft_max_rows_per_output_file".into(),
1872-
// "20".into(),
1873-
// );
1874-
// helper_test_append_new_files_to_table(
1875-
// ParquetFormat::default().get_ext(),
1876-
// FileCompressionType::UNCOMPRESSED,
1877-
// Some(config_map),
1878-
// 1,
1879-
// )
1880-
// .await?;
1881-
// Ok(())
1882-
// }
1873+
#[cfg(feature = "parquet")]
1874+
#[tokio::test]
1875+
async fn test_insert_into_append_1_new_parquet_files_defaults() -> Result<()> {
1876+
let mut config_map: HashMap<String, String> = HashMap::new();
1877+
config_map.insert("datafusion.execution.batch_size".into(), "20".into());
1878+
config_map.insert(
1879+
"datafusion.execution.soft_max_rows_per_output_file".into(),
1880+
"20".into(),
1881+
);
1882+
helper_test_append_new_files_to_table(
1883+
ParquetFormat::default().get_ext(),
1884+
FileCompressionType::UNCOMPRESSED,
1885+
Some(config_map),
1886+
1,
1887+
)
1888+
.await?;
1889+
Ok(())
1890+
}
18831891

18841892
#[tokio::test]
18851893
async fn test_insert_into_sql_csv_defaults() -> Result<()> {

‎datafusion/core/src/datasource/physical_plan/parquet.rs‎

Lines changed: 53 additions & 53 deletions
Original file line numberDiff line numberDiff line change
@@ -1436,59 +1436,59 @@ mod tests {
14361436
Ok(())
14371437
}
14381438

1439-
// #[tokio::test]
1440-
// async fn test_struct_filter_parquet() -> Result<()> {
1441-
// let tmp_dir = TempDir::new()?;
1442-
// let path = tmp_dir.path().to_str().unwrap().to_string() + "/test.parquet";
1443-
// write_file(&path);
1444-
// let ctx = SessionContext::new();
1445-
// let opt = ListingOptions::new(Arc::new(ParquetFormat::default()));
1446-
// ctx.register_listing_table("base_table", path, opt, None, None)
1447-
// .await
1448-
// .unwrap();
1449-
// let sql = "select * from base_table where name='test02'";
1450-
// let batch = ctx.sql(sql).await.unwrap().collect().await.unwrap();
1451-
// assert_eq!(batch.len(), 1);
1452-
// let expected = [
1453-
// "+---------------------+----+--------+",
1454-
// "| struct | id | name |",
1455-
// "+---------------------+----+--------+",
1456-
// "| {id: 4, name: aaa2} | 2 | test02 |",
1457-
// "+---------------------+----+--------+",
1458-
// ];
1459-
// assert_batches_eq!(expected, &batch);
1460-
// Ok(())
1461-
// }
1462-
1463-
// #[tokio::test]
1464-
// async fn test_struct_filter_parquet_with_view_types() -> Result<()> {
1465-
// let tmp_dir = TempDir::new().unwrap();
1466-
// let path = tmp_dir.path().to_str().unwrap().to_string() + "/test.parquet";
1467-
// write_file(&path);
1468-
1469-
// let ctx = SessionContext::new();
1470-
1471-
// let mut options = TableParquetOptions::default();
1472-
// options.global.schema_force_view_types = true;
1473-
// let opt =
1474-
// ListingOptions::new(Arc::new(ParquetFormat::default().with_options(options)));
1475-
1476-
// ctx.register_listing_table("base_table", path, opt, None, None)
1477-
// .await
1478-
// .unwrap();
1479-
// let sql = "select * from base_table where name='test02'";
1480-
// let batch = ctx.sql(sql).await.unwrap().collect().await.unwrap();
1481-
// assert_eq!(batch.len(), 1);
1482-
// let expected = [
1483-
// "+---------------------+----+--------+",
1484-
// "| struct | id | name |",
1485-
// "+---------------------+----+--------+",
1486-
// "| {id: 4, name: aaa2} | 2 | test02 |",
1487-
// "+---------------------+----+--------+",
1488-
// ];
1489-
// assert_batches_eq!(expected, &batch);
1490-
// Ok(())
1491-
// }
1439+
#[tokio::test]
1440+
async fn test_struct_filter_parquet() -> Result<()> {
1441+
let tmp_dir = TempDir::new()?;
1442+
let path = tmp_dir.path().to_str().unwrap().to_string() + "/test.parquet";
1443+
write_file(&path);
1444+
let ctx = SessionContext::new();
1445+
let opt = ListingOptions::new(Arc::new(ParquetFormat::default()));
1446+
ctx.register_listing_table("base_table", path, opt, None, None)
1447+
.await
1448+
.unwrap();
1449+
let sql = "select * from base_table where name='test02'";
1450+
let batch = ctx.sql(sql).await.unwrap().collect().await.unwrap();
1451+
assert_eq!(batch.len(), 1);
1452+
let expected = [
1453+
"+---------------------+----+--------+",
1454+
"| struct | id | name |",
1455+
"+---------------------+----+--------+",
1456+
"| {id: 4, name: aaa2} | 2 | test02 |",
1457+
"+---------------------+----+--------+",
1458+
];
1459+
assert_batches_eq!(expected, &batch);
1460+
Ok(())
1461+
}
1462+
1463+
#[tokio::test]
1464+
async fn test_struct_filter_parquet_with_view_types() -> Result<()> {
1465+
let tmp_dir = TempDir::new().unwrap();
1466+
let path = tmp_dir.path().to_str().unwrap().to_string() + "/test.parquet";
1467+
write_file(&path);
1468+
1469+
let ctx = SessionContext::new();
1470+
1471+
let mut options = TableParquetOptions::default();
1472+
options.global.schema_force_view_types = true;
1473+
let opt =
1474+
ListingOptions::new(Arc::new(ParquetFormat::default().with_options(options)));
1475+
1476+
ctx.register_listing_table("base_table", path, opt, None, None)
1477+
.await
1478+
.unwrap();
1479+
let sql = "select * from base_table where name='test02'";
1480+
let batch = ctx.sql(sql).await.unwrap().collect().await.unwrap();
1481+
assert_eq!(batch.len(), 1);
1482+
let expected = [
1483+
"+---------------------+----+--------+",
1484+
"| struct | id | name |",
1485+
"+---------------------+----+--------+",
1486+
"| {id: 4, name: aaa2} | 2 | test02 |",
1487+
"+---------------------+----+--------+",
1488+
];
1489+
assert_batches_eq!(expected, &batch);
1490+
Ok(())
1491+
}
14921492

14931493
fn write_file(file: &String) {
14941494
let struct_fields = Fields::from(vec![

‎datafusion/core/src/datasource/statistics.rs‎

Lines changed: 38 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -17,10 +17,12 @@
1717
use std::sync::Arc;
1818

1919
use arrow_schema::DataType;
20-
use datafusion_expr::statistics::{ColumnStatistics, ProbabilityDistribution, TableStatistics};
20+
use datafusion_common::ScalarValue;
21+
use datafusion_expr::statistics::{
22+
ColumnStatistics, ProbabilityDistribution, TableStatistics,
23+
};
2124
use datafusion_expr::Operator;
2225
use futures::{Stream, StreamExt};
23-
use datafusion_common::ScalarValue;
2426

2527
use crate::arrow::datatypes::SchemaRef;
2628
use crate::error::Result;
@@ -76,7 +78,9 @@ pub async fn get_statistics_with_limit(
7678
// files. This only applies when we know the number of rows. It also
7779
// currently ignores tables that have no statistics regarding the
7880
// number of rows.
79-
let conservative_num_rows = num_rows.get_value().unwrap_or(&ScalarValue::UInt64(Some(u64::MIN)));
81+
let conservative_num_rows = num_rows
82+
.get_value()
83+
.unwrap_or(&ScalarValue::UInt64(Some(u64::MIN)));
8084
if conservative_num_rows <= &limit {
8185
while let Some(current) = all_files.next().await {
8286
let (mut file, file_stats) = current?;
@@ -90,11 +94,17 @@ pub async fn get_statistics_with_limit(
9094
// counts across all the files in question. If any file does not
9195
// provide any information or provides an inexact value, we demote
9296
// the statistic precision to inexact.
93-
num_rows =
94-
ProbabilityDistribution::combine_distributions(&Operator::Plus, &file_stats.num_rows, &num_rows)?;
95-
96-
total_byte_size =
97-
ProbabilityDistribution::combine_distributions(&Operator::Plus, &file_stats.total_byte_size, &total_byte_size)?;
97+
num_rows = ProbabilityDistribution::combine_distributions(
98+
&Operator::Plus,
99+
&file_stats.num_rows,
100+
&num_rows,
101+
)?;
102+
103+
total_byte_size = ProbabilityDistribution::combine_distributions(
104+
&Operator::Plus,
105+
&file_stats.total_byte_size,
106+
&total_byte_size,
107+
)?;
98108

99109
for (file_col_stats, col_stats) in file_stats
100110
.column_statistics
@@ -109,19 +119,28 @@ pub async fn get_statistics_with_limit(
109119
distinct_count: _,
110120
} = file_col_stats;
111121

112-
col_stats.null_count =
113-
ProbabilityDistribution::combine_distributions(&Operator::Plus, &file_nc, &col_stats.null_count)?;
122+
col_stats.null_count =
123+
ProbabilityDistribution::combine_distributions(
124+
&Operator::Plus,
125+
&file_nc,
126+
&col_stats.null_count,
127+
)?;
114128
set_max_if_greater(file_max, &mut col_stats.max_value);
115129
set_min_if_lesser(file_min, &mut col_stats.min_value);
116-
col_stats.sum_value =
117-
ProbabilityDistribution::combine_distributions(&Operator::Plus, file_sum, &col_stats.sum_value)?;
130+
col_stats.sum_value = ProbabilityDistribution::combine_distributions(
131+
&Operator::Plus,
132+
file_sum,
133+
&col_stats.sum_value,
134+
)?;
118135
}
119136

120137
// If the number of rows exceeds the limit, we can stop processing
121138
// files. This only applies when we know the number of rows. It also
122139
// currently ignores tables that have no statistics regarding the
123140
// number of rows.
124-
let num_rows = num_rows.get_value().unwrap_or(&ScalarValue::UInt64(Some(u64::MIN)));
141+
let num_rows = num_rows
142+
.get_value()
143+
.unwrap_or(&ScalarValue::UInt64(Some(u64::MIN)));
125144
if num_rows > &limit {
126145
break;
127146
}
@@ -151,7 +170,9 @@ fn set_max_if_greater(
151170
max_value: &mut ProbabilityDistribution,
152171
) {
153172
let scalar_null = ScalarValue::try_new_null(&DataType::UInt64).unwrap();
154-
if max_nominee.get_value().unwrap_or(&scalar_null) < max_value.get_value().unwrap_or(&scalar_null) {
173+
if max_nominee.get_value().unwrap_or(&scalar_null)
174+
< max_value.get_value().unwrap_or(&scalar_null)
175+
{
155176
*max_value = max_nominee.clone();
156177
}
157178
}
@@ -163,7 +184,9 @@ fn set_min_if_lesser(
163184
min_value: &mut ProbabilityDistribution,
164185
) {
165186
let scalar_null = ScalarValue::try_new_null(&DataType::UInt64).unwrap();
166-
if min_nominee.get_value().unwrap_or(&scalar_null) > min_value.get_value().unwrap_or(&scalar_null) {
187+
if min_nominee.get_value().unwrap_or(&scalar_null)
188+
> min_value.get_value().unwrap_or(&scalar_null)
189+
{
167190
*min_value = min_nominee.clone();
168191
}
169192
}

0 commit comments

Comments
 (0)