Skip to content

Commit 28c2d5b

Browse files
author
Sergey Zhukov
committed
Statistics: Migrate to Distribution from Precision (apache#14896)
1 parent 5e23a5b commit 28c2d5b

5 files changed

Lines changed: 65 additions & 66 deletions

File tree

‎datafusion/core/src/datasource/listing/table.rs‎

Lines changed: 34 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -1807,41 +1807,41 @@ mod tests {
18071807
Ok(())
18081808
}
18091809

1810-
// #[tokio::test]
1811-
// async fn test_insert_into_append_new_json_files() -> Result<()> {
1812-
// let mut config_map: HashMap<String, String> = HashMap::new();
1813-
// config_map.insert("datafusion.execution.batch_size".into(), "10".into());
1814-
// config_map.insert(
1815-
// "datafusion.execution.soft_max_rows_per_output_file".into(),
1816-
// "10".into(),
1817-
// );
1818-
// helper_test_append_new_files_to_table(
1819-
// JsonFormat::default().get_ext(),
1820-
// FileCompressionType::UNCOMPRESSED,
1821-
// Some(config_map),
1822-
// 2,
1823-
// )
1824-
// .await?;
1825-
// Ok(())
1826-
// }
1810+
#[tokio::test]
1811+
async fn test_insert_into_append_new_json_files() -> Result<()> {
1812+
let mut config_map: HashMap<String, String> = HashMap::new();
1813+
config_map.insert("datafusion.execution.batch_size".into(), "10".into());
1814+
config_map.insert(
1815+
"datafusion.execution.soft_max_rows_per_output_file".into(),
1816+
"10".into(),
1817+
);
1818+
helper_test_append_new_files_to_table(
1819+
JsonFormat::default().get_ext(),
1820+
FileCompressionType::UNCOMPRESSED,
1821+
Some(config_map),
1822+
2,
1823+
)
1824+
.await?;
1825+
Ok(())
1826+
}
18271827

1828-
// #[tokio::test]
1829-
// async fn test_insert_into_append_new_csv_files() -> Result<()> {
1830-
// let mut config_map: HashMap<String, String> = HashMap::new();
1831-
// config_map.insert("datafusion.execution.batch_size".into(), "10".into());
1832-
// config_map.insert(
1833-
// "datafusion.execution.soft_max_rows_per_output_file".into(),
1834-
// "10".into(),
1835-
// );
1836-
// helper_test_append_new_files_to_table(
1837-
// CsvFormat::default().get_ext(),
1838-
// FileCompressionType::UNCOMPRESSED,
1839-
// Some(config_map),
1840-
// 2,
1841-
// )
1842-
// .await?;
1843-
// Ok(())
1844-
// }
1828+
#[tokio::test]
1829+
async fn test_insert_into_append_new_csv_files() -> Result<()> {
1830+
let mut config_map: HashMap<String, String> = HashMap::new();
1831+
config_map.insert("datafusion.execution.batch_size".into(), "10".into());
1832+
config_map.insert(
1833+
"datafusion.execution.soft_max_rows_per_output_file".into(),
1834+
"10".into(),
1835+
);
1836+
helper_test_append_new_files_to_table(
1837+
CsvFormat::default().get_ext(),
1838+
FileCompressionType::UNCOMPRESSED,
1839+
Some(config_map),
1840+
2,
1841+
)
1842+
.await?;
1843+
Ok(())
1844+
}
18451845

18461846
#[cfg(feature = "parquet")]
18471847
#[tokio::test]

‎datafusion/core/src/execution/context/csv.rs‎

Lines changed: 28 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -94,36 +94,36 @@ mod tests {
9494

9595
use tempfile::TempDir;
9696

97-
// #[tokio::test]
98-
// async fn query_csv_with_custom_partition_extension() -> Result<()> {
99-
// let tmp_dir = TempDir::new()?;
97+
#[tokio::test]
98+
async fn query_csv_with_custom_partition_extension() -> Result<()> {
99+
let tmp_dir = TempDir::new()?;
100100

101-
// // The main stipulation of this test: use a file extension that isn't .csv.
102-
// let file_extension = ".tst";
101+
// The main stipulation of this test: use a file extension that isn't .csv.
102+
let file_extension = ".tst";
103103

104-
// let ctx = SessionContext::new();
105-
// let schema = populate_csv_partitions(&tmp_dir, 2, file_extension)?;
106-
// ctx.register_csv(
107-
// "test",
108-
// tmp_dir.path().to_str().unwrap(),
109-
// CsvReadOptions::new()
110-
// .schema(&schema)
111-
// .file_extension(file_extension),
112-
// )
113-
// .await?;
114-
// let results =
115-
// plan_and_collect(&ctx, "SELECT sum(c1), sum(c2), count(*) FROM test").await?;
104+
let ctx = SessionContext::new();
105+
let schema = populate_csv_partitions(&tmp_dir, 2, file_extension)?;
106+
ctx.register_csv(
107+
"test",
108+
tmp_dir.path().to_str().unwrap(),
109+
CsvReadOptions::new()
110+
.schema(&schema)
111+
.file_extension(file_extension),
112+
)
113+
.await?;
114+
let results =
115+
plan_and_collect(&ctx, "SELECT sum(c1), sum(c2), count(*) FROM test").await?;
116116

117-
// assert_eq!(results.len(), 1);
118-
// let expected = [
119-
// "+--------------+--------------+----------+",
120-
// "| sum(test.c1) | sum(test.c2) | count(*) |",
121-
// "+--------------+--------------+----------+",
122-
// "| 10 | 110 | 20 |",
123-
// "+--------------+--------------+----------+",
124-
// ];
125-
// assert_batches_eq!(expected, &results);
117+
assert_eq!(results.len(), 1);
118+
let expected = [
119+
"+--------------+--------------+----------+",
120+
"| sum(test.c1) | sum(test.c2) | count(*) |",
121+
"+--------------+--------------+----------+",
122+
"| 10 | 110 | 20 |",
123+
"+--------------+--------------+----------+",
124+
];
125+
assert_batches_eq!(expected, &results);
126126

127-
// Ok(())
128-
// }
127+
Ok(())
128+
}
129129
}

‎datafusion/core/tests/custom_sources_cases/statistics.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -115,7 +115,7 @@ impl TableProvider for StatisticsValidation {
115115
.iter()
116116
.map(|i| current_stat.column_statistics[*i].clone())
117117
.collect::<Vec<ColumnStatistics>>();
118-
let total_byte_size = ProbabilityDistribution::new_unknown(&DataType::UInt64)?;
118+
let total_byte_size = ProbabilityDistribution::new_unknown(&current_stat.total_byte_size.data_type())?;
119119
Ok(Arc::new(Self::new(
120120
TableStatistics {
121121
num_rows: current_stat.num_rows,

‎datafusion/datasource-parquet/src/file_format.rs‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -756,7 +756,6 @@ pub fn statistics_from_parquet_meta_calc(
756756
}
757757
Err(e) => {
758758
debug!("Failed to create statistics converter: {}", e);
759-
// null_counts_array[idx] = Precision::Exact(num_rows);
760759
let value = ScalarValue::UInt64(Some(num_rows as u64));
761760
null_counts_array[idx] = ProbabilityDistribution::new_exact(value).unwrap();
762761
}

‎datafusion/physical-plan/src/joins/utils.rs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -955,7 +955,7 @@ fn estimate_disjoint_inputs(
955955
let right_max_val = right_stat.max_value.get_value().unwrap_or(&null);
956956
if left_min_val > right_max_val {
957957
return Some(
958-
ProbabilityDistribution::new_zero(&DataType::UInt64).unwrap_or_default()
958+
ProbabilityDistribution::new_zero(&left_min_val.data_type()).unwrap_or_default()
959959
);
960960
}
961961

@@ -964,7 +964,7 @@ fn estimate_disjoint_inputs(
964964
if left_max_val < right_min_val
965965
{
966966
return Some(
967-
ProbabilityDistribution::new_zero(&DataType::UInt64).unwrap_or_default()
967+
ProbabilityDistribution::new_zero(&left_max_val.data_type()).unwrap_or_default()
968968
);
969969
}
970970
}

0 commit comments

Comments
 (0)