Skip to content

Commit 042e4fb

Browse files
authored
Improve stats reporting to DF (#7628)
## Summary By comparing the physical plans between Parquet and Vortex, I've found a few things we're doing that are either wrong or suboptimal: 1. Reporting inexact column and file byte sizes if they are missing (which they are), which at the very least leads DF down a suboptimal path which introduces an extra RepartitionExec which isn't necessary. 2. He have a cute trick to report cardinality estimation for constant columns, problem is if we said we had exactly one record even if `IsConstant` was false. There are still a few stats things that could probably be improved here, especially which stats are calculated and stored where and how that's configured, but I think for now this is a very nice improvement. --------- Signed-off-by: Adam Gutglick <[email protected]>
1 parent cc06c60 commit 042e4fb

2 files changed

Lines changed: 84 additions & 61 deletions

File tree

‎vortex-datafusion/src/convert/stats.rs‎

Lines changed: 35 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ use vortex::dtype::Nullability;
99
use vortex::dtype::PType;
1010
use vortex::error::VortexExpect;
1111
use vortex::error::VortexResult;
12+
use vortex::expr::stats::Precision as VortexPrecision;
1213
use vortex::expr::stats::Stat;
1314
use vortex::scalar::Scalar;
1415

@@ -83,10 +84,40 @@ pub(crate) fn stats_set_to_df(
8384
min_value: min.to_df(),
8485
max_value: max.to_df(),
8586
sum_value: sum.to_df(),
86-
distinct_count: stats_set
87-
.get_as::<bool>(Stat::IsConstant, &DType::Bool(Nullability::NonNullable))
88-
.and_then(|is_constant| is_constant.as_exact().map(|_| Precision::Exact(1)))
89-
.unwrap_or(Precision::Absent),
87+
distinct_count: is_constant_to_distinct_count(
88+
stats_set.get_as::<bool>(Stat::IsConstant, &DType::Bool(Nullability::NonNullable)),
89+
),
9090
byte_size: column_size.to_df(),
9191
})
9292
}
93+
94+
pub(crate) fn is_constant_to_distinct_count(
95+
is_constant: Option<VortexPrecision<bool>>,
96+
) -> Precision<usize> {
97+
match is_constant.and_then(VortexPrecision::as_exact) {
98+
Some(true) => Precision::Exact(1),
99+
Some(false) | None => Precision::Absent,
100+
}
101+
}
102+
103+
#[cfg(test)]
104+
mod tests {
105+
use vortex::expr::stats::Precision as VortexPrecision;
106+
107+
use super::*;
108+
109+
#[test]
110+
fn is_constant_false_does_not_imply_one_distinct_value() -> VortexResult<()> {
111+
let false_constant = StatsSet::of(Stat::IsConstant, VortexPrecision::exact(false));
112+
let false_stats = stats_set_to_df(&false_constant, &DType::Bool(Nullability::NonNullable))?;
113+
114+
assert_eq!(false_stats.distinct_count, Precision::Absent);
115+
116+
let true_constant = StatsSet::of(Stat::IsConstant, VortexPrecision::exact(true));
117+
let true_stats = stats_set_to_df(&true_constant, &DType::Bool(Nullability::NonNullable))?;
118+
119+
assert_eq!(true_stats.distinct_count, Precision::Exact(1));
120+
121+
Ok(())
122+
}
123+
}

‎vortex-datafusion/src/persistent/format.rs‎

Lines changed: 49 additions & 57 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ use datafusion_common::ColumnStatistics;
1414
use datafusion_common::DataFusionError;
1515
use datafusion_common::GetExt;
1616
use datafusion_common::Result as DFResult;
17+
use datafusion_common::ScalarValue as DFScalarValue;
1718
use datafusion_common::Statistics;
1819
use datafusion_common::config::ConfigField;
1920
use datafusion_common::config_namespace;
@@ -60,13 +61,15 @@ use vortex::file::VORTEX_FILE_EXTENSION;
6061
use vortex::io::object_store::ObjectStoreReadAt;
6162
use vortex::io::session::RuntimeSessionExt;
6263
use vortex::scalar::Scalar;
64+
use vortex::scalar::ScalarValue as VortexScalarValue;
6365
use vortex::session::VortexSession;
6466

6567
use super::cache::CachedVortexMetadata;
6668
use super::sink::VortexSink;
6769
use super::source::VortexSource;
6870
use crate::PrecisionExt as _;
6971
use crate::convert::TryToDataFusion;
72+
use crate::convert::stats::is_constant_to_distinct_count;
7073

7174
const DEFAULT_FOOTER_INITIAL_READ_SIZE_BYTES: usize = MAX_POSTSCRIPT_SIZE as usize + EOF_SIZE;
7275

@@ -500,11 +503,13 @@ impl FileFormat for VortexFormat {
500503
.vortex_expect("Row count overflow"),
501504
),
502505
total_byte_size: Precision::Absent,
503-
column_statistics: vec![ColumnStatistics::default(); struct_dtype.nfields()],
506+
column_statistics: vec![
507+
ColumnStatistics::default();
508+
table_schema.fields().len()
509+
],
504510
});
505511
};
506512

507-
let mut sum_of_column_byte_sizes = stats::Precision::exact(0_usize);
508513
let mut column_statistics = Vec::with_capacity(table_schema.fields().len());
509514

510515
for field in table_schema.fields().iter() {
@@ -518,55 +523,22 @@ impl FileFormat for VortexFormat {
518523
let (stats_set, stats_dtype) = file_stats.get(col_idx);
519524

520525
// Update the total size in bytes.
521-
let column_size = stats_set
522-
.get_as::<usize>(Stat::UncompressedSizeInBytes, &PType::U64.into())
523-
.unwrap_or_else(|| stats::Precision::inexact(0_usize));
524-
sum_of_column_byte_sizes = sum_of_column_byte_sizes
525-
.zip(column_size)
526-
.map(|(acc, size)| acc + size);
527-
528-
// TODO(connor): There's a lot that can go wrong here, should probably handle this
529-
// more gracefully...
530-
// Find the min statistic.
531-
let min = stats_set.get(Stat::Min).and_then(|pstat_val| {
532-
pstat_val
533-
.map(|stat_val| {
534-
// Because of DataFusion's Schema evolution, it is possible that the
535-
// type of the min/max stat has changed. Thus we construct the stat as
536-
// the file datatype first and only then do we cast accordingly.
537-
Scalar::try_new(
538-
Stat::Min
539-
.dtype(stats_dtype)
540-
.vortex_expect("must have a valid dtype"),
541-
Some(stat_val),
542-
)
543-
.vortex_expect("`Stat::Min` somehow had an incompatible `DType`")
544-
.cast(&DType::from_arrow(field.as_ref()))
545-
.vortex_expect("Unable to cast to target type that DataFusion wants")
546-
.try_to_df()
547-
.ok()
548-
})
549-
.transpose()
550-
});
551-
552-
// Find the max statistic.
553-
let max = stats_set.get(Stat::Max).and_then(|pstat_val| {
554-
pstat_val
555-
.map(|stat_val| {
556-
Scalar::try_new(
557-
Stat::Max
558-
.dtype(stats_dtype)
559-
.vortex_expect("must have a valid dtype"),
560-
Some(stat_val),
561-
)
562-
.vortex_expect("`Stat::Max` somehow had an incompatible `DType`")
563-
.cast(&DType::from_arrow(field.as_ref()))
564-
.vortex_expect("Unable to cast to target type that DataFusion wants")
565-
.try_to_df()
566-
.ok()
567-
})
568-
.transpose()
569-
});
526+
let column_size =
527+
stats_set.get_as::<usize>(Stat::UncompressedSizeInBytes, &PType::U64.into());
528+
529+
let target_dtype = DType::from_arrow(field.as_ref());
530+
let min = scalar_stat_to_df(
531+
Stat::Min,
532+
stats_set.get(Stat::Min),
533+
stats_dtype,
534+
&target_dtype,
535+
);
536+
let max = scalar_stat_to_df(
537+
Stat::Max,
538+
stats_set.get(Stat::Max),
539+
stats_dtype,
540+
&target_dtype,
541+
);
570542

571543
let null_count = stats_set.get_as::<usize>(Stat::NullCount, &PType::U64.into());
572544

@@ -575,16 +547,19 @@ impl FileFormat for VortexFormat {
575547
min_value: min.to_df(),
576548
max_value: max.to_df(),
577549
sum_value: Precision::Absent,
578-
distinct_count: stats_set
579-
.get_as::<bool>(Stat::IsConstant, &DType::Bool(Nullability::NonNullable))
580-
.and_then(|is_constant| is_constant.as_exact().map(|_| Precision::Exact(1)))
581-
.unwrap_or(Precision::Absent),
582-
// TODO(connor): Is this correct?
550+
distinct_count: is_constant_to_distinct_count(
551+
stats_set.get_as::<bool>(
552+
Stat::IsConstant,
553+
&DType::Bool(Nullability::NonNullable),
554+
),
555+
),
583556
byte_size: column_size.to_df(),
584557
})
585558
}
586559

587-
let total_byte_size = sum_of_column_byte_sizes.to_df();
560+
let total_byte_size = column_statistics
561+
.iter()
562+
.fold(Precision::Exact(0), |acc, cs| acc.add(&cs.byte_size));
588563

589564
Ok(Statistics {
590565
num_rows: Precision::Exact(
@@ -651,6 +626,23 @@ impl FileFormat for VortexFormat {
651626
}
652627
}
653628

629+
fn scalar_stat_to_df(
630+
stat: Stat,
631+
value: Option<stats::Precision<VortexScalarValue>>,
632+
stats_dtype: &DType,
633+
target_dtype: &DType,
634+
) -> Option<stats::Precision<DFScalarValue>> {
635+
let stat_dtype = stat.dtype(stats_dtype)?;
636+
637+
value?
638+
.try_map(|stat_value| {
639+
Scalar::try_new(stat_dtype, Some(stat_value))?
640+
.cast(target_dtype)?
641+
.try_to_df()
642+
})
643+
.ok()
644+
}
645+
654646
#[cfg(test)]
655647
mod tests {
656648

0 commit comments

Comments
 (0)