Skip to content

Commit 264ee3d

Browse files
authored
fix: use session statistics registry in dfbench reports (#25570)
## Which issue does this PR close? Part of #8227. ## Rationale for this change `dfbench statistics` unconditionally enables built-in statistics providers, so its estimates can differ from those used by the session. In TPC-H SF1 Q14, it reports 14.73 billion rows instead of 73,650 for a join that produces 75,983 rows. Unrelated to this PR, but it's pretty weird how the built-in providers are not actually built-in unless explicitly enabled, and it's also weird how they can override the actually built-in code in each operator producing different results. It seems like there's different pieces of code with overlapping intentions and different implementation. ## What changes are included in this PR? Capture estimates using the session's registry, matching `EXPLAIN ANALYZE`. An unconfigured session falls back to operator statistics. ## What is the testing strategy for this PR? - All five existing CLI statistics tests, formatting, and all-feature Clippy pass. ## Are there any user-facing changes? CLI estimates now respect the session's statistics configuration.
1 parent 23c1b59 commit 264ee3d

1 file changed

Lines changed: 51 additions & 4 deletions

File tree

‎benchmarks/src/statistics.rs‎

Lines changed: 51 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -184,7 +184,10 @@ impl RunOpt {
184184
let logical_plan = state.optimize(&logical_plan)?;
185185
let physical_plan = state.create_physical_plan(&logical_plan).await?;
186186

187-
let statistics = capture_statistics(physical_plan.as_ref())?;
187+
let statistics = capture_statistics(
188+
physical_plan.as_ref(),
189+
&state.statistics_registry().cloned().unwrap_or_default(),
190+
)?;
188191
collect(Arc::clone(&physical_plan), state.task_ctx()).await?;
189192

190193
let mut report = Vec::with_capacity(statistics.len());
@@ -248,10 +251,12 @@ enum QError {
248251
ExactZero,
249252
}
250253

251-
fn capture_statistics(plan: &dyn ExecutionPlan) -> Result<Vec<CapturedStatistics>> {
252-
let statistics_context = StatisticsRegistry::new();
254+
fn capture_statistics(
255+
plan: &dyn ExecutionPlan,
256+
statistics_context: &StatisticsRegistry,
257+
) -> Result<Vec<CapturedStatistics>> {
253258
let mut result = vec![];
254-
capture_statistics_inner(plan, &statistics_context, "0", &mut result)?;
259+
capture_statistics_inner(plan, statistics_context, "0", &mut result)?;
255260
Ok(result)
256261
}
257262

@@ -723,6 +728,11 @@ fn collect_parquet_files(path: &Path, files: &mut Vec<PathBuf>) -> Result<()> {
723728
#[cfg(test)]
724729
mod tests {
725730
use super::*;
731+
use datafusion::execution::session_state::SessionStateBuilder;
732+
use datafusion::physical_plan::Statistics;
733+
use datafusion::physical_plan::operator_statistics::{
734+
ClosureStatisticsProvider, StatisticsResult,
735+
};
726736
use tempfile::tempdir;
727737

728738
#[test]
@@ -786,6 +796,43 @@ mod tests {
786796
assert!(!reports.is_empty());
787797
}
788798

799+
#[tokio::test]
800+
async fn reports_estimates_from_session_statistics_registry() {
801+
let directory = tempdir().unwrap();
802+
let options = RunOpt {
803+
query: None,
804+
compare: None,
805+
path: directory.path().to_path_buf(),
806+
query_path: directory.path().to_path_buf(),
807+
};
808+
let mut registry = StatisticsRegistry::new();
809+
registry.register(Arc::new(ClosureStatisticsProvider::new(
810+
|plan, _child_stats| {
811+
Ok(StatisticsResult::Computed(
812+
Statistics {
813+
num_rows: Precision::Inexact(42),
814+
..Statistics::new_unknown(plan.schema().as_ref())
815+
}
816+
.into(),
817+
))
818+
},
819+
)));
820+
let ctx = SessionContext::from(
821+
SessionStateBuilder::new()
822+
.with_default_features()
823+
.with_statistics_registry(registry)
824+
.build(),
825+
);
826+
let statement =
827+
sql_statements("SELECT 1", &ctx.state().config_options().sql_parser)
828+
.unwrap()
829+
.pop_front()
830+
.unwrap();
831+
832+
let reports = options.report_statement(&ctx, statement).await.unwrap();
833+
assert_eq!(reports[0].estimated_rows, StatisticValue::Inexact(42));
834+
}
835+
789836
async fn report_query_files(
790837
options: &RunOpt,
791838
ctx: &SessionContext,

0 commit comments

Comments
 (0)