Skip to content

Commit c147c37

Browse files
adriangbclaude
andcommitted
refactor: hold Arc<Column> in StatisticsRequest per-column variants
A Column owns its relation/name strings, so cloning a StatisticsRequest cloned those strings. The requests live in a BTreeSet<StatisticsRequest> on TableScan that is cloned with the plan throughout optimization, so wrap the per-column payload in Arc<Column> to make those clones cheap. Co-Authored-By: Claude Opus 4.7 (1M context) <[email protected]>
1 parent 6f3cfc9 commit c147c37

2 files changed

Lines changed: 21 additions & 9 deletions

File tree

‎datafusion/core/tests/user_defined/statistics_requests.rs‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -79,7 +79,8 @@ impl OptimizerRule for RequestColumnStatistics {
7979
// fixpoint without a manual "already visited" guard.
8080
let mut changed = scan.statistics_requests.insert(StatisticsRequest::RowCount);
8181
for field in scan.projected_schema.fields() {
82-
let req = StatisticsRequest::Min(Column::new_unqualified(field.name()));
82+
let req =
83+
StatisticsRequest::Min(Arc::new(Column::new_unqualified(field.name())));
8384
changed |= scan.statistics_requests.insert(req);
8485
}
8586
Ok(if changed {
@@ -188,11 +189,15 @@ async fn custom_rule_requests_reach_custom_provider() -> Result<()> {
188189
"expected RowCount, got {got:?}"
189190
);
190191
assert!(
191-
got.contains(&StatisticsRequest::Min(Column::new_unqualified("a"))),
192+
got.contains(&StatisticsRequest::Min(Arc::new(Column::new_unqualified(
193+
"a"
194+
)))),
192195
"expected Min(a), got {got:?}"
193196
);
194197
assert!(
195-
got.contains(&StatisticsRequest::Min(Column::new_unqualified("b"))),
198+
got.contains(&StatisticsRequest::Min(Arc::new(Column::new_unqualified(
199+
"b"
200+
)))),
196201
"expected Min(b), got {got:?}"
197202
);
198203
Ok(())

‎datafusion/expr-common/src/statistics.rs‎

Lines changed: 13 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1695,6 +1695,8 @@ mod tests {
16951695
}
16961696
}
16971697

1698+
use std::sync::Arc;
1699+
16981700
use datafusion_common::Column;
16991701

17001702
/// A statistic a caller would like a provider to supply, if it can do so
@@ -1711,21 +1713,26 @@ use datafusion_common::Column;
17111713
/// Each variant maps onto a field of [`datafusion_common::Statistics`] /
17121714
/// [`datafusion_common::ColumnStatistics`], so a provider that already
17131715
/// populates one can answer the request trivially.
1716+
///
1717+
/// The per-column variants hold an `Arc<Column>` rather than an owned
1718+
/// [`Column`] (which carries owned strings) so cloning a request — and the
1719+
/// `BTreeSet<StatisticsRequest>` stored on `TableScan`, which is cloned with
1720+
/// the plan during optimization — stays cheap.
17141721
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)]
17151722
pub enum StatisticsRequest {
17161723
/// Smallest non-null value of `column`.
1717-
Min(Column),
1724+
Min(Arc<Column>),
17181725
/// Largest non-null value of `column`.
1719-
Max(Column),
1726+
Max(Arc<Column>),
17201727
/// Number of NULLs in `column`.
1721-
NullCount(Column),
1728+
NullCount(Arc<Column>),
17221729
/// Number of distinct values in `column` (exact or estimated).
1723-
DistinctCount(Column),
1730+
DistinctCount(Arc<Column>),
17241731
/// Sum of values in `column` (numerics, widened per
17251732
/// `ColumnStatistics::sum_value`).
1726-
Sum(Column),
1733+
Sum(Arc<Column>),
17271734
/// Encoded/output byte size of `column`.
1728-
ByteSize(Column),
1735+
ByteSize(Arc<Column>),
17291736
/// Number of rows in the container (table / file).
17301737
RowCount,
17311738
/// Total byte size of the container's output.

0 commit comments

Comments
 (0)