Skip to content

Commit 9ff61f9

Browse files
authored
Return provider extensions directly from the statistics walk (apache#25826)
## Which issue does this PR close? Part of apache#25141. ## Rationale for this change `compute_extended` currently reads a root node’s provider extensions back from the cache after computing its statistics. A planned change will stop caching borrowed roots, so that read would lose their extensions. Returning the extensions from the statistics walk preserves the result without depending on a root cache entry. ## What changes are included in this PR? - Return statistics and provider extensions together from the private statistics walk. `compute_extended` uses that result; `compute` keeps returning only the statistics. - Continue caching provider extensions for parent nodes. On a cache hit, read them only when the caller requests them. - Replace `ExtendedStatistics::extensions()` with `into_parts()` to move its statistics and extensions into the walk’s result. The public signatures of `compute` and `compute_extended` are unchanged. ## Are these changes tested? The patch adds `cached_node_keeps_extensions` and `walk_returns_provider_extensions_directly`. They cover extensions returned after a child is cached, extensions returned from a fresh provider result, and cache reads that occur only when requested. ## Are there any user-facing changes? No user-facing change is intended. This refactor preserves provider extension behavior ahead of the planned cache change. ## LLM-generated code disclosure This PR includes LLM-generated code and comments. All LLM-generated content has been manually reviewed.
1 parent 14a5d13 commit 9ff61f9

2 files changed

Lines changed: 111 additions & 33 deletions

File tree

‎datafusion/physical-plan/src/operator_statistics/mod.rs‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -183,9 +183,9 @@ impl ExtendedStatistics {
183183
Self { base, extensions }
184184
}
185185

186-
/// Returns the extension map.
187-
pub(crate) fn extensions(&self) -> &Extensions {
188-
&self.extensions
186+
/// Splits into the base statistics and the extension map without cloning.
187+
pub(crate) fn into_parts(self) -> (Arc<Statistics>, Extensions) {
188+
(self.base, self.extensions)
189189
}
190190
}
191191

‎datafusion/physical-plan/src/statistics.rs‎

Lines changed: 108 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -207,7 +207,8 @@ impl StatisticsContext {
207207
plan: &dyn ExecutionPlan,
208208
args: &StatisticsArgs,
209209
) -> Result<Arc<Statistics>> {
210-
self.compute_base(plan, args)
210+
self.compute_base(plan, args, false)
211+
.map(|(statistics, _)| statistics)
211212
}
212213

213214
/// Computes the [`ExtendedStatistics`] for `plan`: the core statistics plus
@@ -218,27 +219,33 @@ impl StatisticsContext {
218219
plan: &dyn ExecutionPlan,
219220
args: &StatisticsArgs,
220221
) -> Result<Arc<ExtendedStatistics>> {
221-
let statistics = self.compute_base(plan, args)?;
222-
let extensions = self
223-
.cached_extensions(plan, args.partition())
224-
.unwrap_or_default();
222+
let (statistics, extensions) = self.compute_base(plan, args, true)?;
225223
Ok(Arc::new(ExtendedStatistics::new_with_extensions(
226-
statistics, extensions,
224+
statistics,
225+
extensions.unwrap_or_default(),
227226
)))
228227
}
229228

230229
/// Bottom-up walk producing the node's core statistics, resolving children
231230
/// first and consulting the provider chain before the operator's built-in
232-
/// [`ExecutionPlan::statistics_from_inputs`]. Any extensions a provider
233-
/// attaches are recorded in the extension cache for [`Self::compute_extended`].
231+
/// [`ExecutionPlan::statistics_from_inputs`].
232+
///
233+
/// Also returns the extensions a provider attached to this node, so callers
234+
/// never need to read the node's own extensions back out of the cache.
235+
/// Extensions are still recorded in the cache so parents can consume them.
236+
/// A freshly computed provider result always returns its extensions (they
237+
/// are moved, not cloned); on a cache hit they are cloned out of the cache
238+
/// only when `read_cached_extensions` is set, so child lookups that discard
239+
/// them pay nothing.
234240
///
235241
/// When `args.partition()` is `Some(idx)`, `idx` is validated against the
236242
/// plan's partition count.
237243
fn compute_base(
238244
&self,
239245
plan: &dyn ExecutionPlan,
240246
args: &StatisticsArgs,
241-
) -> Result<Arc<Statistics>> {
247+
read_cached_extensions: bool,
248+
) -> Result<(Arc<Statistics>, Option<Extensions>)> {
242249
let partition = args.partition();
243250

244251
if let Some(idx) = partition {
@@ -252,24 +259,33 @@ impl StatisticsContext {
252259
}
253260

254261
if let Some(cached) = self.cached_statistics(plan, partition) {
255-
return Ok(cached);
262+
// Only providers store extensions, so with an empty registry the
263+
// extension cache is never touched.
264+
let extensions =
265+
if read_cached_extensions && !self.registry.providers().is_empty() {
266+
self.cached_extensions(plan, partition)
267+
} else {
268+
None
269+
};
270+
return Ok((cached, extensions));
256271
}
257272

258273
let children = plan.children();
259274
// Try providers before resolving the operator's own children, so a
260275
// provider that overrides this node is not blocked by the fallback walk.
261-
let statistics = match self.try_provider_stats(plan, &children, args)? {
262-
Some(statistics) => statistics,
263-
None => {
264-
let requests = plan.child_stats_requests(partition);
265-
self.validate_child_requests(plan, &children, &requests)?;
266-
let child_statistics =
267-
self.resolve_children(plan, &children, &requests)?;
268-
plan.statistics_from_inputs(&child_statistics, args)?
269-
}
270-
};
276+
let (statistics, extensions) =
277+
match self.try_provider_stats(plan, &children, args)? {
278+
Some(computed) => computed,
279+
None => {
280+
let requests = plan.child_stats_requests(partition);
281+
self.validate_child_requests(plan, &children, &requests)?;
282+
let child_statistics =
283+
self.resolve_children(plan, &children, &requests)?;
284+
(plan.statistics_from_inputs(&child_statistics, args)?, None)
285+
}
286+
};
271287
self.store_statistics(plan, partition, Arc::clone(&statistics));
272-
Ok(statistics)
288+
Ok((statistics, extensions))
273289
}
274290

275291
/// Validates child stat `requests` against `plan`'s children: the count must
@@ -318,7 +334,12 @@ impl StatisticsContext {
318334
.enumerate()
319335
.map(|(i, (child, directive))| match directive {
320336
ChildStats::At(p) => self
321-
.compute_base(child.as_ref(), &StatisticsArgs::new().with_partition(*p))
337+
.compute_base(
338+
child.as_ref(),
339+
&StatisticsArgs::new().with_partition(*p),
340+
false,
341+
)
342+
.map(|(statistics, _)| statistics)
322343
.map_err(|e| {
323344
e.context(format!(
324345
"computing statistics for child {i} ({}) of {} at partition {p:?}",
@@ -334,9 +355,10 @@ impl StatisticsContext {
334355
}
335356

336357
/// Runs the provider chain, returning the first `Computed` result's core
337-
/// statistics (and recording its extensions), or `None` if the chain is empty
338-
/// or all delegate. A partition-blind provider applies only to overall stats
339-
/// (its default `compute_statistics_with_args` delegates per partition).
358+
/// statistics and non-empty extensions (also recording the extensions in
359+
/// the cache), or `None` if the chain is empty or all delegate. A
360+
/// partition-blind provider applies only to overall stats (its default
361+
/// `compute_statistics_with_args` delegates per partition).
340362
///
341363
/// Each provider's child statistics come from its own
342364
/// [`child_stats_requests`](crate::operator_statistics::StatisticsProvider::child_stats_requests)
@@ -346,7 +368,7 @@ impl StatisticsContext {
346368
plan: &dyn ExecutionPlan,
347369
children: &[&Arc<dyn ExecutionPlan>],
348370
args: &StatisticsArgs,
349-
) -> Result<Option<Arc<Statistics>>> {
371+
) -> Result<Option<(Arc<Statistics>, Option<Extensions>)>> {
350372
let providers = self.registry.providers();
351373
if providers.is_empty() {
352374
return Ok(None);
@@ -379,10 +401,14 @@ impl StatisticsContext {
379401
if let StatisticsResult::Computed(computed) =
380402
provider.compute_statistics_with_args(plan, &child_extended, args)?
381403
{
382-
if !computed.extensions().is_empty() {
383-
self.store_extensions(plan, partition, computed.extensions().clone());
384-
}
385-
return Ok(Some(Arc::clone(computed.base_arc())));
404+
let (statistics, extensions) = computed.into_parts();
405+
let extensions = if extensions.is_empty() {
406+
None
407+
} else {
408+
self.store_extensions(plan, partition, extensions.clone());
409+
Some(extensions)
410+
};
411+
return Ok(Some((statistics, extensions)));
386412
}
387413
}
388414
Ok(None)
@@ -652,6 +678,58 @@ mod tests {
652678
assert_eq!(extended.get_extension::<Tag>(), Some(&Tag(14)));
653679
}
654680

681+
#[test]
682+
fn cached_node_keeps_extensions() {
683+
let leaf = make_stats_leaf(100);
684+
let parent: Arc<dyn ExecutionPlan> =
685+
Arc::new(CoalescePartitionsExec::new(Arc::clone(&leaf)));
686+
let ctx = ctx_with(Arc::new(TagLeafProvider { rows: 100, tag: 7 }));
687+
688+
// Walking the parent caches the leaf as a child ...
689+
let _ = ctx
690+
.compute(parent.as_ref(), &StatisticsArgs::new())
691+
.unwrap();
692+
// ... so computing the leaf directly is a cache hit that must still
693+
// return the extensions its provider attached.
694+
let first = ctx
695+
.compute_extended(leaf.as_ref(), &StatisticsArgs::new())
696+
.unwrap();
697+
let second = ctx
698+
.compute_extended(leaf.as_ref(), &StatisticsArgs::new())
699+
.unwrap();
700+
assert_eq!(first.get_extension::<Tag>(), Some(&Tag(7)));
701+
assert_eq!(second.get_extension::<Tag>(), Some(&Tag(7)));
702+
assert!(Arc::ptr_eq(first.base_arc(), second.base_arc()));
703+
}
704+
705+
/// The walk returns a freshly computed node's extensions from the provider
706+
/// result itself, not from the cache, so `compute_extended` does not depend
707+
/// on the root having a cache entry. On a cache hit they are read only on
708+
/// request.
709+
#[test]
710+
fn walk_returns_provider_extensions_directly() {
711+
let leaf = make_stats_leaf(100);
712+
let ctx = ctx_with(Arc::new(TagLeafProvider { rows: 100, tag: 7 }));
713+
let args = StatisticsArgs::new();
714+
715+
// Fresh computation: returned even though no cached read was requested,
716+
// and still recorded for parents.
717+
let (_, extensions) = ctx.compute_base(leaf.as_ref(), &args, false).unwrap();
718+
assert_eq!(extensions.unwrap().get::<Tag>(), Some(&Tag(7)));
719+
assert!(
720+
ctx.cache
721+
.borrow()
722+
.extensions
723+
.contains_key(&cache_key(leaf.as_ref(), None))
724+
);
725+
726+
// Cache hit: extensions are cloned out of the cache only on request.
727+
let (_, extensions) = ctx.compute_base(leaf.as_ref(), &args, false).unwrap();
728+
assert!(extensions.is_none());
729+
let (_, extensions) = ctx.compute_base(leaf.as_ref(), &args, true).unwrap();
730+
assert_eq!(extensions.unwrap().get::<Tag>(), Some(&Tag(7)));
731+
}
732+
655733
#[test]
656734
fn builtin_fallback_drops_extensions() {
657735
let leaf = make_stats_leaf(100);

0 commit comments

Comments
 (0)