Skip to content

Commit 9844182

Browse files
committed
fix(physical-plan): account for retained sort-key capacity
GroupClusteringPartial::heap_size summed ScalarValue::size() over the live elements of the in-progress key, so the vector's spare capacity was never reported. The key vector produced by get_row_at_idx retains capacity for four scalars after extracting a single key, leaving three scalar slots unaccounted and under-reserving memory. Use DFHeapSize for the sort key so its full capacity, including spare slots, and the heap payload of each initialized scalar are counted once. The inline Vec descriptor remains owned by GroupClustering::size(), so it is not double-counted. Update the retained-allocation test to derive the expected key size from ScalarValue::size_of_vec, and add a focused test with spare capacity and a heap-owning scalar. Part of apache#23393 Closes apache#26099
1 parent 4bf32cb commit 9844182

2 files changed

Lines changed: 54 additions & 11 deletions

File tree

‎datafusion/physical-plan/src/aggregates/order/mod.rs‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -194,6 +194,7 @@ mod tests {
194194

195195
use arrow::array::{Int32Array, StringArray};
196196
use datafusion_common::ScalarValue;
197+
use datafusion_common::utils::get_row_at_idx;
197198

198199
#[test]
199200
fn test_size_inline_only() -> Result<()> {
@@ -215,12 +216,18 @@ mod tests {
215216
GroupClustering::Partial(GroupClusteringPartial::try_new(grouping_indices)?);
216217
assert_eq!(group_clustering.size(), expected);
217218

218-
// A variable-width key checks both the scalar descriptor and its payload.
219+
// A variable-width key checks retained sort-key capacity, the scalar
220+
// slots, and the scalar payload.
219221
let key = "retained group key";
220222
let batch_group_values: Vec<ArrayRef> =
221223
vec![Arc::new(StringArray::from(vec!["a", key]))];
222224
group_clustering.new_groups(&batch_group_values, &[0, 1], 2)?;
223-
let in_progress = expected + ScalarValue::Utf8(Some(key.to_owned())).size();
225+
let group_key = get_row_at_idx(&batch_group_values, 1)?;
226+
// `size_of_vec` includes the inline Vec descriptor, which
227+
// `GroupClustering::size` already charges via `size_of::<Self>()`.
228+
let retained_sort_key =
229+
ScalarValue::size_of_vec(&group_key) - size_of::<Vec<ScalarValue>>();
230+
let in_progress = expected + retained_sort_key;
224231
assert_eq!(group_clustering.size(), in_progress);
225232

226233
// Completing drops the key, but retains grouping-index capacity.

‎datafusion/physical-plan/src/aggregates/order/partial.rs‎

Lines changed: 45 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ use std::sync::Arc;
88
use arrow::array::ArrayRef;
99
use arrow::compute::SortOptions;
1010
use arrow_ord::partition::partition;
11+
use datafusion_common::heap_size::{DFHeapSize, DFHeapSizeCtx};
1112
use datafusion_common::utils::{compare_rows, get_row_at_idx};
1213
use datafusion_common::{Result, ScalarValue};
1314
use datafusion_execution::memory_pool::proxy::VecAllocExt;
@@ -87,15 +88,10 @@ enum State {
8788
}
8889

8990
impl State {
90-
fn heap_size(&self) -> usize {
91+
fn heap_size(&self, ctx: &mut DFHeapSizeCtx) -> usize {
9192
match self {
92-
State::Taken => 0,
93-
State::Start => 0,
94-
State::InProgress { group_key, .. } => group_key
95-
.iter()
96-
.map(|scalar_value| scalar_value.size())
97-
.sum(),
98-
State::Complete => 0,
93+
State::InProgress { group_key, .. } => group_key.heap_size(ctx),
94+
State::Taken | State::Start | State::Complete => 0,
9995
}
10096
}
10197
}
@@ -254,14 +250,17 @@ impl GroupClusteringPartial {
254250
/// Returns retained heap allocations, excluding the inline descriptor
255251
/// already counted by [`super::GroupClustering::size`].
256252
pub(crate) fn heap_size(&self) -> usize {
257-
self.grouping_indices.allocated_size() + self.state.heap_size()
253+
self.grouping_indices.allocated_size()
254+
+ self.state.heap_size(&mut DFHeapSizeCtx::default())
258255
}
259256
}
260257

261258
#[cfg(test)]
262259
mod tests {
263260
use super::*;
264261

262+
use std::mem::size_of;
263+
265264
use arrow::array::Int32Array;
266265

267266
#[rstest::rstest]
@@ -346,4 +345,41 @@ mod tests {
346345

347346
Ok(())
348347
}
348+
349+
#[test]
350+
fn heap_size_counts_retained_sort_key_capacity() -> Result<()> {
351+
// An in-progress sort key with spare capacity and a heap-owning scalar.
352+
let mut group_key = Vec::with_capacity(4);
353+
let key = String::from("retained sort key");
354+
let key_capacity = key.capacity();
355+
group_key.push(ScalarValue::Utf8(Some(key)));
356+
let group_key_capacity = group_key.capacity();
357+
assert!(group_key_capacity > group_key.len());
358+
359+
let mut grouping_indices = Vec::with_capacity(8);
360+
grouping_indices.push(0);
361+
let grouping_indices_capacity = grouping_indices.capacity();
362+
363+
let mut partial = GroupClusteringPartial::try_new(grouping_indices)?;
364+
partial.state = State::InProgress {
365+
current_run_start: 0,
366+
group_key,
367+
current: 0,
368+
};
369+
370+
// Every scalar slot, including the spare ones, is charged; the
371+
// scalar's own heap payload is charged on top of its slot.
372+
let expected = grouping_indices_capacity * size_of::<usize>()
373+
+ group_key_capacity * size_of::<ScalarValue>()
374+
+ key_capacity;
375+
assert_eq!(partial.heap_size(), expected);
376+
377+
// Accounting only the live element would miss the spare slots.
378+
let live_only = grouping_indices_capacity * size_of::<usize>()
379+
+ size_of::<ScalarValue>()
380+
+ key_capacity;
381+
assert!(partial.heap_size() > live_only);
382+
383+
Ok(())
384+
}
349385
}

0 commit comments

Comments
 (0)