Skip to content

Commit 36d81dd

Browse files
compheadkosiew
authored andcommitted
chore: adjust size accounting for min_max (apache#23899)
## Which issue does this PR close? <!-- We generally require a GitHub issue to be filed for all bug fixes and enhancements and this helps us generate change logs for our releases. You can link an issue to this PR using the GitHub syntax. For example `Closes apache#123` indicates that this PR will close issue apache#123. --> - Part of apache#23393 . ## Rationale for this change `SlidingMinAccumulator::size` and `SlidingMaxAccumulator::size` only reported the stack size of their `ScalarValue` field plus its heap payload, ignoring the memory held by the underlying `MovingMin` / `MovingMax` sliding-window buffers. For windowed `MIN`/`MAX` over string or list data, the two per-element stacks can hold megabytes of `ScalarValue` payload that the memory pool was never told about, understating accumulator memory usage. ## What changes are included in this PR? - Add a private `heap_size(elem_heap)` method to `MovingMin<T>` and `MovingMax<T>` that reports the two stack buffers' capacity in bytes plus each stored element's heap payload. - Factor the shared implementation into a `moving_stacks_heap_size` free helper so the two types stay in sync. - Include the buffer bytes in `SlidingMinAccumulator::size` and `SlidingMaxAccumulator::size` via the new method. ## Are these changes tested? Yes. Two new unit tests in `datafusion/functions-aggregate/src/min_max.rs`: - `moving_min_max_heap_size_i32` — fixed-width `T`, verifies buffer-only accounting with and without pushed elements. - `moving_min_max_heap_size_counts_elems` — `T = String`, verifies each of the two slots in a `(T, T)` pair contributes independently to the heap payload (mirroring the two independent `Clone`s made by `push`).
1 parent a34cc22 commit 36d81dd

1 file changed

Lines changed: 80 additions & 3 deletions

File tree

‎datafusion/functions-aggregate/src/min_max.rs‎

Lines changed: 80 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,7 @@ use datafusion_expr::{
5252
use datafusion_expr::{GroupsAccumulator, StatisticsArgs};
5353
use datafusion_macros::user_doc;
5454
use half::f16;
55-
use std::mem::size_of_val;
55+
use std::mem::{size_of, size_of_val};
5656
use std::ops::Deref;
5757

5858
fn get_min_max_result_type(input_types: &[DataType]) -> Result<Vec<DataType>> {
@@ -433,7 +433,9 @@ impl Accumulator for SlidingMaxAccumulator {
433433
}
434434

435435
fn size(&self) -> usize {
436-
size_of_val(self) - size_of_val(&self.max) + self.max.size()
436+
size_of_val(self) - size_of_val(&self.max)
437+
+ self.max.size()
438+
+ self.moving_max.heap_size(|sv| sv.size() - size_of_val(sv))
437439
}
438440
}
439441

@@ -721,7 +723,9 @@ impl Accumulator for SlidingMinAccumulator {
721723
}
722724

723725
fn size(&self) -> usize {
724-
size_of_val(self) - size_of_val(&self.min) + self.min.size()
726+
size_of_val(self) - size_of_val(&self.min)
727+
+ self.min.size()
728+
+ self.moving_min.heap_size(|sv| sv.size() - size_of_val(sv))
725729
}
726730
}
727731

@@ -857,6 +861,30 @@ impl<T: Clone + PartialOrd> MovingMin<T> {
857861
pub fn is_empty(&self) -> bool {
858862
self.len() == 0
859863
}
864+
865+
/// Heap bytes owned by the two stack buffers plus each stored `T`'s
866+
/// heap payload as reported by `elem_heap`. Excludes `size_of::<Self>()`.
867+
#[inline]
868+
fn heap_size(&self, elem_heap: impl Fn(&T) -> usize) -> usize {
869+
moving_stacks_heap_size(&self.push_stack, &self.pop_stack, elem_heap)
870+
}
871+
}
872+
873+
/// Shared implementation for [`MovingMin::heap_size`] and
874+
/// [`MovingMax::heap_size`]. Both share the same two-stack layout.
875+
#[inline]
876+
fn moving_stacks_heap_size<T>(
877+
push_stack: &Vec<(T, T)>,
878+
pop_stack: &Vec<(T, T)>,
879+
elem_heap: impl Fn(&T) -> usize,
880+
) -> usize {
881+
let buffers = (push_stack.capacity() + pop_stack.capacity()) * size_of::<(T, T)>();
882+
let elems: usize = push_stack
883+
.iter()
884+
.chain(pop_stack.iter())
885+
.map(|(a, b)| elem_heap(a) + elem_heap(b))
886+
.sum();
887+
buffers + elems
860888
}
861889

862890
/// Keep track of the maximum value in a sliding window.
@@ -975,6 +1003,13 @@ impl<T: Clone + PartialOrd> MovingMax<T> {
9751003
pub fn is_empty(&self) -> bool {
9761004
self.len() == 0
9771005
}
1006+
1007+
/// Heap bytes owned by the two stack buffers plus each stored `T`'s
1008+
/// heap payload as reported by `elem_heap`. Excludes `size_of::<Self>()`.
1009+
#[inline]
1010+
fn heap_size(&self, elem_heap: impl Fn(&T) -> usize) -> usize {
1011+
moving_stacks_heap_size(&self.push_stack, &self.pop_stack, elem_heap)
1012+
}
9781013
}
9791014

9801015
make_udaf_expr_and_func!(
@@ -1213,6 +1248,48 @@ mod tests {
12131248
Ok(())
12141249
}
12151250

1251+
#[test]
1252+
fn moving_min_max_heap_size_i32() {
1253+
// Fixed-width `T` has no per-element heap payload, so `heap_size`
1254+
// reports exactly the two stack buffers' capacity in bytes.
1255+
let mut moving_min = MovingMin::<i32>::with_capacity(4);
1256+
let mut moving_max = MovingMax::<i32>::with_capacity(4);
1257+
let elem = |_: &i32| 0;
1258+
1259+
// Both stacks are `with_capacity(4)`, so total slots = 8.
1260+
let buffer_only = 2 * 4 * size_of::<(i32, i32)>();
1261+
assert_eq!(moving_min.heap_size(elem), buffer_only);
1262+
assert_eq!(moving_max.heap_size(elem), buffer_only);
1263+
1264+
for i in 0..3 {
1265+
moving_min.push(i);
1266+
moving_max.push(i);
1267+
}
1268+
// Elements sit inside the pre-allocated buffers, so still buffer-only.
1269+
assert_eq!(moving_min.heap_size(elem), buffer_only);
1270+
assert_eq!(moving_max.heap_size(elem), buffer_only);
1271+
}
1272+
1273+
#[test]
1274+
fn moving_min_max_heap_size_counts_elems() {
1275+
// Each buffered slot is a `(T, T)` pair, so a stored element is
1276+
// visited twice by `heap_size` — mirrors two independent `Clone`s.
1277+
let mut moving_min = MovingMin::<String>::with_capacity(2);
1278+
let mut moving_max = MovingMax::<String>::with_capacity(2);
1279+
let elem = |s: &String| s.capacity();
1280+
1281+
moving_min.push("abcdef".to_string());
1282+
moving_max.push("abcdef".to_string());
1283+
1284+
// Both `push_stack` and `pop_stack` allocate `capacity` slots.
1285+
let buffers = 2 * 2 * size_of::<(String, String)>();
1286+
// 2 slots per stored element (value + running extremum) times the
1287+
// per-element heap payload from `elem`.
1288+
let elems = 2 * 6;
1289+
assert_eq!(moving_min.heap_size(elem), buffers + elems);
1290+
assert_eq!(moving_max.heap_size(elem), buffers + elems);
1291+
}
1292+
12161293
#[test]
12171294
fn test_min_max_coerce_types() {
12181295
// the coerced types is same with input types

0 commit comments

Comments
 (0)