Skip to content

Commit 65b3f7e

Browse files
adriangbclaude
andcommitted
test: cover the adapter's retained indices end to end instead of via internals
Review feedback asked for the accounting to be tested against end behaviour rather than through the adapter's private fields. Drop the three unit tests that recomputed `allocation_bytes` from the states and extend the memory-limit test instead: * `aggregate_adapter_spills_on_retained_indices` keeps the original check: 4 MiB of retained scratch indices against a 1 MiB limit must spill. It fails on main with `spill_count = 0`. * `aggregate_adapter_charges_retained_indices_once` hands every group four batches under a limit of twice the retained bytes and asserts that the query does not spill. Charging the capacity once per batch would report four times the retained bytes and fail this test. Both share one query runner so the two limits and batch counts are the only differences between them. Co-Authored-By: Claude Fable 5.1 <[email protected]>
1 parent 3ff93a9 commit 65b3f7e

2 files changed

Lines changed: 65 additions & 155 deletions

File tree

‎datafusion/core/tests/memory_limit/mod.rs‎

Lines changed: 65 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -787,59 +787,101 @@ async fn test_spill_file_compressed_with_lz4_frame() -> Result<()> {
787787
Ok(())
788788
}
789789

790+
/// Number of groups the `covar_samp` queries below produce.
791+
const ADAPTER_GROUPS: i64 = 128;
792+
/// Rows per input batch for the `covar_samp` queries below.
793+
const ADAPTER_BATCH_SIZE: i64 = 8192;
794+
/// Bytes of scratch row indices `GroupsAccumulatorAdapter` retains for the
795+
/// `covar_samp` queries below: one `u32` per row of the largest batch each
796+
/// group has ever received, kept for the lifetime of the group.
797+
const ADAPTER_RETAINED_BYTES: usize =
798+
(ADAPTER_GROUPS * ADAPTER_BATCH_SIZE) as usize * size_of::<u32>();
799+
800+
/// Runs a `GROUP BY` `covar_samp` query under `memory_limit` and returns the
801+
/// executed plan so the caller can inspect its metrics.
802+
///
790803
/// `covar_samp` has no native [`GroupsAccumulator`], so its per-group state is
791-
/// held by `GroupsAccumulatorAdapter`. The adapter keeps one scratch
792-
/// `Vec<u32>` of row indices per group, grown to the largest number of rows
793-
/// that group has ever taken from a single input batch and retained (cleared,
794-
/// but not deallocated) for the lifetime of the group.
804+
/// held by `GroupsAccumulatorAdapter`. The adapter keeps one scratch `Vec<u32>`
805+
/// of row indices per group, grown to the largest number of rows that group
806+
/// has ever taken from a single input batch and retained (cleared, but not
807+
/// deallocated) for the lifetime of the group.
795808
///
796-
/// This query hands the aggregate one batch of 8192 rows per group, so 128
797-
/// groups retain `128 * 8192 * 4` bytes = 4 MiB of scratch capacity, four
798-
/// times the 1 MiB limit. Everything else the aggregate holds is two orders of
799-
/// magnitude smaller: without the scratch capacity the reported size peaks
800-
/// around 85 KB, twelve times under the limit, and the query runs to
801-
/// completion without ever asking the pool for what it is really using.
809+
/// The query hands each of the [`ADAPTER_GROUPS`] groups `batches_per_group`
810+
/// consecutive full batches of [`ADAPTER_BATCH_SIZE`] rows, so the adapter
811+
/// retains [`ADAPTER_RETAINED_BYTES`] (4 MiB) of scratch capacity however many
812+
/// batches each group receives. Everything else the aggregate holds is two
813+
/// orders of magnitude smaller.
802814
///
803815
/// `target_partitions = 1` puts the aggregate in `Single` mode, which spills
804816
/// under memory pressure instead of emitting groups early, so the accounting
805817
/// is observable as a spill.
806818
///
807819
/// [`GroupsAccumulator`]: datafusion_expr::GroupsAccumulator
808-
#[tokio::test]
809-
async fn aggregate_adapter_spills_on_retained_indices() -> Result<()> {
810-
const GROUPS: i64 = 128;
811-
const BATCH_SIZE: i64 = 8192;
812-
820+
async fn run_adapter_query(
821+
memory_limit: usize,
822+
batches_per_group: i64,
823+
) -> Result<Arc<dyn ExecutionPlan>> {
813824
let runtime = RuntimeEnvBuilder::new()
814-
.with_memory_pool(Arc::new(GreedyMemoryPool::new(1024 * 1024)))
825+
.with_memory_pool(Arc::new(GreedyMemoryPool::new(memory_limit)))
815826
.with_disk_manager_builder(
816827
DiskManagerBuilder::default().with_mode(DiskManagerMode::OsTmpDirectory),
817828
)
818829
.build_arc()?;
819830

820831
let config = SessionConfig::new()
821832
.with_target_partitions(1)
822-
.with_batch_size(BATCH_SIZE as usize);
833+
.with_batch_size(ADAPTER_BATCH_SIZE as usize);
823834
let ctx = SessionContext::new_with_config_rt(config, runtime);
824835

836+
let rows_per_group = ADAPTER_BATCH_SIZE * batches_per_group;
825837
let sql = format!(
826-
"SELECT v / {BATCH_SIZE} AS g, covar_samp(v, v) AS c \
838+
"SELECT v / {rows_per_group} AS g, covar_samp(v, v) AS c \
827839
FROM generate_series(0, {}) AS t(v) \
828-
GROUP BY v / {BATCH_SIZE}",
829-
GROUPS * BATCH_SIZE - 1
840+
GROUP BY v / {rows_per_group}",
841+
ADAPTER_GROUPS * rows_per_group - 1
830842
);
831843

832844
let plan = ctx.sql(&sql).await?.create_physical_plan().await?;
833845
let batches = collect_batches(Arc::clone(&plan), ctx.task_ctx()).await?;
834846

835847
let rows: usize = batches.iter().map(|batch| batch.num_rows()).sum();
836-
assert_eq!(rows, GROUPS as usize);
848+
assert_eq!(rows, ADAPTER_GROUPS as usize);
849+
850+
Ok(plan)
851+
}
852+
853+
/// The scratch capacity `GroupsAccumulatorAdapter` retains is four times the
854+
/// memory limit, so the aggregate must spill. Without the capacity everything
855+
/// the aggregate reports is far under the limit, and the query runs to
856+
/// completion without ever asking the pool for what it is really using.
857+
#[tokio::test]
858+
async fn aggregate_adapter_spills_on_retained_indices() -> Result<()> {
859+
let plan = run_adapter_query(ADAPTER_RETAINED_BYTES / 4, 1).await?;
837860

838861
let spill_count = plan_spill_count(plan.as_ref());
839862
assert!(
840863
spill_count > 0,
841-
"the aggregate retains 4 MiB of scratch indices against a 1 MiB limit, \
842-
so it must spill, but spill_count was {spill_count}"
864+
"the aggregate retains {ADAPTER_RETAINED_BYTES} bytes of scratch indices \
865+
against a limit of a quarter of that, so it must spill, \
866+
but spill_count was {spill_count}"
867+
);
868+
869+
Ok(())
870+
}
871+
872+
/// The retained scratch capacity is charged once, not once per batch. Every
873+
/// group receives four batches, so charging the capacity per batch would
874+
/// report four times the retained bytes, and the memory limit of twice the
875+
/// retained bytes fits the aggregate only if it is charged once.
876+
#[tokio::test]
877+
async fn aggregate_adapter_charges_retained_indices_once() -> Result<()> {
878+
let plan = run_adapter_query(ADAPTER_RETAINED_BYTES * 2, 4).await?;
879+
880+
let spill_count = plan_spill_count(plan.as_ref());
881+
assert_eq!(
882+
spill_count, 0,
883+
"the aggregate retains {ADAPTER_RETAINED_BYTES} bytes of scratch indices \
884+
under a limit of twice that, so it must not spill"
843885
);
844886

845887
Ok(())

‎datafusion/functions-aggregate-common/src/aggregate/groups_accumulator.rs‎

Lines changed: 0 additions & 132 deletions
Original file line numberDiff line numberDiff line change
@@ -604,136 +604,4 @@ mod tests {
604604
assert!(!accumulator.supports_state_preserving());
605605
Ok(())
606606
}
607-
608-
/// Accumulator whose `size()` is constant, so that the only thing that can
609-
/// move the adapter's reported size is the adapter's own bookkeeping.
610-
#[derive(Debug, Default)]
611-
struct CountingAccumulator {
612-
count: i64,
613-
}
614-
615-
impl Accumulator for CountingAccumulator {
616-
fn update_batch(&mut self, values: &[ArrayRef]) -> Result<()> {
617-
self.count += values[0].len() as i64;
618-
Ok(())
619-
}
620-
621-
fn evaluate(&mut self) -> Result<ScalarValue> {
622-
Ok(ScalarValue::Int64(Some(self.count)))
623-
}
624-
625-
fn size(&self) -> usize {
626-
size_of::<Self>()
627-
}
628-
629-
fn state(&mut self) -> Result<Vec<ScalarValue>> {
630-
Ok(vec![ScalarValue::Int64(Some(self.count))])
631-
}
632-
633-
fn merge_batch(&mut self, states: &[ArrayRef]) -> Result<()> {
634-
let counts = states[0].as_primitive::<Int64Type>();
635-
self.count += counts.iter().flatten().sum::<i64>();
636-
Ok(())
637-
}
638-
}
639-
640-
fn test_adapter() -> GroupsAccumulatorAdapter {
641-
GroupsAccumulatorAdapter::new(|| Ok(Box::new(CountingAccumulator::default())))
642-
}
643-
644-
/// The size the adapter should be reporting, computed directly from the
645-
/// states it is holding rather than from its running total.
646-
fn retained_size(adapter: &GroupsAccumulatorAdapter) -> usize {
647-
adapter
648-
.states
649-
.iter()
650-
.map(|state| state.size())
651-
.sum::<usize>()
652-
+ adapter.states.allocated_size()
653-
}
654-
655-
/// Builds a batch where group 0 takes `hot_rows` rows and group 1 takes one.
656-
fn skewed_batch(hot_rows: usize) -> (Vec<ArrayRef>, Vec<usize>) {
657-
let mut group_indices = vec![0; hot_rows];
658-
group_indices.push(1);
659-
let values: ArrayRef = Arc::new(Int64Array::from(vec![1; group_indices.len()]));
660-
(vec![values], group_indices)
661-
}
662-
663-
fn update(adapter: &mut GroupsAccumulatorAdapter, hot_rows: usize) {
664-
let (values, group_indices) = skewed_batch(hot_rows);
665-
adapter
666-
.update_batch(&values, &group_indices, None, 2)
667-
.unwrap();
668-
}
669-
670-
#[test]
671-
fn accounts_for_retained_indices_capacity() {
672-
const HOT_ROWS: usize = 8192;
673-
674-
let mut adapter = test_adapter();
675-
update(&mut adapter, HOT_ROWS);
676-
677-
// The scratch `indices` vector for group 0 grew to hold `HOT_ROWS`
678-
// `u32`s and keeps that capacity after `clear()`, so it has to show up
679-
// in the reported size.
680-
assert_eq!(adapter.size(), retained_size(&adapter));
681-
assert!(
682-
adapter.size() >= HOT_ROWS * size_of::<u32>(),
683-
"reported {} bytes, which cannot cover {} retained indices",
684-
adapter.size(),
685-
HOT_ROWS
686-
);
687-
688-
// A second batch of the same shape reuses the existing capacity, so the
689-
// reported size must not move: the capacity is charged once, not once
690-
// per batch.
691-
let after_first_batch = adapter.size();
692-
update(&mut adapter, HOT_ROWS);
693-
assert_eq!(adapter.size(), after_first_batch);
694-
assert_eq!(adapter.size(), retained_size(&adapter));
695-
696-
// Growing the hot group further charges only the additional capacity.
697-
update(&mut adapter, HOT_ROWS * 4);
698-
assert!(adapter.size() > after_first_batch);
699-
assert_eq!(adapter.size(), retained_size(&adapter));
700-
701-
// A smaller batch afterwards keeps the capacity, and keeps charging it.
702-
let after_growth = adapter.size();
703-
update(&mut adapter, 1);
704-
assert_eq!(adapter.size(), after_growth);
705-
assert_eq!(adapter.size(), retained_size(&adapter));
706-
}
707-
708-
#[test]
709-
fn releases_retained_indices_capacity_on_emit() {
710-
const HOT_ROWS: usize = 8192;
711-
712-
let mut adapter = test_adapter();
713-
update(&mut adapter, HOT_ROWS);
714-
715-
// Emitting the hot group must release its retained capacity, and must
716-
// not release capacity that belongs to the groups left behind.
717-
adapter.state(EmitTo::First(1)).unwrap();
718-
assert_eq!(adapter.size(), retained_size(&adapter));
719-
720-
update(&mut adapter, HOT_ROWS);
721-
assert_eq!(adapter.size(), retained_size(&adapter));
722-
723-
adapter.evaluate(EmitTo::All).unwrap();
724-
assert_eq!(adapter.size(), 0);
725-
assert_eq!(adapter.size(), retained_size(&adapter));
726-
}
727-
728-
#[test]
729-
fn merge_batch_accounts_for_retained_indices_capacity() {
730-
const HOT_ROWS: usize = 8192;
731-
732-
let mut adapter = test_adapter();
733-
let (values, group_indices) = skewed_batch(HOT_ROWS);
734-
adapter.merge_batch(&values, &group_indices, 2).unwrap();
735-
736-
assert_eq!(adapter.size(), retained_size(&adapter));
737-
assert!(adapter.size() >= HOT_ROWS * size_of::<u32>());
738-
}
739607
}

0 commit comments

Comments
 (0)