Skip to content

Commit a979e98

Browse files
committed
fix: preserve existing spill fallback behavior
Keep async spill support compatible with queries that already spill under tight memory limits. Retain workspace accounting without introducing a new failure condition or changing batching policy. Refs apache#23247
1 parent bdde6ec commit a979e98

3 files changed

Lines changed: 85 additions & 114 deletions

File tree

‎datafusion/physical-plan/src/sorts/builder.rs‎

Lines changed: 2 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -168,15 +168,12 @@ impl BatchBuilder {
168168
// is finished. This means all remaining rows from all but the last batch
169169
// for each stream have been yielded to the newly created record batch
170170
//
171-
// Keep the last batch only if its cursor still has unread rows.
172-
// Fully-consumed batches must release their reservation before the
173-
// output consumer reserves memory for the newly built batch.
171+
// We can therefore drop all but the last batch for each stream
174172
let mut batch_idx = 0;
175173
let mut retained = 0;
176174
self.batches.retain(|(stream_idx, batch)| {
177175
let stream_cursor = &mut self.cursors[*stream_idx];
178-
let retain = stream_cursor.batch_idx == batch_idx
179-
&& stream_cursor.row_idx < batch.num_rows();
176+
let retain = stream_cursor.batch_idx == batch_idx;
180177
batch_idx += 1;
181178

182179
if retain {
@@ -344,29 +341,6 @@ mod tests {
344341
assert!(matches!(error, DataFusionError::Execution(msg) if msg == "boom"));
345342
}
346343

347-
#[test]
348-
fn test_releases_fully_consumed_last_batch() -> Result<()> {
349-
let schema = Arc::new(Schema::new(vec![Field::new("x", DataType::Int32, false)]));
350-
let pool: Arc<dyn MemoryPool> = Arc::new(UnboundedMemoryPool::default());
351-
let reservation = MemoryConsumer::new("test").register(&pool);
352-
let mut builder = BatchBuilder::new(Arc::clone(&schema), 1, 1, reservation);
353-
for values in [[1, 2], [3, 4]] {
354-
let batch = RecordBatch::try_new(
355-
Arc::clone(&schema),
356-
vec![Arc::new(Int32Array::from(values.to_vec()))],
357-
)?;
358-
let size = get_record_batch_memory_size(&batch);
359-
builder.push_batch(0, batch)?;
360-
builder.push_row(0);
361-
assert_eq!(builder.build_record_batch()?.unwrap().num_rows(), 1);
362-
assert_eq!(pool.reserved(), size, "unread rows must remain reserved");
363-
builder.push_row(0);
364-
assert_eq!(builder.build_record_batch()?.unwrap().num_rows(), 1);
365-
assert_eq!(pool.reserved(), 0, "consumed batches must release memory");
366-
}
367-
Ok(())
368-
}
369-
370344
#[test]
371345
fn test_try_interleave_columns_surfaces_arrow_offset_overflow() {
372346
let batch = overflow_list_batch();

‎datafusion/physical-plan/src/sorts/sort.rs‎

Lines changed: 80 additions & 83 deletions
Original file line numberDiff line numberDiff line change
@@ -419,8 +419,16 @@ impl ExternalSorter {
419419
self.metrics.spill_metrics.spill_file_count.value()
420420
}
421421

422-
/// Appends a globally sorted batch, retaining its reservation until written.
423-
async fn consume_and_spill_append(&mut self, batch: RecordBatch) -> Result<()> {
422+
/// Appending globally sorted batches to the in-progress spill file, and clears
423+
/// the `globally_sorted_batches` (also its memory reservation) afterwards.
424+
async fn consume_and_spill_append(
425+
&mut self,
426+
globally_sorted_batches: &mut Vec<RecordBatch>,
427+
) -> Result<()> {
428+
if globally_sorted_batches.is_empty() {
429+
return Ok(());
430+
}
431+
424432
// Lazily initialize the in-progress spill file
425433
if self.in_progress_spill_file.is_none() {
426434
self.in_progress_spill_file =
@@ -429,7 +437,8 @@ impl ExternalSorter {
429437

430438
debug!("Spilling sort data of ExternalSorter to disk whilst inserting");
431439

432-
// Keep the reservation alive while the batch remains in memory across
440+
let batches_to_spill = std::mem::take(globally_sorted_batches);
441+
// Keep the reservation alive while the batches remain in memory across
433442
// asynchronous writes. It is released on success or error via RAII.
434443
let _spill_reservation = self.reservation.take();
435444

@@ -438,8 +447,16 @@ impl ExternalSorter {
438447
internal_datafusion_err!("In-progress spill file should be initialized")
439448
})?;
440449

441-
let gc_sliced_size = in_progress_file.append_batch_async(&batch).await?;
442-
*max_record_batch_size = (*max_record_batch_size).max(gc_sliced_size);
450+
for batch in batches_to_spill {
451+
let gc_sliced_size = in_progress_file.append_batch_async(&batch).await?;
452+
453+
*max_record_batch_size = (*max_record_batch_size).max(gc_sliced_size);
454+
}
455+
456+
assert_or_internal_err!(
457+
globally_sorted_batches.is_empty(),
458+
"This function consumes globally_sorted_batches, so it should be empty after taking."
459+
);
443460

444461
Ok(())
445462
}
@@ -503,35 +520,52 @@ impl ExternalSorter {
503520
self.in_mem_batches.is_empty(),
504521
"in_mem_batches should be empty after constructing sorted stream"
505522
);
523+
// 'global' here refers to all buffered batches when the memory limit is
524+
// reached. This variable will buffer the sorted batches after
525+
// sort-preserving merge and incrementally append to spill files.
526+
let mut globally_sorted_batches: Vec<RecordBatch> = vec![];
527+
506528
while let Some(batch) = sorted_stream.next().await {
507529
let batch = batch?;
508-
// Sorting is complete: retain the batch's footprint, not the input
509-
// estimate that also budgets for creating a sorted copy.
530+
// The output is already sorted, so no sorted-copy budget is needed.
510531
let sorted_size = get_record_batch_memory_size(&batch);
511532
let spill_workspace = match self.reservation.try_grow(sorted_size) {
512533
Ok(()) => None,
513534
Err(_) => {
514-
// Reuse already-reserved workspace without bypassing the
515-
// execution pool's limit. Any remainder still needs a grant
516-
// through the original sort consumer.
517535
let workspace = self.merge_pool.borrow(sorted_size);
518-
self.reservation.try_grow(sorted_size - workspace.size())?;
536+
// The batch is already materialized, so account for it while
537+
// spilling, reusing available workspace before growing the
538+
// reservation even if this exceeds the pool limit.
539+
self.reservation.grow(sorted_size - workspace.size());
519540
Some(workspace)
520541
}
521542
};
522-
// Write each batch before polling the merge again. Accumulating
523-
// output would compete with the merge's workspace without combining
524-
// any writes. Keep both forms of reservation alive across the await.
525-
self.consume_and_spill_append(batch).await?;
526-
drop(spill_workspace);
543+
// Even if the reservation is not enough, the batch is already in
544+
// memory, so it's okay to combine it with previously sorted
545+
// batches, and spill together.
546+
globally_sorted_batches.push(batch);
547+
if let Some(_spill_workspace) = spill_workspace {
548+
self.consume_and_spill_append(&mut globally_sorted_batches)
549+
.await?; // reservation is released when the spill completes
550+
}
527551
}
528552

529553
// Drop early to free up memory reserved by the sorted stream, otherwise the
530554
// upcoming `self.reserve_memory_for_merge()` may fail due to insufficient memory.
531555
drop(sorted_stream);
532556

557+
self.consume_and_spill_append(&mut globally_sorted_batches)
558+
.await?;
533559
self.spill_finish().await?;
534560

561+
// Sanity check after spilling
562+
let buffers_cleared_property =
563+
self.in_mem_batches.is_empty() && globally_sorted_batches.is_empty();
564+
assert_or_internal_err!(
565+
buffers_cleared_property,
566+
"in_mem_batches and globally_sorted_batches should be cleared before"
567+
);
568+
535569
// Reserve headroom for next sort/merge
536570
self.reserve_memory_for_merge()?;
537571

@@ -3355,32 +3389,6 @@ mod tests {
33553389
Ok(())
33563390
}
33573391

3358-
#[tokio::test]
3359-
async fn test_spill_output_respects_memory_limit() -> Result<()> {
3360-
let result = test_sort_output_batch_size_and_base_metrics(10, 25, |batches| {
3361-
let batches_memory = batches.iter().map(|b| b.get_array_memory_size()).sum();
3362-
TaskContext::default()
3363-
.with_session_config(
3364-
SessionConfig::new()
3365-
.with_batch_size(100)
3366-
.with_sort_in_place_threshold_bytes(1)
3367-
.with_sort_spill_reservation_bytes(1),
3368-
)
3369-
.with_runtime(
3370-
RuntimeEnvBuilder::default()
3371-
.with_memory_limit(batches_memory, 1.0)
3372-
.build_arc()
3373-
.unwrap(),
3374-
)
3375-
})
3376-
.await;
3377-
assert!(matches!(
3378-
result,
3379-
Err(DataFusionError::ResourcesExhausted(_))
3380-
));
3381-
Ok(())
3382-
}
3383-
33843392
#[tokio::test]
33853393
async fn should_return_stream_with_batches_in_the_requested_size_and_update_metrics_when_having_to_spill()
33863394
-> Result<()> {
@@ -3391,23 +3399,18 @@ mod tests {
33913399
.iter()
33923400
.map(|b| b.get_array_memory_size())
33933401
.sum::<usize>();
3394-
// Leave space for one output batch while the merge still holds
3395-
// partially consumed input batches. The insufficient-budget case
3396-
// is covered by test_spill_output_respects_memory_limit.
3397-
let spill_workspace =
3398-
make_partition(batch_size as i32).get_array_memory_size();
33993402

34003403
TaskContext::default()
34013404
.with_session_config(
34023405
SessionConfig::new()
34033406
.with_batch_size(batch_size)
34043407
// To make sure there is no in place sorting
34053408
.with_sort_in_place_threshold_bytes(1)
3406-
.with_sort_spill_reservation_bytes(spill_workspace),
3409+
.with_sort_spill_reservation_bytes(1),
34073410
)
34083411
.with_runtime(
34093412
RuntimeEnvBuilder::default()
3410-
.with_memory_limit(batches_memory + spill_workspace, 1.0)
3413+
.with_memory_limit(batches_memory, 1.0)
34113414
.build_arc()
34123415
.unwrap(),
34133416
)
@@ -3955,29 +3958,15 @@ mod tests {
39553958
Ok(())
39563959
}
39573960

3961+
#[rstest::rstest]
3962+
#[case::no_workspace(0)]
3963+
#[case::partial_workspace(6)]
3964+
#[case::full_workspace(12)]
39583965
#[tokio::test]
3959-
async fn test_spill_reservation_held_during_async_write() -> Result<()> {
3960-
let schema = Arc::new(Schema::new(vec![Field::new(
3961-
"x",
3962-
DataType::Utf8View,
3963-
false,
3964-
)]));
3965-
let batch = RecordBatch::try_new(
3966-
Arc::clone(&schema),
3967-
vec![Arc::new(StringViewArray::from_iter_values(
3968-
(0..4096).rev().map(|i| format!("{i:08}{}", "x".repeat(87))),
3969-
))],
3970-
)?;
3971-
let ordering =
3972-
[PhysicalSortExpr::new_default(Arc::new(Column::new("x", 0)))].into();
3973-
let batch_size = 1024;
3974-
let sorted_bytes = sort_batch_chunked(&batch, &ordering, batch_size)?
3975-
.iter()
3976-
.map(get_record_batch_memory_size)
3977-
.sum::<usize>();
3978-
let workspace_bytes = sorted_bytes - get_reserved_bytes_for_record_batch(&batch)?;
3979-
assert!(workspace_bytes > 0);
3980-
let pool = spill_tests::AdjustablePool::new(sorted_bytes);
3966+
async fn test_spill_reservation_held_during_async_write(
3967+
#[case] workspace_bytes: usize,
3968+
) -> Result<()> {
3969+
let pool: Arc<dyn MemoryPool> = Arc::new(GreedyMemoryPool::new(0));
39813970
let write_started = Arc::new(Notify::new());
39823971
let aborted = Arc::new(Notify::new());
39833972
let abort_count = Arc::new(AtomicUsize::new(0));
@@ -3989,26 +3978,34 @@ mod tests {
39893978
}),
39903979
);
39913980
let runtime = RuntimeEnvBuilder::new()
3992-
.with_memory_pool(Arc::clone(&pool) as Arc<dyn MemoryPool>)
3981+
.with_memory_pool(Arc::clone(&pool))
39933982
.with_disk_manager_builder(disk_manager_builder)
39943983
.build_arc()?;
3984+
let schema = Arc::new(Schema::new(vec![Field::new("x", DataType::Int32, false)]));
39953985
let metrics = ExecutionPlanMetricsSet::new();
39963986
let mut sorter = ExternalSorter::new(
39973987
0,
3998-
schema,
3999-
ordering,
4000-
batch_size,
4001-
workspace_bytes,
3988+
Arc::clone(&schema),
3989+
[PhysicalSortExpr::new_default(Arc::new(Column::new("x", 0)))].into(),
3990+
128,
40023991
0,
3992+
usize::MAX,
40033993
SpillCompression::Uncompressed,
40043994
&metrics,
40053995
runtime,
40063996
)?;
4007-
sorter.insert_batch(batch).await?;
4008-
// Existing reservations remain valid, but the emitted batch must reuse
4009-
// workspace rather than request new parent capacity.
4010-
pool.set_limit(sorted_bytes - 1);
3997+
let batch = RecordBatch::try_new(
3998+
schema,
3999+
vec![Arc::new(Int32Array::from(vec![3, 2, 1]))],
4000+
)?;
4001+
let reserved_bytes = get_record_batch_memory_size(&batch);
4002+
sorter
4003+
.reservation
4004+
.grow(get_reserved_bytes_for_record_batch(&batch)?);
4005+
sorter.merge_reservation.grow(workspace_bytes);
4006+
sorter.merge_pool.retain(workspace_bytes);
40114007
let merge_pool = Arc::clone(&sorter.merge_pool);
4008+
sorter.in_mem_batches.push(batch);
40124009

40134010
#[expect(clippy::disallowed_methods)] // spawn allowed only in tests
40144011
let task =
@@ -4025,12 +4022,12 @@ mod tests {
40254022
let _ = task.await;
40264023
panic!("spill write did not start before the timeout");
40274024
}
4028-
// Idle workspace must be releasable without releasing the loan held by
4029-
// the pending write. The stream still owns the remaining sorted batches.
4025+
// An idle retained workspace must not mask an early release of the loan.
40304026
merge_pool.release_unused();
4027+
assert_eq!(merge_pool.reserved(), workspace_bytes);
40314028
assert_eq!(
40324029
pool.reserved(),
4033-
sorted_bytes,
4030+
reserved_bytes,
40344031
"resident batches must remain accounted for while an async spill is pending"
40354032
);
40364033

‎datafusion/physical-plan/src/sorts/sort/spill_tests.rs‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -57,13 +57,13 @@ struct AllocationState {
5757
/// Existing allocations survive a lower limit, but fresh allocations must fit.
5858
/// The test changes the limit before insertion, independently of spill internals.
5959
#[derive(Debug)]
60-
pub(super) struct AdjustablePool {
60+
struct AdjustablePool {
6161
capacity: usize,
6262
state: Mutex<AllocationState>,
6363
}
6464

6565
impl AdjustablePool {
66-
pub(super) fn new(capacity: usize) -> Arc<Self> {
66+
fn new(capacity: usize) -> Arc<Self> {
6767
Arc::new(Self {
6868
capacity,
6969
state: Mutex::new(AllocationState {
@@ -73,7 +73,7 @@ impl AdjustablePool {
7373
})
7474
}
7575

76-
pub(super) fn set_limit(&self, limit: usize) {
76+
fn set_limit(&self, limit: usize) {
7777
assert!(limit <= self.capacity);
7878
self.state.lock().unwrap().limit = limit;
7979
}

0 commit comments

Comments
 (0)