Skip to content

Commit 7652570

Browse files
committed
Fix massive spill files for StringView/BinaryView columns
Add garbage collection for StringView and BinaryView arrays before spilling to disk. This prevents sliced arrays from carrying their entire original buffers when written to spill files. Changes: - Add gc_view_arrays() function to apply GC on view arrays - Integrate GC into InProgressSpillFile::append_batch() - Use simple threshold-based heuristic (100+ rows, 10KB+ buffer size) Fixes apache#19414 where GROUP BY on StringView columns created 820MB spill files instead of 33MB due to sliced arrays maintaining references to original buffers. Testing shows 80-98% reduction in spill file sizes for typical GROUP BY workloads.
1 parent 1825f97 commit 7652570

2 files changed

Lines changed: 521 additions & 4 deletions

File tree

‎datafusion/physical-plan/src/spill/in_progress_spill_file.rs‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ use arrow::array::RecordBatch;
2424
use datafusion_common::exec_datafusion_err;
2525
use datafusion_execution::disk_manager::RefCountedTempFile;
2626

27-
use super::{IPCStreamWriter, spill_manager::SpillManager};
27+
use super::{IPCStreamWriter, gc_view_arrays, spill_manager::SpillManager};
2828

2929
/// Represents an in-progress spill file used for writing `RecordBatch`es to disk, created by `SpillManager`.
3030
/// Caller is able to use this struct to incrementally append in-memory batches to
@@ -50,6 +50,7 @@ impl InProgressSpillFile {
5050
}
5151

5252
/// Appends a `RecordBatch` to the spill file, initializing the writer if necessary.
53+
/// Performs garbage collection on StringView/BinaryView arrays to reduce spill file size.
5354
///
5455
/// # Errors
5556
/// - Returns an error if the file is not active (has been finalized)
@@ -61,8 +62,11 @@ impl InProgressSpillFile {
6162
"Append operation failed: No active in-progress file. The file may have already been finalized."
6263
));
6364
}
65+
66+
let gc_batch = gc_view_arrays(batch)?;
67+
6468
if self.writer.is_none() {
65-
let schema = batch.schema();
69+
let schema = gc_batch.schema();
6670
if let Some(in_progress_file) = &mut self.in_progress_file {
6771
self.writer = Some(IPCStreamWriter::new(
6872
in_progress_file.path(),
@@ -83,7 +87,7 @@ impl InProgressSpillFile {
8387
}
8488
}
8589
if let Some(writer) = &mut self.writer {
86-
let (spilled_rows, _) = writer.write(batch)?;
90+
let (spilled_rows, _) = writer.write(&gc_batch)?;
8791
if let Some(in_progress_file) = &mut self.in_progress_file {
8892
let pre_size = in_progress_file.current_disk_usage();
8993
in_progress_file.update_disk_usage()?;

0 commit comments

Comments
 (0)