Skip to content

Commit f441237

Browse files
GC view arrays during spill
Signed-off-by: bharath-techie <[email protected]>
1 parent 887aa9f commit f441237

1 file changed

Lines changed: 70 additions & 23 deletions

File tree

‎datafusion/physical-plan/src/spill/spill_manager.rs‎

Lines changed: 70 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -14,23 +14,19 @@
1414
// KIND, either express or implied. See the License for the
1515
// specific language governing permissions and limitations
1616
// under the License.
17-
1817
//! Define the `SpillManager` struct, which is responsible for reading and writing `RecordBatch`es to raw files based on the provided configurations.
19-
20-
use arrow::array::StringViewArray;
18+
use arrow::array::{ArrayRef, BinaryViewArray, StringViewArray};
2119
use arrow::datatypes::SchemaRef;
2220
use arrow::record_batch::RecordBatch;
2321
use datafusion_execution::runtime_env::RuntimeEnv;
2422
use std::sync::Arc;
25-
23+
use log::debug;
2624
use datafusion_common::{Result, config::SpillCompression};
2725
use datafusion_execution::SendableRecordBatchStream;
2826
use datafusion_execution::disk_manager::RefCountedTempFile;
29-
3027
use super::{SpillReaderStream, in_progress_spill_file::InProgressSpillFile};
3128
use crate::coop::cooperative;
3229
use crate::{common::spawn_buffered, metrics::SpillMetrics};
33-
3430
/// The `SpillManager` is responsible for the following tasks:
3531
/// - Reading and writing `RecordBatch`es to raw files based on the provided configurations.
3632
/// - Updating the associated metrics.
@@ -47,7 +43,42 @@ pub struct SpillManager {
4743
/// general-purpose compression options
4844
pub(crate) compression: SpillCompression,
4945
}
50-
46+
fn format_bytes(bytes: usize) -> String {
47+
const KB: usize = 1024;
48+
const MB: usize = 1024 * KB;
49+
const GB: usize = 1024 * MB;
50+
if bytes >= GB {
51+
format!("{:.2} GB", bytes as f64 / GB as f64)
52+
} else if bytes >= MB {
53+
format!("{:.2} MB", bytes as f64 / MB as f64)
54+
} else if bytes >= KB {
55+
format!("{:.2} KB", bytes as f64 / KB as f64)
56+
} else {
57+
format!("{} B", bytes)
58+
}
59+
}
60+
/// GC StringViewArray/BinaryViewArray to compact data buffers.
61+
/// Sliced view arrays still reference original large buffers; gc() creates compact copies.
62+
fn gc_view_arrays(batch: &RecordBatch) -> Result<RecordBatch> {
63+
let mut columns: Vec<ArrayRef> = Vec::with_capacity(batch.num_columns());
64+
let mut any_gc = false;
65+
for col in batch.columns() {
66+
if let Some(sv) = col.as_any().downcast_ref::<StringViewArray>() {
67+
columns.push(Arc::new(sv.gc()));
68+
any_gc = true;
69+
} else if let Some(bv) = col.as_any().downcast_ref::<BinaryViewArray>() {
70+
columns.push(Arc::new(bv.gc()));
71+
any_gc = true;
72+
} else {
73+
columns.push(Arc::clone(col));
74+
}
75+
}
76+
if any_gc {
77+
RecordBatch::try_new(batch.schema(), columns).map_err(Into::into)
78+
} else {
79+
Ok(batch.clone())
80+
}
81+
}
5182
impl SpillManager {
5283
pub fn new(env: Arc<RuntimeEnv>, metrics: SpillMetrics, schema: SchemaRef) -> Self {
5384
Self {
@@ -85,6 +116,10 @@ impl SpillManager {
85116
request_msg: &str,
86117
) -> Result<InProgressSpillFile> {
87118
let temp_file = self.env.disk_manager.create_tmp_file(request_msg)?;
119+
debug!(
120+
"[SPILL_MANAGER] Created temp file: {:?}",
121+
temp_file.path()
122+
);
88123
Ok(InProgressSpillFile::new(Arc::new(self.clone()), temp_file))
89124
}
90125

@@ -123,27 +158,40 @@ impl SpillManager {
123158
row_limit: usize,
124159
) -> Result<Option<(RefCountedTempFile, usize)>> {
125160
let total_rows = batch.num_rows();
126-
let mut batches = Vec::new();
161+
let mut in_progress_file = self.create_in_progress_file(request_description)?;
162+
let mut max_record_batch_size = 0;
163+
let mut total_written = 0usize;
127164
let mut offset = 0;
165+
let mut batch_idx = 0;
128166

129-
// It's ok to calculate all slices first, because slicing is zero-copy.
130167
while offset < total_rows {
131168
let length = std::cmp::min(total_rows - offset, row_limit);
132-
let sliced_batch = batch.slice(offset, length);
133-
batches.push(sliced_batch);
169+
let sliced = batch.slice(offset, length);
170+
171+
// Compare before/after gc
172+
let before_size = sliced.get_array_memory_size();
173+
let compacted = gc_view_arrays(&sliced)?;
174+
let after_size = compacted.get_array_memory_size();
175+
176+
debug!(
177+
"[SPILL_MANAGER] Batch {}: before_gc={}, after_gc={}, saved={}",
178+
batch_idx,
179+
format_bytes(before_size),
180+
format_bytes(after_size),
181+
format_bytes(before_size.saturating_sub(after_size))
182+
);
183+
in_progress_file.append_batch(&compacted)?;
184+
batch_idx += 1;
185+
max_record_batch_size = max_record_batch_size.max(after_size);
186+
total_written += after_size;
134187
offset += length;
135188
}
136-
137-
let mut in_progress_file = self.create_in_progress_file(request_description)?;
138-
139-
let mut max_record_batch_size = 0;
140-
141-
for batch in batches {
142-
in_progress_file.append_batch(&batch)?;
143-
144-
max_record_batch_size = max_record_batch_size.max(batch.get_sliced_size()?);
145-
}
146-
189+
debug!(
190+
"[SPILL_MANAGER] Total: {} batches, total_written={}, max_record={}",
191+
batch_idx,
192+
format_bytes(total_written),
193+
format_bytes(max_record_batch_size)
194+
);
147195
let file = in_progress_file.finish()?;
148196

149197
Ok(file.map(|f| (f, max_record_batch_size)))
@@ -172,7 +220,6 @@ impl SpillManager {
172220

173221
Ok(file.map(|f| (f, max_record_batch_size)))
174222
}
175-
176223
/// Reads a spill file as a stream. The file must be created by the current `SpillManager`.
177224
/// This method will generate output in FIFO order: the batch appended first
178225
/// will be read first.

0 commit comments

Comments
 (0)