Skip to content

Commit 25cc81b

Browse files
committed
GC & Deduplicate String View on Spill
1 parent f755cb4 commit 25cc81b

1 file changed

Lines changed: 17 additions & 3 deletions

File tree

  • datafusion/physical-plan/src/spill

‎datafusion/physical-plan/src/spill/mod.rs‎

Lines changed: 17 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ use std::task::{Context, Poll};
3434

3535
use arrow::array::{
3636
Array, ArrayRef, BinaryViewArray, BufferSpec, GenericByteViewArray, StringViewArray,
37-
layout, make_array,
37+
builder::GenericByteViewBuilder, layout, make_array,
3838
};
3939
use arrow::buffer::Buffer;
4040
use arrow::datatypes::DataType;
@@ -420,6 +420,20 @@ pub(crate) fn gc_view_arrays(batch: &RecordBatch) -> Result<RecordBatch> {
420420
}
421421
}
422422

423+
/// Garbage collect and deduplicate a string view before writing it out to disk
424+
///
425+
/// This is to avoid inflating disk usage and also to ensure that deduplication reduces memory pressure when reading back.
426+
fn gc_dedup_view<T: ByteViewType>(
427+
array: &GenericByteViewArray<T>,
428+
) -> GenericByteViewArray<T> {
429+
let mut builder = GenericByteViewBuilder::<T>::with_capacity(array.len())
430+
.with_deduplicate_strings();
431+
for v in array.iter() {
432+
builder.append_option(v);
433+
}
434+
builder.finish()
435+
}
436+
423437
fn gc_array(array: &ArrayRef) -> Result<(ArrayRef, bool)> {
424438
match array.data_type() {
425439
DataType::Utf8View => {
@@ -428,7 +442,7 @@ fn gc_array(array: &ArrayRef) -> Result<(ArrayRef, bool)> {
428442
.downcast_ref::<StringViewArray>()
429443
.expect("Utf8View array should downcast to StringViewArray");
430444
if should_gc_view_array(string_view) {
431-
Ok((Arc::new(string_view.gc()) as ArrayRef, true))
445+
Ok((Arc::new(gc_dedup_view(string_view)) as ArrayRef, true))
432446
} else {
433447
Ok((Arc::clone(array), false))
434448
}
@@ -439,7 +453,7 @@ fn gc_array(array: &ArrayRef) -> Result<(ArrayRef, bool)> {
439453
.downcast_ref::<BinaryViewArray>()
440454
.expect("BinaryView array should downcast to BinaryViewArray");
441455
if should_gc_view_array(binary_view) {
442-
Ok((Arc::new(binary_view.gc()) as ArrayRef, true))
456+
Ok((Arc::new(gc_dedup_view(binary_view)) as ArrayRef, true))
443457
} else {
444458
Ok((Arc::clone(array), false))
445459
}

0 commit comments

Comments
 (0)