@@ -34,7 +34,7 @@ use std::task::{Context, Poll};
3434
3535use arrow:: array:: {
3636 Array , ArrayRef , BinaryViewArray , BufferSpec , GenericByteViewArray , StringViewArray ,
37- layout, make_array,
37+ builder :: GenericByteViewBuilder , layout, make_array,
3838} ;
3939use arrow:: buffer:: Buffer ;
4040use arrow:: datatypes:: DataType ;
@@ -420,6 +420,20 @@ pub(crate) fn gc_view_arrays(batch: &RecordBatch) -> Result<RecordBatch> {
420420 }
421421}
422422
423+ /// Garbage collect and deduplicate a string view before writing it out to disk
424+ ///
425+ /// This is to avoid inflating disk usage and also to ensure that deduplication reduces memory pressure when reading back.
426+ fn gc_dedup_view < T : ByteViewType > (
427+ array : & GenericByteViewArray < T > ,
428+ ) -> GenericByteViewArray < T > {
429+ let mut builder = GenericByteViewBuilder :: < T > :: with_capacity ( array. len ( ) )
430+ . with_deduplicate_strings ( ) ;
431+ for v in array. iter ( ) {
432+ builder. append_option ( v) ;
433+ }
434+ builder. finish ( )
435+ }
436+
423437fn gc_array ( array : & ArrayRef ) -> Result < ( ArrayRef , bool ) > {
424438 match array. data_type ( ) {
425439 DataType :: Utf8View => {
@@ -428,7 +442,7 @@ fn gc_array(array: &ArrayRef) -> Result<(ArrayRef, bool)> {
428442 . downcast_ref :: < StringViewArray > ( )
429443 . expect ( "Utf8View array should downcast to StringViewArray" ) ;
430444 if should_gc_view_array ( string_view) {
431- Ok ( ( Arc :: new ( string_view . gc ( ) ) as ArrayRef , true ) )
445+ Ok ( ( Arc :: new ( gc_dedup_view ( string_view ) ) as ArrayRef , true ) )
432446 } else {
433447 Ok ( ( Arc :: clone ( array) , false ) )
434448 }
@@ -439,7 +453,7 @@ fn gc_array(array: &ArrayRef) -> Result<(ArrayRef, bool)> {
439453 . downcast_ref :: < BinaryViewArray > ( )
440454 . expect ( "BinaryView array should downcast to BinaryViewArray" ) ;
441455 if should_gc_view_array ( binary_view) {
442- Ok ( ( Arc :: new ( binary_view . gc ( ) ) as ArrayRef , true ) )
456+ Ok ( ( Arc :: new ( gc_dedup_view ( binary_view ) ) as ArrayRef , true ) )
443457 } else {
444458 Ok ( ( Arc :: clone ( array) , false ) )
445459 }
0 commit comments