@@ -54,6 +54,10 @@ pub(crate) struct InProgressByteViewArray<B: ByteViewType> {
5454 /// Phantom so we can use the same struct for both StringViewArray and
5555 /// BinaryViewArray
5656 _phantom : PhantomData < B > ,
57+ /// The size in bytes the [`Buffer`]s in [`Self::completed`] is taking
58+ completed_buffers_size : usize ,
59+ /// The size in bytes from [`Self::source`] that it is being used in [`Self::completed`]
60+ size_of_completed_buffers_from_current_source : usize ,
5761}
5862
5963struct Source {
@@ -89,6 +93,8 @@ impl<B: ByteViewType> InProgressByteViewArray<B> {
8993 nulls : NullBufferBuilder :: new ( batch_size) , // no allocation
9094 current : None ,
9195 completed : vec ! [ ] ,
96+ completed_buffers_size : 0 ,
97+ size_of_completed_buffers_from_current_source : 0 ,
9298 buffer_source,
9399 _phantom : PhantomData ,
94100 }
@@ -109,7 +115,10 @@ impl<B: ByteViewType> InProgressByteViewArray<B> {
109115 let Some ( next_buffer) = self . current . take ( ) else {
110116 return ;
111117 } ;
112- self . completed . push ( next_buffer. into ( ) ) ;
118+ let buffer: Buffer = next_buffer. into ( ) ;
119+
120+ self . completed_buffers_size += buffer. capacity ( ) ;
121+ self . completed . push ( buffer) ;
113122 }
114123
115124 fn append_views_by_filter ( & mut self , views : & [ u128 ] , filter : & FilterPredicate ) {
@@ -169,10 +178,26 @@ impl<B: ByteViewType> InProgressByteViewArray<B> {
169178
170179 /// Append views to self.views, updating the buffer index if necessary
171180 #[ inline( never) ]
172- fn append_views_and_update_buffer_index ( & mut self , views : & [ u128 ] , buffers : & [ Buffer ] ) {
181+ fn append_views_and_update_buffer_index (
182+ & mut self ,
183+ views : & [ u128 ] ,
184+ buffers : & [ Buffer ] ,
185+ is_reused : bool ,
186+ ) {
173187 if let Some ( buffer) = self . current . take ( ) {
174- self . completed . push ( buffer. into ( ) ) ;
188+ let buffer: Buffer = buffer. into ( ) ;
189+ self . completed_buffers_size += buffer. capacity ( ) ;
190+ self . completed . push ( buffer) ;
175191 }
192+
193+ let buffers_size = buffers. iter ( ) . map ( |b| b. capacity ( ) ) . sum :: < usize > ( ) ;
194+ if !is_reused {
195+ self . completed_buffers_size += buffers_size;
196+ } else if self . size_of_completed_buffers_from_current_source == 0 {
197+ // Don't double count buffers size if already counted that
198+ self . size_of_completed_buffers_from_current_source += buffers_size;
199+ }
200+
176201 let starting_buffer: u32 = self . completed . len ( ) . try_into ( ) . expect ( "too many buffers" ) ;
177202 self . completed . extend_from_slice ( buffers) ;
178203
@@ -251,8 +276,9 @@ impl<B: ByteViewType> InProgressByteViewArray<B> {
251276 let remaining_view_buffer_size = view_buffer_size - string_bytes_to_copy;
252277
253278 self . append_views_and_copy_strings_inner ( first_views, current, buffers) ;
254- let completed = self . current . take ( ) . expect ( "completed" ) ;
255- self . completed . push ( completed. into ( ) ) ;
279+ let completed: Buffer = self . current . take ( ) . expect ( "completed" ) . into ( ) ;
280+ self . completed_buffers_size += completed. capacity ( ) ;
281+ self . completed . push ( completed) ;
256282
257283 // Copy any remaining views into a new buffer
258284 let remaining_views = & views[ num_view_to_current..] ;
@@ -328,13 +354,18 @@ impl<B: ByteViewType> InProgressByteViewArray<B> {
328354 }
329355 b. as_u128 ( )
330356 } ) ;
357+
331358 self . views . extend ( new_views) ;
332359 self . current = Some ( dst_buffer) ;
333360 }
334361}
335362
336363impl < B : ByteViewType > InProgressArray for InProgressByteViewArray < B > {
337364 fn set_source ( & mut self , source : Option < ArrayRef > ) {
365+ // If used values from source, add only the size that was used
366+ self . completed_buffers_size += self . size_of_completed_buffers_from_current_source ;
367+ self . size_of_completed_buffers_from_current_source = 0 ;
368+
338369 self . source = source. map ( |array| {
339370 let s = array. as_byte_view :: < B > ( ) ;
340371
@@ -358,7 +389,7 @@ impl<B: ByteViewType> InProgressArray for InProgressByteViewArray<B> {
358389 need_gc,
359390 ideal_buffer_size,
360391 }
361- } )
392+ } ) ;
362393 }
363394
364395 fn copy_rows ( & mut self , offset : usize , len : usize ) -> Result < ( ) , ArrowError > {
@@ -397,7 +428,7 @@ impl<B: ByteViewType> InProgressArray for InProgressByteViewArray<B> {
397428 if source. need_gc {
398429 self . append_views_and_copy_strings ( views, source. ideal_buffer_size , buffers) ;
399430 } else {
400- self . append_views_and_update_buffer_index ( views, buffers) ;
431+ self . append_views_and_update_buffer_index ( views, buffers, true ) ;
401432 }
402433 self . source = Some ( source) ;
403434 Ok ( ( ) )
@@ -452,7 +483,7 @@ impl<B: ByteViewType> InProgressArray for InProgressByteViewArray<B> {
452483 } else {
453484 self . nulls . append_n_non_nulls ( filter. count ( ) ) ;
454485 }
455- self . append_views_and_update_buffer_index ( filtered. views ( ) , filtered. data_buffers ( ) ) ;
486+ self . append_views_and_update_buffer_index ( filtered. views ( ) , filtered. data_buffers ( ) , false ) ;
456487 Ok ( ( ) )
457488 }
458489
@@ -464,12 +495,27 @@ impl<B: ByteViewType> InProgressArray for InProgressByteViewArray<B> {
464495 let nulls = self . nulls . finish ( ) ;
465496 self . nulls = NullBufferBuilder :: new ( self . batch_size ) ;
466497
498+ // Not reusing anything since we took all complete
499+ self . size_of_completed_buffers_from_current_source = 0 ;
500+ self . completed_buffers_size = 0 ;
501+
467502 // Safety: we created valid views and buffers above and the
468503 // input arrays had value data and nulls
469504 let new_array =
470505 unsafe { GenericByteViewArray :: < B > :: new_unchecked ( views. into ( ) , buffers, nulls) } ;
471506 Ok ( Arc :: new ( new_array) )
472507 }
508+
509+ fn size ( & self ) -> usize {
510+ self . completed_buffers_size
511+ + self . current . as_ref ( ) . map_or ( 0 , |c| c. capacity ( ) )
512+ + self . nulls . allocated_size ( )
513+ + self . views . capacity ( ) * size_of :: < u128 > ( )
514+ + self
515+ . source
516+ . as_ref ( )
517+ . map_or ( 0 , |s| s. array . get_array_memory_size ( ) )
518+ }
473519}
474520
475521const STARTING_BLOCK_SIZE : usize = 4 * 1024 ; // (note the first size used is actually 8KiB)
@@ -607,4 +653,87 @@ mod tests {
607653 "expected filtered output to reuse the source data buffer"
608654 ) ;
609655 }
656+
657+ /// Build a compacted BinaryViewArray whose values all spill into external
658+ /// data buffers. `gc()` makes the buffers dense so the coalescer reuses them
659+ /// (`need_gc == false`) rather than copying/compacting them.
660+ fn non_inline_array ( n : usize ) -> ( BinaryViewArray , usize ) {
661+ let values = ( 0 ..n)
662+ . map ( |i| format ! ( "This value is longer than 12 bytes: {i}" ) . into_bytes ( ) )
663+ . collect :: < Vec < _ > > ( ) ;
664+ let array = BinaryViewArray :: from_iter ( values. iter ( ) . map ( |v| Some ( v. as_slice ( ) ) ) ) . gc ( ) ;
665+ assert ! ( !array. data_buffers( ) . is_empty( ) ) ;
666+ let buffer_capacity = array. data_buffers ( ) . iter ( ) . map ( |b| b. capacity ( ) ) . sum ( ) ;
667+ ( array, buffer_capacity)
668+ }
669+
670+ #[ test]
671+ fn test_size_empty ( ) {
672+ let in_progress = InProgressByteViewArray :: < BinaryViewType > :: new ( 64 ) ;
673+ assert_eq ! ( in_progress. size( ) , 0 ) ;
674+ }
675+
676+ #[ test]
677+ fn test_size_reused_buffers_not_double_counted ( ) {
678+ let ( array, buffer_capacity) = non_inline_array ( 64 ) ;
679+ let source: ArrayRef = Arc :: new ( array) ;
680+
681+ let mut in_progress = InProgressByteViewArray :: < BinaryViewType > :: new ( 64 ) ;
682+ in_progress. set_source ( Some ( Arc :: clone ( & source) ) ) ;
683+
684+ in_progress. copy_rows ( 0 , 60 ) . unwrap ( ) ;
685+ in_progress. copy_rows ( 60 , 4 ) . unwrap ( ) ;
686+
687+ // The reused buffers now live in `completed`, but while the source is
688+ // still set they are counted via the source, not `completed_buffers_size`,
689+ // to avoid double counting them.
690+ assert_eq ! ( in_progress. completed_buffers_size, 0 ) ;
691+ assert_eq ! (
692+ in_progress. size_of_completed_buffers_from_current_source,
693+ buffer_capacity,
694+ ) ;
695+
696+ // Setting a new source commits the pending reused-buffer bytes, since the
697+ // old source (and its double count) is dropped.
698+ let other: ArrayRef = Arc :: new ( BinaryViewArray :: from_iter ( std:: iter:: once ( Some (
699+ b"short" . as_slice ( ) ,
700+ ) ) ) ) ;
701+ in_progress. set_source ( Some ( Arc :: clone ( & other) ) ) ;
702+ assert_eq ! ( in_progress. completed_buffers_size, buffer_capacity) ;
703+ assert_eq ! ( in_progress. size_of_completed_buffers_from_current_source, 0 ) ;
704+
705+ // finish() releases everything.
706+ in_progress. finish ( ) . unwrap ( ) ;
707+ assert_eq ! ( in_progress. completed_buffers_size, 0 ) ;
708+ assert_eq ! ( in_progress. size_of_completed_buffers_from_current_source, 0 ) ;
709+ }
710+
711+ #[ test]
712+ fn size_should_be_the_same_if_copying_multiple_time_from_same_source_or_once ( ) {
713+ let ( array, _buffer_capacity) = non_inline_array ( 64 ) ;
714+ let source: ArrayRef = Arc :: new ( array) ;
715+
716+ let in_progress_size_with_split = {
717+ let mut in_progress = InProgressByteViewArray :: < BinaryViewType > :: new ( 64 ) ;
718+ in_progress. set_source ( Some ( Arc :: clone ( & source) ) ) ;
719+
720+ in_progress. copy_rows ( 0 , 60 ) . unwrap ( ) ;
721+ in_progress. copy_rows ( 60 , 4 ) . unwrap ( ) ;
722+ in_progress. set_source ( None ) ;
723+
724+ in_progress. size ( )
725+ } ;
726+
727+ let in_progress_size_without_split = {
728+ let mut in_progress = InProgressByteViewArray :: < BinaryViewType > :: new ( 64 ) ;
729+ in_progress. set_source ( Some ( Arc :: clone ( & source) ) ) ;
730+
731+ in_progress. copy_rows ( 0 , 64 ) . unwrap ( ) ;
732+ in_progress. set_source ( None ) ;
733+
734+ in_progress. size ( )
735+ } ;
736+
737+ assert_eq ! ( in_progress_size_with_split, in_progress_size_without_split) ;
738+ }
610739}
0 commit comments