@@ -1523,32 +1523,70 @@ impl OrderSensitiveArrayAggAccumulator {
15231523 } else {
15241524 ( values, None )
15251525 } ;
1526+
15261527 let ordering_values = filtered_ordering_values
15271528 . as_deref ( )
15281529 . unwrap_or ( ordering_values) ;
1529- // Detach the stored payload from potentially oversized backing buffers.
1530- let values = make_array ( copy_array_data ( & values. to_data ( ) ) ) ;
1531- let values = compact_payload ( values) ?;
15321530
1533- let row_count = values. len ( ) ;
15341531 // RowConverter validates the number, lengths, and types of ordering columns.
15351532 self . ordering_converter
15361533 . append ( & mut self . ordering_rows , ordering_values) ?;
1534+
1535+ let row_count = values. len ( ) ;
1536+
15371537 if row_count == 0 {
15381538 return Ok ( None ) ;
15391539 }
15401540
1541+ // Concat copies these payloads into fresh buffers when they fit in the tail.
1542+ let concat_copies = values. data_type ( ) . is_primitive ( )
1543+ || matches ! (
1544+ values. data_type( ) ,
1545+ DataType :: Boolean
1546+ | DataType :: Utf8
1547+ | DataType :: LargeUtf8
1548+ | DataType :: Binary
1549+ | DataType :: LargeBinary
1550+ ) ;
1551+
1552+ let fits_tail = |value : & ArrayRef | {
1553+ self . batches . last ( ) . is_some_and ( |last| {
1554+ last. len ( ) + row_count <= ORDERED_ARRAY_AGG_COALESCE_ROWS
1555+ && last. get_buffer_memory_size ( ) + value. get_array_memory_size ( )
1556+ <= ORDERED_ARRAY_AGG_COALESCE_BYTES
1557+ } )
1558+ } ;
1559+
1560+ // Skip detaching only when concat will copy this input into the tail.
1561+ let skip_detach = concat_copies && fits_tail ( & values) ;
1562+
1563+ // Detach unless concat is guaranteed to copy this input into the tail.
1564+ let values = if skip_detach {
1565+ values
1566+ } else {
1567+ compact_payload ( make_array ( copy_array_data ( & values. to_data ( ) ) ) ) ?
1568+ } ;
1569+
1570+ // Recheck the cap after compaction, which can shrink view and dictionary buffers.
1571+ let should_coalesce = fits_tail ( & values) ;
1572+
15411573 let start = self . entries . len ( ) ;
15421574 let ( batch_idx, row_offset) = match self . batches . last ( ) {
1543- Some ( last_batch)
1544- if last_batch. len ( ) + row_count <= ORDERED_ARRAY_AGG_COALESCE_ROWS
1545- && last_batch. get_buffer_memory_size ( )
1546- + values. get_buffer_memory_size ( )
1547- <= ORDERED_ARRAY_AGG_COALESCE_BYTES =>
1548- {
1575+ Some ( last_batch) if should_coalesce => {
15491576 let merged =
15501577 arrow:: compute:: concat ( & [ last_batch. as_ref ( ) , values. as_ref ( ) ] ) ?;
15511578
1579+ // View concat shares input data buffers. Compact them so the
1580+ // tail does not retain one buffer per small input batch.
1581+ let merged = if matches ! (
1582+ merged. data_type( ) ,
1583+ DataType :: Utf8View | DataType :: BinaryView
1584+ ) {
1585+ compact_payload ( merged) ?
1586+ } else {
1587+ merged
1588+ } ;
1589+
15521590 // Concatenation preserves the existing row offsets, while new rows
15531591 // start at the previous length of the tail batch.
15541592 let batch_idx = self . batches . len ( ) - 1 ;
@@ -2180,6 +2218,19 @@ mod tests {
21802218 Ok ( ( ) )
21812219 }
21822220
2221+ #[ test]
2222+ fn ordered_array_agg_detaches_unmerged_slices ( ) -> Result < ( ) > {
2223+ use arrow:: array:: { Int64Array , StringArray } ;
2224+
2225+ let integers = Int64Array :: from_iter_values ( 0 ..10_000 ) ;
2226+ let strings = StringArray :: from_iter_values (
2227+ ( 0 ..10_000 ) . map ( |i| format ! ( "payload-value-{i:03}" ) ) ,
2228+ ) ;
2229+ assert_compact_ordered_payload ( Arc :: new ( integers) , 2_000 ) ?;
2230+ assert_compact_ordered_payload ( Arc :: new ( strings) , 2_000 ) ?;
2231+ Ok ( ( ) )
2232+ }
2233+
21832234 #[ test]
21842235 fn ordered_array_agg_compacts_views ( ) -> Result < ( ) > {
21852236 use arrow:: array:: { BinaryViewArray , StringViewArray } ;
@@ -3089,17 +3140,59 @@ mod tests {
30893140 . collect:: <Vec <_>>( ) ,
30903141 vec![ ORDERED_ARRAY_AGG_COALESCE_ROWS , 36 ] ,
30913142 ) ;
3143+ assert_eq ! ( acc. batches[ 0 ] . as_string_view( ) . data_buffers( ) . len( ) , 1 ) ;
30923144
30933145 let ScalarValue :: List ( result) = acc. evaluate ( ) ? else {
30943146 return internal_err ! ( "expect list" ) ;
30953147 } ;
30963148
3097- let values = result
3098- . values ( )
3099- . as_any ( )
3100- . downcast_ref :: < StringViewArray > ( )
3101- . expect ( "expected StringViewArray" ) ;
3149+ let values = result. values ( ) . as_string_view ( ) ;
3150+
3151+ for i in 0 ..total_payloads {
3152+ assert_eq ! ( values. value( i) , value( i) ) ;
3153+ }
3154+
3155+ Ok ( ( ) )
3156+ }
3157+
3158+ #[ test]
3159+ fn ordered_array_agg_sorts_across_coalesced_binary_view_batches ( ) -> Result < ( ) > {
3160+ use arrow:: array:: { BinaryViewArray , Int64Array } ;
3161+
3162+ let mut acc = ordered_accumulator (
3163+ DataType :: BinaryView ,
3164+ DataType :: Int64 ,
3165+ SortOptions :: new ( false , false ) ,
3166+ false ,
3167+ false ,
3168+ ) ?;
3169+
3170+ // Longer than 12 bytes so view payloads are stored in data buffers.
3171+ let value = |i : usize | format ! ( "payload-value-{i:03}" ) . into_bytes ( ) ;
3172+ let total_payloads = 100 ;
3173+
3174+ for i in ( 0 ..total_payloads) . rev ( ) {
3175+ let payload =
3176+ Arc :: new ( BinaryViewArray :: from_iter_values ( [ value ( i) ] ) ) as ArrayRef ;
3177+ let ordering =
3178+ Arc :: new ( Int64Array :: from ( vec ! [ i64 :: try_from( i) . unwrap( ) ] ) ) as ArrayRef ;
3179+ acc. update_batch ( & [ payload, ordering] ) ?;
3180+ }
3181+
3182+ assert_eq ! (
3183+ acc. batches
3184+ . iter( )
3185+ . map( |batch| batch. len( ) )
3186+ . collect:: <Vec <_>>( ) ,
3187+ vec![ ORDERED_ARRAY_AGG_COALESCE_ROWS , 36 ] ,
3188+ ) ;
3189+ assert_eq ! ( acc. batches[ 0 ] . as_binary_view( ) . data_buffers( ) . len( ) , 1 ) ;
3190+
3191+ let ScalarValue :: List ( result) = acc. evaluate ( ) ? else {
3192+ return internal_err ! ( "expect list" ) ;
3193+ } ;
31023194
3195+ let values = result. values ( ) . as_binary_view ( ) ;
31033196 for i in 0 ..total_payloads {
31043197 assert_eq ! ( values. value( i) , value( i) ) ;
31053198 }
0 commit comments