1414// KIND, either express or implied. See the License for the
1515// specific language governing permissions and limitations
1616// under the License.
17-
1817//! Define the `SpillManager` struct, which is responsible for reading and writing `RecordBatch`es to raw files based on the provided configurations.
19-
20- use arrow:: array:: StringViewArray ;
18+ use arrow:: array:: { ArrayRef , BinaryViewArray , StringViewArray } ;
2119use arrow:: datatypes:: SchemaRef ;
2220use arrow:: record_batch:: RecordBatch ;
2321use datafusion_execution:: runtime_env:: RuntimeEnv ;
2422use std:: sync:: Arc ;
25-
23+ use log :: debug ;
2624use datafusion_common:: { Result , config:: SpillCompression } ;
2725use datafusion_execution:: SendableRecordBatchStream ;
2826use datafusion_execution:: disk_manager:: RefCountedTempFile ;
29-
3027use super :: { SpillReaderStream , in_progress_spill_file:: InProgressSpillFile } ;
3128use crate :: coop:: cooperative;
3229use crate :: { common:: spawn_buffered, metrics:: SpillMetrics } ;
33-
3430/// The `SpillManager` is responsible for the following tasks:
3531/// - Reading and writing `RecordBatch`es to raw files based on the provided configurations.
3632/// - Updating the associated metrics.
@@ -47,7 +43,42 @@ pub struct SpillManager {
4743 /// general-purpose compression options
4844 pub ( crate ) compression : SpillCompression ,
4945}
50-
46+ fn format_bytes ( bytes : usize ) -> String {
47+ const KB : usize = 1024 ;
48+ const MB : usize = 1024 * KB ;
49+ const GB : usize = 1024 * MB ;
50+ if bytes >= GB {
51+ format ! ( "{:.2} GB" , bytes as f64 / GB as f64 )
52+ } else if bytes >= MB {
53+ format ! ( "{:.2} MB" , bytes as f64 / MB as f64 )
54+ } else if bytes >= KB {
55+ format ! ( "{:.2} KB" , bytes as f64 / KB as f64 )
56+ } else {
57+ format ! ( "{} B" , bytes)
58+ }
59+ }
60+ /// GC StringViewArray/BinaryViewArray to compact data buffers.
61+ /// Sliced view arrays still reference original large buffers; gc() creates compact copies.
62+ fn gc_view_arrays ( batch : & RecordBatch ) -> Result < RecordBatch > {
63+ let mut columns: Vec < ArrayRef > = Vec :: with_capacity ( batch. num_columns ( ) ) ;
64+ let mut any_gc = false ;
65+ for col in batch. columns ( ) {
66+ if let Some ( sv) = col. as_any ( ) . downcast_ref :: < StringViewArray > ( ) {
67+ columns. push ( Arc :: new ( sv. gc ( ) ) ) ;
68+ any_gc = true ;
69+ } else if let Some ( bv) = col. as_any ( ) . downcast_ref :: < BinaryViewArray > ( ) {
70+ columns. push ( Arc :: new ( bv. gc ( ) ) ) ;
71+ any_gc = true ;
72+ } else {
73+ columns. push ( Arc :: clone ( col) ) ;
74+ }
75+ }
76+ if any_gc {
77+ RecordBatch :: try_new ( batch. schema ( ) , columns) . map_err ( Into :: into)
78+ } else {
79+ Ok ( batch. clone ( ) )
80+ }
81+ }
5182impl SpillManager {
5283 pub fn new ( env : Arc < RuntimeEnv > , metrics : SpillMetrics , schema : SchemaRef ) -> Self {
5384 Self {
@@ -85,6 +116,10 @@ impl SpillManager {
85116 request_msg : & str ,
86117 ) -> Result < InProgressSpillFile > {
87118 let temp_file = self . env . disk_manager . create_tmp_file ( request_msg) ?;
119+ debug ! (
120+ "[SPILL_MANAGER] Created temp file: {:?}" ,
121+ temp_file. path( )
122+ ) ;
88123 Ok ( InProgressSpillFile :: new ( Arc :: new ( self . clone ( ) ) , temp_file) )
89124 }
90125
@@ -123,27 +158,40 @@ impl SpillManager {
123158 row_limit : usize ,
124159 ) -> Result < Option < ( RefCountedTempFile , usize ) > > {
125160 let total_rows = batch. num_rows ( ) ;
126- let mut batches = Vec :: new ( ) ;
161+ let mut in_progress_file = self . create_in_progress_file ( request_description) ?;
162+ let mut max_record_batch_size = 0 ;
163+ let mut total_written = 0usize ;
127164 let mut offset = 0 ;
165+ let mut batch_idx = 0 ;
128166
129- // It's ok to calculate all slices first, because slicing is zero-copy.
130167 while offset < total_rows {
131168 let length = std:: cmp:: min ( total_rows - offset, row_limit) ;
132- let sliced_batch = batch. slice ( offset, length) ;
133- batches. push ( sliced_batch) ;
169+ let sliced = batch. slice ( offset, length) ;
170+
171+ // Compare before/after gc
172+ let before_size = sliced. get_array_memory_size ( ) ;
173+ let compacted = gc_view_arrays ( & sliced) ?;
174+ let after_size = compacted. get_array_memory_size ( ) ;
175+
176+ debug ! (
177+ "[SPILL_MANAGER] Batch {}: before_gc={}, after_gc={}, saved={}" ,
178+ batch_idx,
179+ format_bytes( before_size) ,
180+ format_bytes( after_size) ,
181+ format_bytes( before_size. saturating_sub( after_size) )
182+ ) ;
183+ in_progress_file. append_batch ( & compacted) ?;
184+ batch_idx += 1 ;
185+ max_record_batch_size = max_record_batch_size. max ( after_size) ;
186+ total_written += after_size;
134187 offset += length;
135188 }
136-
137- let mut in_progress_file = self . create_in_progress_file ( request_description) ?;
138-
139- let mut max_record_batch_size = 0 ;
140-
141- for batch in batches {
142- in_progress_file. append_batch ( & batch) ?;
143-
144- max_record_batch_size = max_record_batch_size. max ( batch. get_sliced_size ( ) ?) ;
145- }
146-
189+ debug ! (
190+ "[SPILL_MANAGER] Total: {} batches, total_written={}, max_record={}" ,
191+ batch_idx,
192+ format_bytes( total_written) ,
193+ format_bytes( max_record_batch_size)
194+ ) ;
147195 let file = in_progress_file. finish ( ) ?;
148196
149197 Ok ( file. map ( |f| ( f, max_record_batch_size) ) )
@@ -172,7 +220,6 @@ impl SpillManager {
172220
173221 Ok ( file. map ( |f| ( f, max_record_batch_size) ) )
174222 }
175-
176223 /// Reads a spill file as a stream. The file must be created by the current `SpillManager`.
177224 /// This method will generate output in FIFO order: the batch appended first
178225 /// will be read first.
0 commit comments