2424//! See [`datafusion::execution::memory_pool`] for more information on how
2525//! DataFusion decides when operators should spill, and [`SpillFile`] for the
2626//! spill file abstraction this example implements.
27- use std:: future:: Future ;
28- use std:: io:: Write ;
27+ //!
28+ //! This example exercises the asynchronous external-sort spill path. Execution
29+ //! paths that require a partially written local file to be readable still use
30+ //! the synchronous spill API.
2931use std:: path:: Path as StdPath ;
3032use std:: pin:: Pin ;
3133use std:: sync:: Arc ;
3234use std:: sync:: atomic:: { AtomicU64 , Ordering } ;
3335
36+ use async_trait:: async_trait;
3437use bytes:: Bytes ;
35- use datafusion:: common:: Result ;
38+ use datafusion:: common:: { Result , not_impl_err } ;
3639use datafusion:: execution:: disk_manager:: DiskManagerBuilder ;
3740use datafusion:: execution:: runtime_env:: RuntimeEnvBuilder ;
38- use datafusion:: execution:: { SpillFile , SpillWriter , TempFileFactory } ;
41+ use datafusion:: execution:: { AsyncSpillWriter , SpillFile , SpillWriter , TempFileFactory } ;
3942use datafusion:: prelude:: { SessionConfig , SessionContext } ;
40- use datafusion_common:: exec_err;
4143use futures:: { Stream , StreamExt , TryStreamExt , stream} ;
4244use object_store:: local:: LocalFileSystem ;
4345use object_store:: path:: Path ;
44- use object_store:: { ObjectStore , ObjectStoreExt , PutPayload } ;
46+ use object_store:: { MultipartUpload , ObjectStore , ObjectStoreExt , PutPayloadMut } ;
4547use tempfile:: tempdir;
4648
49+ /// Most remote object stores require non-final multipart parts to be at least 5 MiB.
50+ const OBJECT_STORE_PART_SIZE : usize = 5 * 1024 * 1024 ;
51+
4752/// Demonstrates configuring DataFusion with spill files backed by an ObjectStore.
4853pub async fn object_store_spill ( ) -> Result < ( ) > {
4954 // A real system would use S3, GCS, Azure, or some other ObjectStore for
@@ -54,7 +59,7 @@ pub async fn object_store_spill() -> Result<()> {
5459 Arc :: new ( LocalFileSystem :: new_with_prefix ( tmp_dir. path ( ) ) ?) ;
5560
5661 // Create the custom TempFileFactory that creates spill files in the ObjectStore.
57- let temp_file_factory = Arc :: new ( ObjectStoreTempFileFactory :: new ( store) ) ;
62+ let temp_file_factory = Arc :: new ( ObjectStoreTempFileFactory :: new ( Arc :: clone ( & store) ) ) ;
5863 let disk_manager_builder =
5964 DiskManagerBuilder :: default ( ) . with_temp_file_factory ( temp_file_factory. clone ( ) ) ;
6065 let runtime = RuntimeEnvBuilder :: new ( )
@@ -94,6 +99,19 @@ pub async fn object_store_spill() -> Result<()> {
9499 temp_file_factory. created_files( ) > 0 ,
95100 "expected the custom TempFileFactory to be used for spilling"
96101 ) ;
102+ // Ensure the workload crosses a multipart boundary so the example uploads
103+ // at least one complete part before the final `finish` call.
104+ let spill_prefix = Path :: from ( "spill" ) ;
105+ let spill_objects = store
106+ . list ( Some ( & spill_prefix) )
107+ . try_collect :: < Vec < _ > > ( )
108+ . await ?;
109+ assert ! (
110+ spill_objects
111+ . iter( )
112+ . any( |object| object. size >= OBJECT_STORE_PART_SIZE as u64 ) ,
113+ "expected at least one spill object to exceed the multipart part size"
114+ ) ;
97115
98116 Ok ( ( ) )
99117}
@@ -154,8 +172,8 @@ impl TempFileFactory for ObjectStoreTempFileFactory {
154172
155173/// Logical spill file stored at an ObjectStore path.
156174///
157- /// DataFusion writes spill data by calling [`SpillFile::open_writer `] and reads
158- /// it back by calling [`SpillFile::read_stream`].
175+ /// DataFusion writes spill data by calling [`SpillFile::open_async_writer `] and
176+ /// reads it back by calling [`SpillFile::read_stream`].
159177struct ObjectStoreSpillFile {
160178 /// ObjectStore containing the spill object.
161179 store : Arc < dyn ObjectStore > ,
@@ -165,6 +183,7 @@ struct ObjectStoreSpillFile {
165183 size : Arc < AtomicU64 > ,
166184}
167185
186+ #[ async_trait]
168187impl SpillFile for ObjectStoreSpillFile {
169188 /// Return no local filesystem path because the spill file is accessed through ObjectStore.
170189 fn path ( & self ) -> Option < & StdPath > {
@@ -193,81 +212,180 @@ impl SpillFile for ObjectStoreSpillFile {
193212 Ok ( Box :: pin ( stream) )
194213 }
195214
196- /// Open a synchronous writer for this spill file .
215+ /// This example backend supports the asynchronous external-sort spill path only .
197216 fn open_writer ( & self ) -> Result < Box < dyn SpillWriter > > {
198- // Create a writer that buffers bytes and uploads them on finish.
217+ not_impl_err ! ( "Synchronous spill writing is not supported by this backend" )
218+ }
219+
220+ /// Open an asynchronous, multipart-capable writer for this spill file.
221+ async fn open_async_writer ( & self ) -> Result < Box < dyn AsyncSpillWriter > > {
222+ let upload = self . store . put_multipart ( & self . location ) . await ?;
199223 Ok ( Box :: new ( ObjectStoreSpillWriter {
200- store : Arc :: clone ( & self . store ) ,
201- location : self . location . clone ( ) ,
224+ upload ,
225+ buffer : PutPayloadMut :: new ( ) ,
202226 size : Arc :: clone ( & self . size ) ,
203- buffer : Vec :: new ( ) ,
227+ bytes_written : 0 ,
204228 } ) )
205229 }
206230}
207231
208- /// Adapts DataFusion's [`SpillWriter`] API to ObjectStore.
209- ///
210- /// This simple example buffers bytes in memory and uploads them in
211- /// [`SpillWriter::finish`]. A production remote implementation should consider
212- /// multipart or streaming uploads.
232+ /// Adapts DataFusion's [`AsyncSpillWriter`] API to ObjectStore.
213233struct ObjectStoreSpillWriter {
214- /// ObjectStore to read/write bytes to .
215- store : Arc < dyn ObjectStore > ,
216- /// ObjectStore path to upload to .
217- location : Path ,
234+ /// Multipart upload used to stream completed parts to the store .
235+ upload : Box < dyn MultipartUpload > ,
236+ /// Buffers at most one part while preserving owned `Bytes` chunks .
237+ buffer : PutPayloadMut ,
218238 /// Shared size field on the corresponding [`ObjectStoreSpillFile`].
219239 size : Arc < AtomicU64 > ,
220- /// Buffered spill bytes waiting to be uploaded.
221- ///
222- /// This simple example buffers the spill and uploads it on finish.
223- /// Production remote stores should consider multipart or streaming uploads.
224- buffer : Vec < u8 > ,
240+ /// Number of bytes passed to the writer.
241+ bytes_written : u64 ,
242+ }
243+
244+ impl ObjectStoreSpillWriter {
245+ async fn flush_part ( & mut self ) -> object_store:: Result < ( ) > {
246+ if self . buffer . is_empty ( ) {
247+ return Ok ( ( ) ) ;
248+ }
249+
250+ let part = std:: mem:: take ( & mut self . buffer ) . freeze ( ) ;
251+ self . upload . put_part ( part) . await
252+ }
225253}
226254
227- impl Write for ObjectStoreSpillWriter {
228- /// Append bytes to the in-memory buffer.
229- fn write ( & mut self , buf : & [ u8 ] ) -> std:: io:: Result < usize > {
230- // Buffer bytes written through the synchronous Write API.
231- self . buffer . extend_from_slice ( buf) ;
232- Ok ( buf. len ( ) )
255+ #[ async_trait]
256+ impl AsyncSpillWriter for ObjectStoreSpillWriter {
257+ async fn write_all ( & mut self , mut data : Bytes ) -> Result < ( ) > {
258+ let len = data. len ( ) as u64 ;
259+ while !data. is_empty ( ) {
260+ let remaining = OBJECT_STORE_PART_SIZE - self . buffer . content_length ( ) ;
261+ if data. len ( ) < remaining {
262+ // A Bytes slice can pin an entire Arrow allocation. Copy only
263+ // the tail retained after this call so it remains accurately
264+ // bounded after DataFusion releases the batch reservation.
265+ self . buffer . push ( Bytes :: copy_from_slice ( & data) ) ;
266+ break ;
267+ }
268+
269+ self . buffer . push ( data. split_to ( remaining) ) ;
270+ self . flush_part ( ) . await ?;
271+ }
272+ self . bytes_written += len;
273+ Ok ( ( ) )
274+ }
275+
276+ async fn finish ( & mut self ) -> Result < ( ) > {
277+ self . flush_part ( ) . await ?;
278+ self . upload . complete ( ) . await ?;
279+ self . size . store ( self . bytes_written , Ordering :: Relaxed ) ;
280+ Ok ( ( ) )
233281 }
234282
235- /// No-op because data is committed in [`SpillWriter::finish`].
236- fn flush ( & mut self ) -> std:: io:: Result < ( ) > {
283+ async fn abort ( & mut self ) -> Result < ( ) > {
284+ // Release locally buffered spill data before waiting on remote cleanup.
285+ self . buffer = PutPayloadMut :: new ( ) ;
286+ self . upload . abort ( ) . await ?;
237287 Ok ( ( ) )
238288 }
239289}
240290
241- impl SpillWriter for ObjectStoreSpillWriter {
242- /// Upload buffered bytes to ObjectStore and mark the spill file complete.
243- fn finish ( & mut self ) -> Result < ( ) > {
244- // Move the buffered bytes into the upload future.
245- let store = Arc :: clone ( & self . store ) ;
246- let location = self . location . clone ( ) ;
247- let data = std:: mem:: take ( & mut self . buffer ) ;
248- let size = data. len ( ) as u64 ;
249-
250- // This simple example buffers the spill and uploads it on finish.
251- // Production remote stores should consider multipart or streaming uploads.
252- block_on_object_store ( async move {
253- store
254- . put ( & location, PutPayload :: from_bytes ( data. into ( ) ) )
255- . await ?;
291+ #[ cfg( test) ]
292+ mod tests {
293+ use super :: * ;
294+ use object_store:: { PutPayload , PutResult , UploadPart } ;
295+ use std:: sync:: Mutex ;
296+ use std:: sync:: atomic:: AtomicBool ;
297+
298+ #[ derive( Debug ) ]
299+ struct RecordingMultipartUpload {
300+ part_sizes : Arc < Mutex < Vec < usize > > > ,
301+ }
302+
303+ #[ async_trait]
304+ impl MultipartUpload for RecordingMultipartUpload {
305+ fn put_part ( & mut self , data : PutPayload ) -> UploadPart {
306+ self . part_sizes . lock ( ) . unwrap ( ) . push ( data. content_length ( ) ) ;
307+ Box :: pin ( async { Ok ( ( ) ) } )
308+ }
309+
310+ async fn complete ( & mut self ) -> object_store:: Result < PutResult > {
311+ Ok ( PutResult {
312+ e_tag : None ,
313+ version : None ,
314+ } )
315+ }
316+
317+ async fn abort ( & mut self ) -> object_store:: Result < ( ) > {
256318 Ok ( ( ) )
257- } ) ?;
319+ }
320+ }
321+
322+ struct DropTrackingOwner {
323+ data : Vec < u8 > ,
324+ dropped : Arc < AtomicBool > ,
325+ }
326+
327+ impl AsRef < [ u8 ] > for DropTrackingOwner {
328+ fn as_ref ( & self ) -> & [ u8 ] {
329+ & self . data
330+ }
331+ }
332+
333+ impl Drop for DropTrackingOwner {
334+ fn drop ( & mut self ) {
335+ self . dropped . store ( true , Ordering :: Relaxed ) ;
336+ }
337+ }
338+
339+ #[ tokio:: test]
340+ async fn uploads_full_part_before_finish ( ) -> Result < ( ) > {
341+ let part_sizes = Arc :: new ( Mutex :: new ( Vec :: new ( ) ) ) ;
342+ let upload = RecordingMultipartUpload {
343+ part_sizes : Arc :: clone ( & part_sizes) ,
344+ } ;
345+ let mut writer = ObjectStoreSpillWriter {
346+ upload : Box :: new ( upload) ,
347+ buffer : PutPayloadMut :: new ( ) ,
348+ size : Arc :: new ( AtomicU64 :: new ( 0 ) ) ,
349+ bytes_written : 0 ,
350+ } ;
258351
259- self . size . store ( size, Ordering :: Relaxed ) ;
352+ writer
353+ . write_all ( Bytes :: from ( vec ! [ 42 ; OBJECT_STORE_PART_SIZE + 1 ] ) )
354+ . await ?;
355+ assert_eq ! (
356+ part_sizes. lock( ) . unwrap( ) . as_slice( ) ,
357+ & [ OBJECT_STORE_PART_SIZE ]
358+ ) ;
359+ assert_eq ! ( writer. buffer. content_length( ) , 1 ) ;
360+ writer. abort ( ) . await ?;
260361 Ok ( ( ) )
261362 }
262- }
263363
264- /// Run an async ObjectStore operation.
265- ///
266- /// Adding a native async API is tracked in <https://github.com/apache/datafusion/issues/23247>
267- fn block_on_object_store < T > ( future : impl Future < Output = Result < T > > ) -> Result < T > {
268- if let Ok ( handle) = tokio:: runtime:: Handle :: try_current ( ) {
269- tokio:: task:: block_in_place ( || handle. block_on ( future) )
270- } else {
271- exec_err ! ( "No current Tokio runtime available" )
364+ #[ tokio:: test]
365+ async fn buffered_tail_does_not_retain_input_allocation ( ) -> Result < ( ) > {
366+ let tmp_dir = tempdir ( ) ?;
367+ let store: Arc < dyn ObjectStore > =
368+ Arc :: new ( LocalFileSystem :: new_with_prefix ( tmp_dir. path ( ) ) ?) ;
369+ let location = Path :: from ( "tail-retention-test" ) ;
370+ let upload = store. put_multipart ( & location) . await ?;
371+ let dropped = Arc :: new ( AtomicBool :: new ( false ) ) ;
372+ let data = Bytes :: from_owner ( DropTrackingOwner {
373+ data : vec ! [ 42 ; 128 ] ,
374+ dropped : Arc :: clone ( & dropped) ,
375+ } ) ;
376+ let mut writer = ObjectStoreSpillWriter {
377+ upload,
378+ buffer : PutPayloadMut :: new ( ) ,
379+ size : Arc :: new ( AtomicU64 :: new ( 0 ) ) ,
380+ bytes_written : 0 ,
381+ } ;
382+
383+ writer. write_all ( data) . await ?;
384+ assert ! (
385+ dropped. load( Ordering :: Relaxed ) ,
386+ "buffered tail should not retain the input allocation"
387+ ) ;
388+ writer. abort ( ) . await ?;
389+ Ok ( ( ) )
272390 }
273391}
0 commit comments