Skip to content

Commit ed00fe4

Browse files
committed
feat: support async spill writers
Object store spill backends previously had to block while uploading data. Add an async writer path so external sort can stream multipart uploads without buffering complete spill files in memory. CLOSES apache#23247
1 parent 74ee24b commit ed00fe4

7 files changed

Lines changed: 1244 additions & 201 deletions

File tree

‎datafusion-examples/examples/data_io/object_store_spill.rs‎

Lines changed: 180 additions & 62 deletions
Original file line numberDiff line numberDiff line change
@@ -24,26 +24,31 @@
2424
//! See [`datafusion::execution::memory_pool`] for more information on how
2525
//! DataFusion decides when operators should spill, and [`SpillFile`] for the
2626
//! spill file abstraction this example implements.
27-
use std::future::Future;
28-
use std::io::Write;
27+
//!
28+
//! This example exercises the asynchronous external-sort spill path. Execution
29+
//! paths that require a partially written local file to be readable still use
30+
//! the synchronous spill API.
2931
use std::path::Path as StdPath;
3032
use std::pin::Pin;
3133
use std::sync::Arc;
3234
use std::sync::atomic::{AtomicU64, Ordering};
3335

36+
use async_trait::async_trait;
3437
use bytes::Bytes;
35-
use datafusion::common::Result;
38+
use datafusion::common::{Result, not_impl_err};
3639
use datafusion::execution::disk_manager::DiskManagerBuilder;
3740
use datafusion::execution::runtime_env::RuntimeEnvBuilder;
38-
use datafusion::execution::{SpillFile, SpillWriter, TempFileFactory};
41+
use datafusion::execution::{AsyncSpillWriter, SpillFile, SpillWriter, TempFileFactory};
3942
use datafusion::prelude::{SessionConfig, SessionContext};
40-
use datafusion_common::exec_err;
4143
use futures::{Stream, StreamExt, TryStreamExt, stream};
4244
use object_store::local::LocalFileSystem;
4345
use object_store::path::Path;
44-
use object_store::{ObjectStore, ObjectStoreExt, PutPayload};
46+
use object_store::{MultipartUpload, ObjectStore, ObjectStoreExt, PutPayloadMut};
4547
use tempfile::tempdir;
4648

49+
/// Most remote object stores require non-final multipart parts to be at least 5 MiB.
50+
const OBJECT_STORE_PART_SIZE: usize = 5 * 1024 * 1024;
51+
4752
/// Demonstrates configuring DataFusion with spill files backed by an ObjectStore.
4853
pub async fn object_store_spill() -> Result<()> {
4954
// A real system would use S3, GCS, Azure, or some other ObjectStore for
@@ -54,7 +59,7 @@ pub async fn object_store_spill() -> Result<()> {
5459
Arc::new(LocalFileSystem::new_with_prefix(tmp_dir.path())?);
5560

5661
// Create the custom TempFileFactory that creates spill files in the ObjectStore.
57-
let temp_file_factory = Arc::new(ObjectStoreTempFileFactory::new(store));
62+
let temp_file_factory = Arc::new(ObjectStoreTempFileFactory::new(Arc::clone(&store)));
5863
let disk_manager_builder =
5964
DiskManagerBuilder::default().with_temp_file_factory(temp_file_factory.clone());
6065
let runtime = RuntimeEnvBuilder::new()
@@ -94,6 +99,19 @@ pub async fn object_store_spill() -> Result<()> {
9499
temp_file_factory.created_files() > 0,
95100
"expected the custom TempFileFactory to be used for spilling"
96101
);
102+
// Ensure the workload crosses a multipart boundary so the example uploads
103+
// at least one complete part before the final `finish` call.
104+
let spill_prefix = Path::from("spill");
105+
let spill_objects = store
106+
.list(Some(&spill_prefix))
107+
.try_collect::<Vec<_>>()
108+
.await?;
109+
assert!(
110+
spill_objects
111+
.iter()
112+
.any(|object| object.size >= OBJECT_STORE_PART_SIZE as u64),
113+
"expected at least one spill object to exceed the multipart part size"
114+
);
97115

98116
Ok(())
99117
}
@@ -154,8 +172,8 @@ impl TempFileFactory for ObjectStoreTempFileFactory {
154172

155173
/// Logical spill file stored at an ObjectStore path.
156174
///
157-
/// DataFusion writes spill data by calling [`SpillFile::open_writer`] and reads
158-
/// it back by calling [`SpillFile::read_stream`].
175+
/// DataFusion writes spill data by calling [`SpillFile::open_async_writer`] and
176+
/// reads it back by calling [`SpillFile::read_stream`].
159177
struct ObjectStoreSpillFile {
160178
/// ObjectStore containing the spill object.
161179
store: Arc<dyn ObjectStore>,
@@ -165,6 +183,7 @@ struct ObjectStoreSpillFile {
165183
size: Arc<AtomicU64>,
166184
}
167185

186+
#[async_trait]
168187
impl SpillFile for ObjectStoreSpillFile {
169188
/// Return no local filesystem path because the spill file is accessed through ObjectStore.
170189
fn path(&self) -> Option<&StdPath> {
@@ -193,81 +212,180 @@ impl SpillFile for ObjectStoreSpillFile {
193212
Ok(Box::pin(stream))
194213
}
195214

196-
/// Open a synchronous writer for this spill file.
215+
/// This example backend supports the asynchronous external-sort spill path only.
197216
fn open_writer(&self) -> Result<Box<dyn SpillWriter>> {
198-
// Create a writer that buffers bytes and uploads them on finish.
217+
not_impl_err!("Synchronous spill writing is not supported by this backend")
218+
}
219+
220+
/// Open an asynchronous, multipart-capable writer for this spill file.
221+
async fn open_async_writer(&self) -> Result<Box<dyn AsyncSpillWriter>> {
222+
let upload = self.store.put_multipart(&self.location).await?;
199223
Ok(Box::new(ObjectStoreSpillWriter {
200-
store: Arc::clone(&self.store),
201-
location: self.location.clone(),
224+
upload,
225+
buffer: PutPayloadMut::new(),
202226
size: Arc::clone(&self.size),
203-
buffer: Vec::new(),
227+
bytes_written: 0,
204228
}))
205229
}
206230
}
207231

208-
/// Adapts DataFusion's [`SpillWriter`] API to ObjectStore.
209-
///
210-
/// This simple example buffers bytes in memory and uploads them in
211-
/// [`SpillWriter::finish`]. A production remote implementation should consider
212-
/// multipart or streaming uploads.
232+
/// Adapts DataFusion's [`AsyncSpillWriter`] API to ObjectStore.
213233
struct ObjectStoreSpillWriter {
214-
/// ObjectStore to read/write bytes to.
215-
store: Arc<dyn ObjectStore>,
216-
/// ObjectStore path to upload to.
217-
location: Path,
234+
/// Multipart upload used to stream completed parts to the store.
235+
upload: Box<dyn MultipartUpload>,
236+
/// Buffers at most one part while preserving owned `Bytes` chunks.
237+
buffer: PutPayloadMut,
218238
/// Shared size field on the corresponding [`ObjectStoreSpillFile`].
219239
size: Arc<AtomicU64>,
220-
/// Buffered spill bytes waiting to be uploaded.
221-
///
222-
/// This simple example buffers the spill and uploads it on finish.
223-
/// Production remote stores should consider multipart or streaming uploads.
224-
buffer: Vec<u8>,
240+
/// Number of bytes passed to the writer.
241+
bytes_written: u64,
242+
}
243+
244+
impl ObjectStoreSpillWriter {
245+
async fn flush_part(&mut self) -> object_store::Result<()> {
246+
if self.buffer.is_empty() {
247+
return Ok(());
248+
}
249+
250+
let part = std::mem::take(&mut self.buffer).freeze();
251+
self.upload.put_part(part).await
252+
}
225253
}
226254

227-
impl Write for ObjectStoreSpillWriter {
228-
/// Append bytes to the in-memory buffer.
229-
fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
230-
// Buffer bytes written through the synchronous Write API.
231-
self.buffer.extend_from_slice(buf);
232-
Ok(buf.len())
255+
#[async_trait]
256+
impl AsyncSpillWriter for ObjectStoreSpillWriter {
257+
async fn write_all(&mut self, mut data: Bytes) -> Result<()> {
258+
let len = data.len() as u64;
259+
while !data.is_empty() {
260+
let remaining = OBJECT_STORE_PART_SIZE - self.buffer.content_length();
261+
if data.len() < remaining {
262+
// A Bytes slice can pin an entire Arrow allocation. Copy only
263+
// the tail retained after this call so it remains accurately
264+
// bounded after DataFusion releases the batch reservation.
265+
self.buffer.push(Bytes::copy_from_slice(&data));
266+
break;
267+
}
268+
269+
self.buffer.push(data.split_to(remaining));
270+
self.flush_part().await?;
271+
}
272+
self.bytes_written += len;
273+
Ok(())
274+
}
275+
276+
async fn finish(&mut self) -> Result<()> {
277+
self.flush_part().await?;
278+
self.upload.complete().await?;
279+
self.size.store(self.bytes_written, Ordering::Relaxed);
280+
Ok(())
233281
}
234282

235-
/// No-op because data is committed in [`SpillWriter::finish`].
236-
fn flush(&mut self) -> std::io::Result<()> {
283+
async fn abort(&mut self) -> Result<()> {
284+
// Release locally buffered spill data before waiting on remote cleanup.
285+
self.buffer = PutPayloadMut::new();
286+
self.upload.abort().await?;
237287
Ok(())
238288
}
239289
}
240290

241-
impl SpillWriter for ObjectStoreSpillWriter {
242-
/// Upload buffered bytes to ObjectStore and mark the spill file complete.
243-
fn finish(&mut self) -> Result<()> {
244-
// Move the buffered bytes into the upload future.
245-
let store = Arc::clone(&self.store);
246-
let location = self.location.clone();
247-
let data = std::mem::take(&mut self.buffer);
248-
let size = data.len() as u64;
249-
250-
// This simple example buffers the spill and uploads it on finish.
251-
// Production remote stores should consider multipart or streaming uploads.
252-
block_on_object_store(async move {
253-
store
254-
.put(&location, PutPayload::from_bytes(data.into()))
255-
.await?;
291+
#[cfg(test)]
292+
mod tests {
293+
use super::*;
294+
use object_store::{PutPayload, PutResult, UploadPart};
295+
use std::sync::Mutex;
296+
use std::sync::atomic::AtomicBool;
297+
298+
#[derive(Debug)]
299+
struct RecordingMultipartUpload {
300+
part_sizes: Arc<Mutex<Vec<usize>>>,
301+
}
302+
303+
#[async_trait]
304+
impl MultipartUpload for RecordingMultipartUpload {
305+
fn put_part(&mut self, data: PutPayload) -> UploadPart {
306+
self.part_sizes.lock().unwrap().push(data.content_length());
307+
Box::pin(async { Ok(()) })
308+
}
309+
310+
async fn complete(&mut self) -> object_store::Result<PutResult> {
311+
Ok(PutResult {
312+
e_tag: None,
313+
version: None,
314+
})
315+
}
316+
317+
async fn abort(&mut self) -> object_store::Result<()> {
256318
Ok(())
257-
})?;
319+
}
320+
}
321+
322+
struct DropTrackingOwner {
323+
data: Vec<u8>,
324+
dropped: Arc<AtomicBool>,
325+
}
326+
327+
impl AsRef<[u8]> for DropTrackingOwner {
328+
fn as_ref(&self) -> &[u8] {
329+
&self.data
330+
}
331+
}
332+
333+
impl Drop for DropTrackingOwner {
334+
fn drop(&mut self) {
335+
self.dropped.store(true, Ordering::Relaxed);
336+
}
337+
}
338+
339+
#[tokio::test]
340+
async fn uploads_full_part_before_finish() -> Result<()> {
341+
let part_sizes = Arc::new(Mutex::new(Vec::new()));
342+
let upload = RecordingMultipartUpload {
343+
part_sizes: Arc::clone(&part_sizes),
344+
};
345+
let mut writer = ObjectStoreSpillWriter {
346+
upload: Box::new(upload),
347+
buffer: PutPayloadMut::new(),
348+
size: Arc::new(AtomicU64::new(0)),
349+
bytes_written: 0,
350+
};
258351

259-
self.size.store(size, Ordering::Relaxed);
352+
writer
353+
.write_all(Bytes::from(vec![42; OBJECT_STORE_PART_SIZE + 1]))
354+
.await?;
355+
assert_eq!(
356+
part_sizes.lock().unwrap().as_slice(),
357+
&[OBJECT_STORE_PART_SIZE]
358+
);
359+
assert_eq!(writer.buffer.content_length(), 1);
360+
writer.abort().await?;
260361
Ok(())
261362
}
262-
}
263363

264-
/// Run an async ObjectStore operation.
265-
///
266-
/// Adding a native async API is tracked in <https://github.com/apache/datafusion/issues/23247>
267-
fn block_on_object_store<T>(future: impl Future<Output = Result<T>>) -> Result<T> {
268-
if let Ok(handle) = tokio::runtime::Handle::try_current() {
269-
tokio::task::block_in_place(|| handle.block_on(future))
270-
} else {
271-
exec_err!("No current Tokio runtime available")
364+
#[tokio::test]
365+
async fn buffered_tail_does_not_retain_input_allocation() -> Result<()> {
366+
let tmp_dir = tempdir()?;
367+
let store: Arc<dyn ObjectStore> =
368+
Arc::new(LocalFileSystem::new_with_prefix(tmp_dir.path())?);
369+
let location = Path::from("tail-retention-test");
370+
let upload = store.put_multipart(&location).await?;
371+
let dropped = Arc::new(AtomicBool::new(false));
372+
let data = Bytes::from_owner(DropTrackingOwner {
373+
data: vec![42; 128],
374+
dropped: Arc::clone(&dropped),
375+
});
376+
let mut writer = ObjectStoreSpillWriter {
377+
upload,
378+
buffer: PutPayloadMut::new(),
379+
size: Arc::new(AtomicU64::new(0)),
380+
bytes_written: 0,
381+
};
382+
383+
writer.write_all(data).await?;
384+
assert!(
385+
dropped.load(Ordering::Relaxed),
386+
"buffered tail should not retain the input allocation"
387+
);
388+
writer.abort().await?;
389+
Ok(())
272390
}
273391
}

‎datafusion/execution/src/lib.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,6 @@ pub mod registry {
4949
pub use async_stream::{Emitter, TryEmitter, async_stream, async_try_stream};
5050
pub use disk_manager::DiskManager;
5151
pub use registry::FunctionRegistry;
52-
pub use spill_file::{SpillFile, SpillWriter, TempFileFactory};
52+
pub use spill_file::{AsyncSpillWriter, SpillFile, SpillWriter, TempFileFactory};
5353
pub use stream::{RecordBatchStream, SendableRecordBatchStream};
5454
pub use task::{TaskContext, TaskContextProvider};

0 commit comments

Comments
 (0)