Skip to content

Commit fd3ce43

Browse files
committed
Use chunked reads for object store spill example
1 parent 0db29d8 commit fd3ce43

1 file changed

Lines changed: 11 additions & 4 deletions

File tree

‎datafusion-examples/examples/data_io/object_store_spill.rs‎

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@ use datafusion::execution::runtime_env::RuntimeEnvBuilder;
3838
use datafusion::execution::{SpillFile, SpillWriter, TempFileFactory};
3939
use datafusion::prelude::{SessionConfig, SessionContext};
4040
use datafusion_common::exec_err;
41-
use futures::{Stream, StreamExt, stream};
41+
use futures::{Stream, StreamExt, TryStreamExt, stream};
4242
use object_store::local::LocalFileSystem;
4343
use object_store::path::Path;
4444
use object_store::{ObjectStore, ObjectStoreExt, PutPayload};
@@ -184,9 +184,16 @@ impl SpillFile for ObjectStoreSpillFile {
184184

185185
// Note: we use `stream::once` to defer the ObjectStore read until
186186
// DataFusion polls the returned stream.
187-
Ok(Box::pin(stream::once(async move {
188-
Ok(store.get(&location).await?.bytes().await?)
189-
})))
187+
let stream = stream::once(async move {
188+
store
189+
.get(&location)
190+
.await
191+
.map(|result| result.into_stream())
192+
})
193+
.try_flatten()
194+
.map_err(Into::into);
195+
196+
Ok(Box::pin(stream))
190197
}
191198

192199
/// Open a synchronous writer for this spill file.

0 commit comments

Comments
 (0)