Skip to content

Commit 5852636

Browse files
Give each inline V2 scan root its own IO session (#11226)
When the DataFusion V2 scan driver runs chunks inline (`concurrency × available parallelism <= 1`), it admits each spawned chunk to the scan's own `Run` as another root. Every root was given the same `IoSource` session. The run clears a root's session when that root retires, so once the first chunk finished, its `clear()` cancelled the reads of the others, and they waited forever for completions that never came. - `vortex-datafusion/src/persistent/scan_v2.rs`: `RunState` holds the scan's `IoService` instead of a single session, and the driver calls `io.session()` for every root it admits: the first root and each inline child. Each root's session is now cleared on its own. All sessions still read from the same `SegmentSource`. --------- Signed-off-by: Joe Isaacs <[email protected]>
1 parent e5f7534 commit 5852636

1 file changed

Lines changed: 68 additions & 0 deletions

File tree

  • vortex-datafusion/src/persistent

‎vortex-datafusion/src/persistent/tests.rs‎

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -724,6 +724,74 @@ async fn test_repartitioned_scan_matches_non_repartitioned_for_uneven_splits(
724724
Ok(())
725725
}
726726

727+
/// With a scan concurrency of 0, a V2 scan runs every chunk on its own run as another root. Each
728+
/// root needs an IO session of its own: the run clears a root's session when the root retires, so
729+
/// a shared session left the remaining chunks waiting for reads that were never issued.
730+
#[tokio::test]
731+
async fn v2_inline_chunks_read_through_their_own_sessions() -> anyhow::Result<()> {
732+
let store = Arc::new(InMemory::new()) as Arc<dyn ObjectStore>;
733+
let session = VortexSession::default();
734+
let path = object_store::path::Path::parse("/inline-chunks.vortex")?;
735+
736+
let chunk_len = 1_000_i32;
737+
let chunk_count = 4;
738+
let chunks = (0..chunk_count).map(|chunk| {
739+
let start = chunk * chunk_len;
740+
StructArray::try_new(
741+
["value"].into(),
742+
vec![Buffer::from_iter(start..start + chunk_len).into_array()],
743+
usize::try_from(chunk_len).expect("chunk length is positive"),
744+
Validity::NonNullable,
745+
)
746+
.expect("chunk is a valid struct")
747+
.into_array()
748+
});
749+
let table = ChunkedArray::from_iter(chunks).into_array();
750+
let strategy: Arc<dyn LayoutStrategy> = Arc::new(TableStrategy::new(
751+
Arc::new(FlatLayoutStrategy::default()),
752+
Arc::new(ChunkedLayoutStrategy::new(FlatLayoutStrategy::default())),
753+
));
754+
let mut writer = ObjectStoreWrite::new(Arc::clone(&store), &path).await?;
755+
session
756+
.write_options()
757+
.with_strategy(strategy)
758+
.write(&mut writer, table.to_array_stream())
759+
.await?;
760+
writer.shutdown().await?;
761+
762+
let factory = Arc::new(
763+
VortexFormatFactory::new().with_options(crate::VortexTableOptions {
764+
scan_v2: true,
765+
scan_concurrency: Some(0),
766+
..Default::default()
767+
}),
768+
);
769+
let mut state = SessionStateBuilder::new()
770+
.with_config(SessionConfig::new().with_target_partitions(1))
771+
.with_default_features()
772+
.with_object_store(&url::Url::try_from("file://")?, store);
773+
if let Some(file_formats) = state.file_formats() {
774+
file_formats.push(factory as _);
775+
}
776+
let ctx = SessionContext::new_with_state(state.build()).enable_url_table();
777+
778+
let batches = tokio::time::timeout(std::time::Duration::from_secs(60), async {
779+
ctx.sql("SELECT value FROM '/inline-chunks.vortex' ORDER BY value")
780+
.await?
781+
.collect()
782+
.await
783+
})
784+
.await
785+
.map_err(|_| anyhow!("inline V2 scan did not finish"))??;
786+
787+
assert_eq!(
788+
batch_values(&batches),
789+
(0..chunk_count * chunk_len).collect::<Vec<_>>()
790+
);
791+
792+
Ok(())
793+
}
794+
727795
/// Roundtrip an `arrow.uuid` extension column through a Vortex file: write the column directly
728796
/// via the session-aware Arrow→Vortex conversion, then `SELECT *` and assert both the field
729797
/// metadata and the underlying values survive the trip.

0 commit comments

Comments
 (0)