Skip to content

Commit 33c2702

Browse files
gatesncodex
andcommitted
Add an engine-independent Scan API driver
Share bounded partition execution, ordering, limits and cancellation between direct array consumers and the DataFusion v2 source. Co-authored-by: Codex <[email protected]> Signed-off-by: Nicholas Gates <[email protected]>
1 parent ba71c3b commit 33c2702

5 files changed

Lines changed: 479 additions & 62 deletions

File tree

‎vortex-datafusion/src/v2/source.rs‎

Lines changed: 39 additions & 62 deletions
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,7 @@
7070
use std::any::Any;
7171
use std::fmt;
7272
use std::fmt::Formatter;
73+
use std::num::NonZeroUsize;
7374
use std::pin::Pin;
7475
use std::sync::Arc;
7576
use std::sync::atomic::AtomicBool;
@@ -106,7 +107,6 @@ use datafusion_physical_expr_common::sort_expr::PhysicalSortExpr;
106107
use datafusion_physical_plan::DisplayFormatType;
107108
use datafusion_physical_plan::filter_pushdown::FilterPushdownPropagation;
108109
use datafusion_physical_plan::filter_pushdown::PushedDown;
109-
use datafusion_physical_plan::limit::LimitStream;
110110
use datafusion_physical_plan::metrics::BaselineMetrics;
111111
use datafusion_physical_plan::metrics::ExecutionPlanMetricsSet;
112112
use datafusion_physical_plan::stream::RecordBatchStreamAdapter;
@@ -116,15 +116,20 @@ use futures::TryStreamExt;
116116
use futures::future::try_join_all;
117117
use futures::lock::Mutex;
118118
use futures::stream;
119+
use futures::stream::BoxStream;
119120
use futures::task::AtomicWaker;
120121
use tokio::sync::OnceCell;
122+
use vortex::array::ArrayRef;
121123
use vortex::array::VortexSessionExecute;
124+
use vortex::array::stream::SendableArrayStream;
122125
use vortex::dtype::DType;
123126
use vortex::dtype::FieldPath;
124127
use vortex::dtype::Nullability;
125128
use vortex::error::VortexError;
129+
use vortex::error::VortexExpect as _;
126130
use vortex::error::VortexResult;
127131
use vortex::error::vortex_bail;
132+
use vortex::error::vortex_err;
128133
use vortex::expr::Expression;
129134
use vortex::expr::and_collect;
130135
use vortex::expr::get_item;
@@ -134,8 +139,8 @@ use vortex::expr::stats::Precision;
134139
use vortex::expr::transform::replace;
135140
use vortex::io::session::RuntimeSessionExt;
136141
use vortex::scan::DataSourceRef;
137-
use vortex::scan::PartitionStream;
138142
use vortex::scan::ScanRequest;
143+
use vortex::scan::driver::ScanDriver;
139144
use vortex::session::VortexSession;
140145
use vortex_arrow::ArrowSessionExt;
141146
use vortex_utils::parallelism::get_available_parallelism;
@@ -493,38 +498,39 @@ impl VortexDataSource {
493498
..Default::default()
494499
};
495500
let data_source = Arc::clone(&self.data_source);
496-
let partitions = stream::try_unfold(Arc::clone(&execution), move |execution| {
497-
let data_source = Arc::clone(&data_source);
498-
let request = scan_request.clone();
501+
let output_count = NonZeroUsize::new(self.output_partition_count())
502+
.vortex_expect("positive scan output count");
503+
let arrays = stream::once({
504+
let execution = Arc::clone(&execution);
499505
async move {
500506
// Cancellation may retry initialization, but never runs attempts concurrently.
501-
let partitions = execution
502-
.scan_partitions
507+
let streams = execution
508+
.scan_streams
503509
.get_or_init(|| async {
504-
data_source
505-
.scan(request)
506-
.await
507-
.map(|scan| Mutex::new(Some(scan.partitions())))
510+
let scan = data_source.scan(scan_request.clone()).await?;
511+
Ok(Mutex::new(
512+
ScanDriver::new(scan, &scan_request)
513+
.into_streams(
514+
output_count,
515+
NonZeroUsize::new(SCAN_PARTITION_READ_AHEAD)
516+
.vortex_expect("positive scan read-ahead"),
517+
)
518+
.into_iter()
519+
.map(Some)
520+
.collect(),
521+
))
508522
})
509523
.await
510524
.as_ref()
511525
.map_err(Clone::clone)?;
512-
let partition = {
513-
let mut partitions = partitions.lock().await;
514-
let next = match partitions.as_mut() {
515-
Some(partitions) => partitions.try_next().await?,
516-
None => None,
517-
};
518-
if next.is_none() {
519-
drop(partitions.take());
520-
}
521-
next
522-
};
523-
Ok(partition.map(|partition| (partition, execution)))
526+
streams.lock().await[partition]
527+
.take()
528+
.ok_or_else(|| vortex_err!("scan output partition already opened"))
524529
}
525530
})
531+
.try_flatten()
526532
.boxed();
527-
self.batch_stream(partitions, partition, metrics.clone())?
533+
self.batch_stream(arrays, metrics.clone())?
528534
};
529535
Ok(Box::pin(OutputPartitionStream {
530536
partition,
@@ -537,8 +543,7 @@ impl VortexDataSource {
537543

538544
fn batch_stream(
539545
&self,
540-
scan_partitions: PartitionStream,
541-
partition: usize,
546+
arrays: BoxStream<'static, VortexResult<ArrayRef>>,
542547
metrics: BaselineMetrics,
543548
) -> DFResult<SendableRecordBatchStream> {
544549
let projected_schema = Arc::clone(&self.projected_schema);
@@ -553,8 +558,6 @@ impl VortexDataSource {
553558
} else {
554559
1
555560
};
556-
let ordered = self.ordered;
557-
let limit = self.limit;
558561

559562
// Pre-build the leftover projector (if any) so we can apply it after batch conversion.
560563
let leftover_projector = self
@@ -565,22 +568,8 @@ impl VortexDataSource {
565568

566569
// Defer the async DataSource::scan() call to the first poll of the stream.
567570
let stream = stream::once(async move {
568-
let scan_streams = scan_partitions.map(|split| split?.execute());
569-
570-
// When ordering is requested we must emit partition output in partition order, so we
571-
// flatten the split streams sequentially. Otherwise we use try_flatten_unordered to
572-
// poll multiple split streams concurrently so that the next split is already warm when
573-
// the current one finishes.
574-
let flattened = if ordered {
575-
scan_streams.try_flatten().left_stream()
576-
} else {
577-
scan_streams
578-
.try_flatten_unordered(Some(SCAN_PARTITION_READ_AHEAD))
579-
.right_stream()
580-
};
581-
582571
let handle = session.handle();
583-
let stream = flattened
572+
let stream = arrays
584573
.map(move |result| {
585574
let session = session.clone();
586575
let target_field = Arc::clone(&projected_target_field);
@@ -601,20 +590,6 @@ impl VortexDataSource {
601590
.buffered(conversion_concurrency)
602591
.map(|result| result.map_err(|e| DataFusionError::External(Box::new(e))));
603592

604-
// DataFusion drops its own limit operator once `with_fetch` accepts the limit, so the
605-
// fetch must be exact here. Unordered scans share the limit across partitions, but
606-
// ordered partitions each apply it locally and rely on this trim.
607-
let stream = match limit {
608-
Some(limit) => LimitStream::new(
609-
Box::pin(RecordBatchStreamAdapter::new(projected_schema, stream)),
610-
0,
611-
Some(limit),
612-
BaselineMetrics::new(&ExecutionPlanMetricsSet::new(), partition),
613-
)
614-
.boxed(),
615-
None => stream.boxed(),
616-
};
617-
618593
// Apply leftover projection (expressions that couldn't be pushed into Vortex).
619594
let stream = if let Some(projector) = leftover_projector {
620595
stream
@@ -646,7 +621,7 @@ struct OutputPartitionState {
646621
struct ScanExecution {
647622
output_partitions: Vec<OutputPartitionState>,
648623
cancelled: AtomicBool,
649-
scan_partitions: OnceCell<VortexResult<Mutex<Option<PartitionStream>>>>,
624+
scan_streams: OnceCell<VortexResult<Mutex<Vec<Option<SendableArrayStream>>>>>,
650625
}
651626

652627
impl ScanExecution {
@@ -656,7 +631,7 @@ impl ScanExecution {
656631
.map(|_| OutputPartitionState::default())
657632
.collect(),
658633
cancelled: AtomicBool::new(false),
659-
scan_partitions: OnceCell::new(),
634+
scan_streams: OnceCell::new(),
660635
}
661636
}
662637

@@ -732,10 +707,12 @@ impl Drop for OutputPartitionStream {
732707
.take(),
733708
);
734709
if self.execution.cancelled.load(Ordering::Acquire)
735-
&& let Some(Ok(partitions)) = self.execution.scan_partitions.get()
736-
&& let Some(mut partitions) = partitions.try_lock()
710+
&& let Some(Ok(streams)) = self.execution.scan_streams.get()
711+
&& let Some(mut streams) = streams.try_lock()
737712
{
738-
drop(partitions.take());
713+
for stream in streams.iter_mut() {
714+
drop(stream.take());
715+
}
739716
}
740717
self.metrics.done();
741718
}

‎vortex-scan/Cargo.toml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ roaring = { workspace = true }
2929
tracing = { workspace = true }
3030

3131
[dev-dependencies]
32+
futures = { workspace = true, features = ["executor"] }
3233
parking_lot = { workspace = true }
3334
rstest = { workspace = true }
3435
vortex-array = { workspace = true, features = ["_test-harness"] }

0 commit comments

Comments
 (0)