Skip to content

Commit 5b45ae4

Browse files
Compose and drive a V2 scan in DataFusion (#10708)
Third of a stack of three that lands the planner-driven V2 scan for DataFusion behind a switch, cut from #10694. Stacked on #10707 (the planner chain), itself on #10706. With this merged, `scan_v2` runs TPC-H end to end; the perf and diagnostics commits of #10694 follow as separate PRs on top. ## Measured on its own Against the V1 scan, locally (14 cores, interleaved A/B/B/A, 10 and 5 iterations a run) at the same code before the stack was cut (da8a7c8): | query | SF1 | SF10 | |---|---|---| | q1 | 1.074 | 1.024 | | q6 | 0.966 | 0.832 | | q10 | 1.025 | 0.882 | | q12 | 0.939 | 0.811 | | q13 | 1.640 | 1.125 | | q19 | 0.996 | 0.909 | | geomean | 1.085 | 0.924 | SF10 improves over #10707 (0.977 to 0.924) as the per-step task hops go; SF1 does not yet, and q13 at SF1 is far behind because every partition still plans the filter and projection itself (the plan-once and interning commits of #10694 fix that, taking the full branch to 0.963 at SF1). This PR's base is a branch, so CI's benchmark dispatch does not run on it. --------- Signed-off-by: Joe Isaacs <[email protected]>
1 parent 33675d9 commit 5b45ae4

18 files changed

Lines changed: 807 additions & 639 deletions

File tree

‎benchmarks/compress-bench/src/vortex.rs‎

Lines changed: 12 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -3,22 +3,25 @@
33

44
use std::io::Cursor;
55
use std::path::Path;
6-
use std::sync::Arc;
76
use std::time::Duration;
87
use std::time::Instant;
98

109
use anyhow::Result;
1110
use anyhow::bail;
11+
use arrow_schema::Field;
1212
use async_trait::async_trait;
1313
use bytes::Bytes;
1414
use futures::StreamExt;
1515
use futures::pin_mut;
1616
use vortex::array::IntoArray;
17+
use vortex::array::VortexSessionExecute;
1718
use vortex::dtype::FieldNames;
1819
use vortex::expr::root;
1920
use vortex::expr::select;
2021
use vortex::file::OpenOptionsSessionExt;
2122
use vortex::file::WriteOptionsSessionExt;
23+
use vortex::file::v2::ScanDriver;
24+
use vortex::io::session::RuntimeSessionExt;
2225
use vortex::layout::scan::v2::ScanBuilder;
2326
use vortex_arrow::ArrowSessionExt;
2427
use vortex_bench::Format;
@@ -96,13 +99,17 @@ async fn vortex_decompress_read(data: Bytes) -> Result<()> {
9699
.optimize_recursive()?;
97100
scan = scan.with_projection(projection);
98101
}
99-
let schema = Arc::new(SESSION.arrow().to_arrow_schema(scan.dtype())?);
102+
let schema = SESSION.arrow().to_arrow_schema(scan.dtype())?;
103+
let field = Field::new_struct("", schema.fields().clone(), false);
100104

101-
let stream = scan.into_record_batch_stream(schema)?;
105+
let stream = ScanDriver::new(SESSION.handle(), 4, true).run(scan);
102106
pin_mut!(stream);
103107

104-
while let Some(batch) = stream.next().await {
105-
let _batch = batch?;
108+
while let Some(chunk) = stream.next().await {
109+
let mut ctx = SESSION.create_execution_ctx();
110+
let _batch = SESSION
111+
.arrow()
112+
.execute_arrow(chunk?, Some(&field), &mut ctx)?;
106113
}
107114
Ok(())
108115
}

‎vortex-bench/src/random_access/take.rs‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,8 @@ use vortex::array::stream::ArrayStreamExt;
3535
use vortex::buffer::Buffer;
3636
use vortex::file::OpenOptionsSessionExt;
3737
use vortex::file::VortexFile;
38+
use vortex::file::v2::ScanDriver;
39+
use vortex::io::session::RuntimeSessionExt;
3840
use vortex::layout::scan::v2::ScanBuilder;
3941
use vortex::scan::strict_sorted_buffer::StrictSortedBuffer;
4042

@@ -190,9 +192,9 @@ impl RandomAccessor for VortexRandomAccessor {
190192

191193
async fn take(&self, indices: &[u64]) -> anyhow::Result<RandomAccessorRet> {
192194
let indices = StrictSortedBuffer::try_new(Buffer::from(indices.to_vec()))?;
193-
let stream = ScanBuilder::new(SESSION.clone(), self.file.scan_file())
194-
.with_row_indices(indices)
195-
.into_stream()?;
195+
let scan =
196+
ScanBuilder::new(SESSION.clone(), self.file.scan_file()).with_row_indices(indices);
197+
let stream = ScanDriver::new(SESSION.handle(), 4, true).run(scan);
196198
let array = ArrayStreamAdapter::new(self.file.dtype().clone(), stream)
197199
.read_all()
198200
.await?;

‎vortex-datafusion/src/persistent/mod.rs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ mod format;
2828
pub mod metrics;
2929
mod opener;
3030
pub mod reader;
31+
mod scan_v2;
3132
mod sink;
3233
mod source;
3334
mod stream;

‎vortex-datafusion/src/persistent/opener.rs‎

Lines changed: 37 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -53,15 +53,20 @@ use vortex::error::VortexExpect;
5353
use vortex::expr::and_collect;
5454
use vortex::file::OpenOptionsSessionExt;
5555
use vortex::io::InstrumentedReadAt;
56+
use vortex::io::session::RuntimeSessionExt;
5657
use vortex::layout::LayoutReader;
5758
use vortex::layout::scan::scan_builder::ScanBuilder;
59+
use vortex::layout::scan::v2::ChunkPlanner;
60+
use vortex::layout::scan::v2::PrunePlanner;
5861
use vortex::layout::scan::v2::ScanBuilder as V2ScanBuilder;
5962
use vortex::layout::scan::v2::ScanCache;
6063
use vortex::layout::scan::v2::WeakScanCache;
6164
use vortex::layout::scan::v2::partition_row_range;
6265
use vortex::layout::segments::SegmentSource;
66+
use vortex::mask::Mask;
6367
use vortex::metrics::Label;
6468
use vortex::metrics::MetricsRegistry;
69+
use vortex::scan::planning::planner::Planner;
6570
use vortex::session::VortexSession;
6671
use vortex_arrow::ArrowSessionExt;
6772
use vortex_utils::aliases::dash_map::DashMap;
@@ -77,6 +82,7 @@ use crate::metrics::PARTITION_LABEL;
7782
use crate::metrics::PATH_LABEL;
7883
use crate::persistent::cache::CachedVortexMetadata;
7984
use crate::persistent::reader::VortexReaderFactory;
85+
use crate::persistent::scan_v2::ScanDriver;
8086
use crate::persistent::stream::PrunableStream;
8187

8288
#[derive(Clone)]
@@ -116,7 +122,7 @@ pub(crate) struct VortexOpener {
116122
/// shared layout reader.
117123
pub segment_sources: Arc<DashMap<Path, Weak<dyn SegmentSource>>>,
118124
/// The V2 scan cache each open file's partitions share: its lowered layout, filter and
119-
/// projection plans, proved zones and dictionary values, built once per file.
125+
/// projection plans, pruned zones and dictionary values, built once per file.
120126
pub scan_caches: Arc<DashMap<Path, WeakScanCache>>,
121127
/// Shared V1 full-file natural splits keyed by file path.
122128
pub natural_splits: Arc<DashMap<Path, Arc<NaturalSplits>>>,
@@ -437,8 +443,7 @@ impl FileOpener for VortexOpener {
437443
scan_file.cache = Some(shared_scan_cache(&scan_caches, &file.object_meta.location));
438444
let mut builder = V2ScanBuilder::new(vxf.session().clone(), scan_file)
439445
.with_projection(scan_projection)
440-
.with_some_filter(native_filter)
441-
.with_ordered(has_output_ordering);
446+
.with_some_filter(native_filter);
442447
if let Some(bytes) = byte_range {
443448
// Every partition maps the same byte boundary to the same row boundary.
444449
let rows = partition_row_range(bytes, file.object_meta.size, vxf.row_count());
@@ -462,8 +467,35 @@ impl FileOpener for VortexOpener {
462467
.div_ceil(partitions)
463468
.max(1)
464469
});
465-
builder = builder.with_concurrency(concurrency);
466-
builder.map(map_batch).into_stream()
470+
let handle = vxf.session().handle();
471+
// Lowering and planning a wide file is real CPU: it runs where compute runs.
472+
let scan = handle
473+
.spawn_cpu(move || builder.prepare())
474+
.await
475+
.map_err(|e| exec_datafusion_err!("Failed to plan Vortex scan: {e}"))?;
476+
let scan = Arc::new(scan);
477+
478+
// The planners, composed: with a filter, its pruning plans first over the whole
479+
// partition, continuing into the chunks of the rows they keep; without a
480+
// filter, the chunks directly. A chunk plans its conjuncts, then its
481+
// projection.
482+
let chunks = {
483+
let scan = Arc::clone(&scan);
484+
move |kept: Option<Mask>| -> Box<dyn Planner> {
485+
let rows = scan.rows().clone();
486+
let selection = scan.selection().clone();
487+
Box::new(ChunkPlanner::new(scan, rows, selection, kept))
488+
}
489+
};
490+
let root: Box<dyn Planner> = if scan.filter().is_some() {
491+
let rows = scan.rows().clone();
492+
Box::new(PrunePlanner::new(Arc::clone(&scan), rows, Box::new(chunks)))
493+
} else {
494+
chunks(None)
495+
};
496+
497+
let driver = ScanDriver::new(handle, concurrency, has_output_ordering);
498+
Ok(driver.run(scan, root, map_batch))
467499
} else {
468500
// Share V1 layout readers across partitions of the file.
469501
let layout_reader = match layout_readers.entry(file.object_meta.location.clone()) {

0 commit comments

Comments
 (0)