Skip to content

Commit 31ae92c

Browse files
fix(scan): keep capped JNI scans within Arrow lookahead
Drive capped Arrow streams directly instead of feeding the runtime result queue, which can retain one extra completed batch per core. Preserve the existing queue for uncapped scans. Add paused-consumer regressions with two background workers. Saturate per-worker concurrency scaling before applying the absolute cap and add an overflow regression. Clarify that split concurrency reduces decoded memory rather than bounding total native memory or I/O read-ahead. Signed-off-by: Christopher Wirt <[email protected]>
1 parent a7b6e71 commit 31ae92c

5 files changed

Lines changed: 105 additions & 25 deletions

File tree

‎docs/user-guide/spark.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -236,7 +236,7 @@ depends on, on the driver and executor classpath, for example
236236
| Option | Default | Description |
237237
|----------------------------|---------|-------------|
238238
| `vortex.readConcurrency` | `0` | Maximum Hadoop read upcalls per file; `0` uses the native default. |
239-
| `vortex.splitConcurrency` | `0` | Most row splits of a file decoded at once by one task; `0` uses the native default of four per core. Each split holds the decoded chunks of every column it reads, and every task in an executor runs its own scan, so lowering this bounds a task's native memory. |
239+
| `vortex.splitConcurrency` | `0` | Most row splits of a file decoded at once by one task; `0` uses the native default of four per core. Lowering this reduces memory held by concurrent split decoding and Arrow conversion. It does not cap total native memory or I/O read-ahead. |
240240
| `vortex.workerThreads` | `4` | Background threads driving Vortex futures. JVM-wide, and set once per JVM by the first read or write. |
241241
| `vortex.session.provider` | — | Fully-qualified name of a `VortexSessionProvider` supplying a custom session on the driver and on every executor. |
242242
| `vortex.aggregatePushdown` | `true` | Answer `COUNT(*)` from file footers. |

‎java/vortex-spark/common/src/main/java/dev/vortex/spark/io/VortexIo.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -33,8 +33,8 @@ public final class VortexIo implements Serializable {
3333
/**
3434
* Option bounding how many splits of a file the native scan reads at once, in all. Each split in flight holds the
3535
* decoded chunks of every column it overlaps, and every task in an executor runs its own scan, which by default
36-
* keeps several splits in flight per core, so setting this bounds a task's native memory. Zero keeps the native
37-
* default.
36+
* keeps several splits in flight per core. Lowering this reduces memory held by concurrent split decoding and Arrow
37+
* conversion; it does not cap total native memory or I/O read-ahead. Zero keeps the native default.
3838
*/
3939
public static final String SPLIT_CONCURRENCY_OPTION = "vortex.splitConcurrency";
4040

‎vortex-jni/src/scan.rs‎

Lines changed: 87 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ use arrow_array::ffi::FFI_ArrowSchema;
2222
use arrow_array::ffi_stream::FFI_ArrowArrayStream;
2323
use arrow_schema::ArrowError;
2424
use arrow_schema::Field;
25+
use futures::Stream;
2526
use futures::StreamExt;
2627
use jni::EnvUnowned;
2728
use jni::objects::JByteArray;
@@ -40,6 +41,7 @@ use vortex::expr::Expression;
4041
use vortex::expr::root;
4142
use vortex::expr::stats::Precision;
4243
use vortex::io::runtime::BlockingRuntime;
44+
use vortex::io::runtime::current::CurrentThreadRuntime;
4345
use vortex::layout::scan::arrow::RecordBatchIteratorAdapter;
4446
use vortex::scan::DataSourceScan;
4547
use vortex::scan::PartitionRef;
@@ -380,28 +382,27 @@ pub extern "system" fn Java_dev_vortex_jni_NativePartition_scanArrow(
380382
let schema = Arc::new(session.arrow().to_arrow_schema(&dtype)?);
381383
let target = Arc::new(Field::new_struct("", schema.fields().clone(), false));
382384

383-
let iter = RUNTIME
384-
.block_on_stream_thread_safe(|handle| {
385-
array_stream
386-
.map(move |chunk| {
387-
let session = session.clone();
388-
let target = Arc::clone(&target);
389-
handle.spawn(async move {
390-
let chunk = chunk?;
391-
let mut ctx = session.create_execution_ctx();
392-
let arrow = session.arrow().execute_arrow(
393-
chunk,
394-
Some(target.as_ref()),
395-
&mut ctx,
396-
)?;
397-
rebase_offsets(RecordBatch::from(arrow.as_struct().clone()))
398-
})
399-
})
400-
.buffered(lookahead)
385+
let handle = RUNTIME.handle();
386+
let batches = array_stream
387+
.map(move |chunk| {
388+
let session = session.clone();
389+
let target = Arc::clone(&target);
390+
handle.spawn(async move {
391+
let chunk = chunk?;
392+
let mut ctx = session.create_execution_ctx();
393+
let arrow =
394+
session
395+
.arrow()
396+
.execute_arrow(chunk, Some(target.as_ref()), &mut ctx)?;
397+
rebase_offsets(RecordBatch::from(arrow.as_struct().clone()))
398+
})
401399
})
402-
.map(|result: VortexResult<RecordBatch>| {
400+
.buffered(lookahead);
401+
let iter = scan_iterator(&RUNTIME, split_concurrency, batches).map(
402+
|result: VortexResult<RecordBatch>| {
403403
result.map_err(|e| ArrowError::ExternalError(Box::new(e)))
404-
});
404+
},
405+
);
405406

406407
let reader = RecordBatchIteratorAdapter::new(iter, schema);
407408
let arrow_stream = FFI_ArrowArrayStream::new(Box::new(reader));
@@ -411,3 +412,69 @@ pub extern "system" fn Java_dev_vortex_jni_NativePartition_scanArrow(
411412
Ok(())
412413
});
413414
}
415+
416+
fn scan_iterator<R: Send + 'static>(
417+
runtime: &CurrentThreadRuntime,
418+
split_concurrency: Option<NonZeroUsize>,
419+
stream: impl Stream<Item = R> + Send + 'static,
420+
) -> Box<dyn Iterator<Item = R> + Send> {
421+
if split_concurrency.is_some() {
422+
// The thread-safe bridge queues another batch per core, independently of the stream's
423+
// conversion lookahead. Drive capped scans directly so that queue cannot bypass the cap.
424+
Box::new(runtime.block_on_stream(stream))
425+
} else {
426+
Box::new(runtime.block_on_stream_thread_safe(|_| stream))
427+
}
428+
}
429+
430+
#[cfg(test)]
431+
mod tests {
432+
use std::num::NonZeroUsize;
433+
use std::sync::Arc;
434+
use std::sync::atomic::AtomicUsize;
435+
use std::sync::atomic::Ordering;
436+
437+
use futures::StreamExt;
438+
use futures::stream;
439+
use rstest::rstest;
440+
use vortex::io::runtime::BlockingRuntime;
441+
use vortex::io::runtime::current::CurrentThreadRuntime;
442+
443+
use super::scan_iterator;
444+
445+
#[rstest]
446+
#[case(1)]
447+
#[case(2)]
448+
fn split_concurrency_bounds_completed_batches_ahead_of_the_consumer(#[case] max: usize) {
449+
let runtime = CurrentThreadRuntime::new();
450+
let pool = runtime.new_pool();
451+
pool.set_workers(2);
452+
let completed = Arc::new(AtomicUsize::new(0));
453+
let counter = Arc::clone(&completed);
454+
let handle = runtime.handle();
455+
let batches = stream::iter(0..16)
456+
.map(move |value| {
457+
let counter = Arc::clone(&counter);
458+
handle.spawn(async move {
459+
counter.fetch_add(1, Ordering::SeqCst);
460+
value
461+
})
462+
})
463+
.buffered(max);
464+
let mut iter = scan_iterator(&runtime, NonZeroUsize::new(max), batches);
465+
assert_eq!(iter.next(), Some(0));
466+
467+
// Let background work finish while the consumer keeps its first batch open.
468+
runtime.block_on(async {
469+
for _ in 0..64 {
470+
runtime.handle().spawn(async {}).await;
471+
}
472+
});
473+
assert!(
474+
completed.load(Ordering::SeqCst) <= max + 1,
475+
"completed batches escaped the conversion lookahead bound"
476+
);
477+
478+
assert_eq!(iter.collect::<Vec<_>>(), (1..16).collect::<Vec<_>>());
479+
}
480+
}

‎vortex-layout/src/scan/repeated_scan.rs‎

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -394,7 +394,9 @@ pub(crate) struct SplitConcurrency {
394394

395395
impl SplitConcurrency {
396396
fn splits_in_flight(self) -> usize {
397-
let scaled = self.per_worker * get_available_parallelism().unwrap_or(1);
397+
let scaled = self
398+
.per_worker
399+
.saturating_mul(get_available_parallelism().unwrap_or(1));
398400
self.max.map_or(scaled, |max| scaled.min(max.get()))
399401
}
400402
}
@@ -423,6 +425,7 @@ fn intersect_ranges(left: Option<&Range<u64>>, right: Option<Range<u64>>) -> Opt
423425

424426
#[cfg(test)]
425427
mod tests {
428+
use std::num::NonZeroUsize;
426429
use std::sync::Arc;
427430
use std::task::Poll;
428431

@@ -435,10 +438,20 @@ mod tests {
435438
use vortex_error::VortexResult;
436439
use vortex_error::vortex_err;
437440

441+
use super::SplitConcurrency;
438442
use super::limit_rows;
439443
use super::stop_on_error;
440444
use crate::scan::limit::RowLimit;
441445

446+
#[test]
447+
fn split_cap_handles_oversized_per_worker_concurrency() {
448+
let concurrency = SplitConcurrency {
449+
per_worker: usize::MAX,
450+
max: NonZeroUsize::new(1),
451+
};
452+
assert_eq!(concurrency.splits_in_flight(), 1);
453+
}
454+
442455
#[rstest]
443456
#[case::whole_array(4)]
444457
#[case::trimmed_array(2)]

‎vortex-layout/src/scan/scan_builder.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -249,7 +249,7 @@ impl<A: 'static + Send> ScanBuilder<A> {
249249
///
250250
/// Each split in progress holds the decoded chunks of every column it overlaps, so a scan
251251
/// that runs alongside others, such as one task of several on a machine, can set this to
252-
/// bound its memory.
252+
/// reduce memory held by concurrent decoding. This does not cap total memory or I/O read-ahead.
253253
pub fn with_some_max_splits_in_flight(mut self, max: Option<NonZeroUsize>) -> Self {
254254
self.concurrency.max = max;
255255
self

0 commit comments

Comments
 (0)