Skip to content

Commit b9d7594

Browse files
perf(jni): bound completed scan outputs with background prefetch (#11198)
## Summary - Tracking Issue: #11197 A JNI scan hands batches to Java through `block_on_stream_thread_safe`. That bridge queues one completed batch per CPU core, but each JNI stream has only one consumer. On wide or nested Spark scans those queued Arrow batches add up, so peak memory grows with the host's core count. This adds a single-consumer bridge that keeps one completed batch ahead of Java, and uses it for JNI scans. **Result:** local Spark measurements, medians of macOS peak physical footprint: | Workload | Runs | Peak GiB, before → after | Change | | --- | ---: | ---: | ---: | | Flat-layout read | 3 | 1.46 → 1.05 | −28% | | List-layout read | 2 | 3.25 → 2.22 | −32% | | Ordered Vortex rewrite | 2 | 1.86 → 1.43 | −23% | - Batch sizes, compression and decode concurrency don't change. The rewrite's output was identical in size and row count. - Timing was too noisy on a shared host to claim a speedup or rule out a regression. Only one Spark task with four native workers was measured. ## Changes - **`vortex-io`:** `CurrentThreadRuntime::block_on_stream_prefetch`, returning a `PrefetchIterator`. - It drives the stream in a background task, so native work still overlaps Java consumption. - It keeps at most one completed item ahead of the consumer, and skips entering the executor when an item is already waiting. - Dropping it cancels the producer. A producer panic is re-raised at end of stream. - **JNI:** `NativePartition.scanArrow` uses it instead of `block_on_stream_thread_safe`. Other callers, including DuckDB, are unchanged. --------- Signed-off-by: Christopher Wirt <[email protected]> Co-authored-by: Claude Opus 5.5 <[email protected]>
1 parent 42e47a7 commit b9d7594

2 files changed

Lines changed: 114 additions & 21 deletions

File tree

‎vortex-io/src/runtime/current.rs‎

Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,8 +5,10 @@ use std::future::Future;
55
use std::sync::Arc;
66

77
use async_channel::TryRecvError;
8+
use futures::SinkExt;
89
use futures::Stream;
910
use futures::StreamExt;
11+
use futures::channel::mpsc;
1012
use futures::stream::BoxStream;
1113
use parking_lot::Mutex;
1214
use smol::block_on;
@@ -91,6 +93,35 @@ impl CurrentThreadRuntime {
9193
driver: Arc::new(Mutex::new(Some(driver))),
9294
}
9395
}
96+
97+
/// Returns an iterator that drives a stream in a background task, keeping at most one
98+
/// completed item ready ahead of the single consumer.
99+
///
100+
/// Use this instead of [`block_on_stream_thread_safe`](Self::block_on_stream_thread_safe)
101+
/// when items are large and a per-core output queue would retain too many of them.
102+
pub fn block_on_stream_prefetch<S, R>(&self, stream: S) -> PrefetchIterator<R>
103+
where
104+
S: Stream<Item = R> + Send + 'static,
105+
R: Send + 'static,
106+
{
107+
// A zero-capacity mpsc still gives the sender one slot, and `send` resolves only once the
108+
// receiver takes the item, so the driver never pulls a second item ahead.
109+
let (mut result_tx, result_rx) = mpsc::channel(0);
110+
let driver = self.executor.async_executor().spawn(async move {
111+
futures::pin_mut!(stream);
112+
while let Some(item) = stream.next().await {
113+
if result_tx.send(item).await.is_err() {
114+
break;
115+
}
116+
}
117+
});
118+
119+
PrefetchIterator {
120+
executor: Arc::clone(&self.executor),
121+
results: result_rx,
122+
driver: Some(driver),
123+
}
124+
}
94125
}
95126

96127
impl BlockingRuntime for CurrentThreadRuntime {
@@ -184,6 +215,32 @@ impl<T> ThreadSafeIterator<T> {
184215
}
185216
}
186217

218+
/// A single-consumer iterator that keeps at most one completed item ahead of the consumer.
219+
pub struct PrefetchIterator<T> {
220+
executor: Arc<SmolExecutor>,
221+
results: mpsc::Receiver<T>,
222+
/// Joined once the stream ends so a panic raised while driving it is re-raised.
223+
driver: Option<smol::Task<()>>,
224+
}
225+
226+
impl<T> Iterator for PrefetchIterator<T> {
227+
type Item = T;
228+
229+
fn next(&mut self) -> Option<Self::Item> {
230+
let item = match self.results.try_recv() {
231+
Ok(item) => Some(item),
232+
Err(mpsc::TryRecvError::Closed) => None,
233+
Err(mpsc::TryRecvError::Empty) => block_on(self.executor.run(self.results.next())),
234+
};
235+
if item.is_none()
236+
&& let Some(driver) = self.driver.take()
237+
{
238+
block_on(self.executor.run(driver));
239+
}
240+
item
241+
}
242+
}
243+
187244
#[cfg(test)]
188245
mod tests {
189246
use std::any::Any;
@@ -627,4 +684,37 @@ mod tests {
627684
assert_eq!(results.len(), num_items);
628685
assert_eq!(results, (0..num_items).collect::<Vec<_>>());
629686
}
687+
688+
#[test]
689+
fn test_block_on_stream_prefetch_keeps_one_item_ahead() {
690+
let runtime = CurrentThreadRuntime::new();
691+
let produced = Arc::new(AtomicUsize::new(0));
692+
let counter = Arc::clone(&produced);
693+
let items = stream::iter(0..16).inspect(move |_| {
694+
counter.fetch_add(1, Ordering::SeqCst);
695+
});
696+
let mut iter = runtime.block_on_stream_prefetch(items);
697+
assert_eq!(iter.next(), Some(0));
698+
runtime.block_on(async {
699+
for _ in 0..64 {
700+
runtime.handle().spawn(async {}).await;
701+
}
702+
});
703+
assert_eq!(produced.load(Ordering::SeqCst), 2);
704+
assert_eq!(iter.collect::<Vec<_>>(), (1..16).collect::<Vec<_>>());
705+
}
706+
707+
#[test]
708+
fn test_block_on_stream_prefetch_propagates_driver_panic() {
709+
let runtime = CurrentThreadRuntime::new();
710+
let items = stream::iter(0..2).map(|value| {
711+
assert!(value != 1, "stream driver panic");
712+
value
713+
});
714+
let mut iter = runtime.block_on_stream_prefetch(items);
715+
assert_eq!(iter.next(), Some(0));
716+
let panic = std::panic::catch_unwind(AssertUnwindSafe(|| iter.next()))
717+
.expect_err("stream panic must propagate through iterator");
718+
assert!(panic_message(panic.as_ref()).contains("stream driver panic"));
719+
}
630720
}

‎vortex-jni/src/scan.rs‎

Lines changed: 24 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -358,28 +358,31 @@ pub extern "system" fn Java_dev_vortex_jni_NativePartition_scanArrow(
358358
let schema = Arc::new(session.arrow().to_arrow_schema(&dtype)?);
359359
let target = Arc::new(Field::new_struct("", schema.fields().clone(), false));
360360

361-
let iter = RUNTIME
362-
.block_on_stream_thread_safe(|handle| {
363-
array_stream
364-
.map(move |chunk| {
365-
let session = session.clone();
366-
let target = Arc::clone(&target);
367-
handle.spawn(async move {
368-
let chunk = chunk?;
369-
let mut ctx = session.create_execution_ctx();
370-
let arrow = session.arrow().execute_arrow(
371-
chunk,
372-
Some(target.as_ref()),
373-
&mut ctx,
374-
)?;
375-
rebase_offsets(RecordBatch::from(arrow.as_struct().clone()))
376-
})
377-
})
378-
.buffered(POOL.worker_count().max(1))
361+
let handle = RUNTIME.handle();
362+
let batches = array_stream
363+
.map(move |chunk| {
364+
let session = session.clone();
365+
let target = Arc::clone(&target);
366+
handle.spawn(async move {
367+
let chunk = chunk?;
368+
let mut ctx = session.create_execution_ctx();
369+
let arrow =
370+
session
371+
.arrow()
372+
.execute_arrow(chunk, Some(target.as_ref()), &mut ctx)?;
373+
rebase_offsets(RecordBatch::from(arrow.as_struct().clone()))
374+
})
379375
})
380-
.map(|result: VortexResult<RecordBatch>| {
381-
result.map_err(|e| ArrowError::ExternalError(Box::new(e)))
382-
});
376+
.buffered(POOL.worker_count().max(1));
377+
378+
// Each JNI stream has a single consumer, so keep only one completed batch ahead of Java
379+
// rather than the per-core queue of `block_on_stream_thread_safe`.
380+
let iter =
381+
RUNTIME
382+
.block_on_stream_prefetch(batches)
383+
.map(|result: VortexResult<RecordBatch>| {
384+
result.map_err(|e| ArrowError::ExternalError(Box::new(e)))
385+
});
383386

384387
let reader = RecordBatchIteratorAdapter::new(iter, schema);
385388
let arrow_stream = FFI_ArrowArrayStream::new(Box::new(reader));

0 commit comments

Comments
 (0)