Skip to content

Commit 22871ce

Browse files
committed
feat(file): a coalescing IO service for planning-protocol scans
Serves the planning protocol's `IoService` reads of a file: - `FileIoService` coalesces the ranges a batch of requests or announcements asks for, reads each range once for every session interested in it, and withdraws a dropped session's interest. `FileScanIo` serves the same protocol through the file's segment source and read driver. - Local files reuse their handles and admit native reads up to a bound; object-store reads stream completions as they land. - The read driver keeps its coalescing scratch between passes and can leave optional requests out of coalescing. - Segment caches can answer synchronously from `get_if_ready`. - Request tracing ids for diagnostics. A file opened over a reader carries the service as `VortexFile::scan_io`. Nothing drives it yet; the planning scan that does follows separately. Co-Authored-By: Claude Opus 5.5 <[email protected]> Claude-Session: https://claude.ai/code/session_01P1oQtzTgSzkvrGDKisoNko Signed-off-by: Claude <[email protected]>
1 parent ce04bf9 commit 22871ce

17 files changed

Lines changed: 3301 additions & 76 deletions

File tree

‎Cargo.lock‎

Lines changed: 1 addition & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎vortex-file/src/file.rs‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ use vortex_array::dtype::FieldMask;
1717
use vortex_array::expr::Expression;
1818
use vortex_buffer::ByteBuffer;
1919
use vortex_error::VortexResult;
20+
use vortex_io::request::IoService;
2021
use vortex_layout::LayoutReader;
2122
use vortex_layout::scan::layout::LayoutReaderDataSource;
2223
use vortex_layout::scan::scan_builder::ScanBuilder;
@@ -48,6 +49,8 @@ pub struct VortexFile {
4849
metadata: Arc<HashMap<String, ByteBuffer>>,
4950
/// None id LayoutReader caching is turned off
5051
layout_reader_cache: Option<OnceLock<Arc<dyn LayoutReader>>>,
52+
/// Serves the planning protocol's reads, when the file was opened over a reader.
53+
scan_io: Option<Arc<dyn IoService>>,
5154
}
5255

5356
fn layout_reader(
@@ -84,9 +87,21 @@ impl VortexFile {
8487
session,
8588
metadata: Arc::new(HashMap::new()),
8689
layout_reader_cache: None,
90+
scan_io: None,
8791
}
8892
}
8993

94+
pub(crate) fn with_scan_io(mut self, scan_io: Arc<dyn IoService>) -> Self {
95+
self.scan_io = Some(scan_io);
96+
self
97+
}
98+
99+
/// The service that serves the planning protocol's reads of this file, when the file was
100+
/// opened over a reader.
101+
pub fn scan_io(&self) -> Option<&Arc<dyn IoService>> {
102+
self.scan_io.as_ref()
103+
}
104+
90105
pub(crate) fn with_metadata(mut self, metadata: Arc<HashMap<String, ByteBuffer>>) -> Self {
91106
self.metadata = metadata;
92107
self
@@ -103,6 +118,7 @@ impl VortexFile {
103118
session: self.session,
104119
metadata: self.metadata,
105120
layout_reader_cache: Some(OnceLock::new()),
121+
scan_io: self.scan_io,
106122
}
107123
}
108124

@@ -158,6 +174,8 @@ impl VortexFile {
158174
/// replacement source.
159175
pub fn with_segment_source(mut self, segment_source: Arc<dyn SegmentSource>) -> Self {
160176
self.segment_source = segment_source;
177+
// Reads now go to the replacement, which the old read driver knows nothing of.
178+
self.scan_io = None;
161179
if self.layout_reader_cache.is_some() {
162180
self.layout_reader_cache = Some(OnceLock::new());
163181
}

‎vortex-file/src/open.rs‎

Lines changed: 25 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ use vortex_error::VortexExpect;
1414
use vortex_error::VortexResult;
1515
use vortex_error::vortex_err;
1616
use vortex_io::VortexReadAt;
17+
use vortex_io::request::IoService;
1718
use vortex_io::session::RuntimeSessionExt;
1819
use vortex_layout::segments::InstrumentedSegmentCache;
1920
use vortex_layout::segments::NoOpSegmentCache;
@@ -35,6 +36,7 @@ use crate::MAX_POSTSCRIPT_SIZE;
3536
use crate::VortexFile;
3637
use crate::footer::Footer;
3738
use crate::segments::BufferSegmentSource;
39+
use crate::segments::FileIoService;
3840
use crate::segments::FileSegmentSource;
3941
use crate::segments::InitialReadSegmentCache;
4042
use crate::segments::RequestMetrics;
@@ -281,6 +283,7 @@ impl VortexOpenOptions {
281283
///
282284
/// This is the common path for files, object stores, and custom random-access sources.
283285
pub async fn open_read<R: VortexReadAt + Clone>(self, reader: R) -> VortexResult<VortexFile> {
286+
let has_segment_cache = self.segment_cache.is_some();
284287
let segment_cache = self
285288
.segment_cache
286289
.clone()
@@ -327,13 +330,26 @@ impl VortexOpenOptions {
327330

328331
let metrics = RequestMetrics::new(metrics_registry.as_ref(), self.labels);
329332

333+
// Scans that drive the planning protocol read through their own service when it is
334+
// selected, and through the segment source's read driver otherwise.
335+
let batch_io = FileIoService::enabled().then(|| {
336+
Arc::new(FileIoService::open(
337+
footer.segment_specs_with_metadata(),
338+
reader.clone(),
339+
self.session.handle(),
340+
metrics.clone(),
341+
)) as Arc<dyn IoService>
342+
});
343+
330344
// Create a segment source backed by the VortexRead implementation.
331-
let segment_source = Arc::new(SharedSegmentSource::new(FileSegmentSource::open(
345+
let file_source = FileSegmentSource::open(
332346
footer.segment_specs_with_metadata(),
333347
reader,
334348
self.session.handle(),
335349
metrics,
336-
)));
350+
);
351+
let scan_io = batch_io.unwrap_or_else(|| Arc::new(file_source.scan_io()));
352+
let segment_source = Arc::new(SharedSegmentSource::new(file_source));
337353

338354
// Wrap up the segment source to first resolve segments from the initial read cache.
339355
let segment_source: Arc<dyn SegmentSource> = Arc::new(SegmentCacheSourceAdapter::new(
@@ -348,6 +364,13 @@ impl VortexOpenOptions {
348364
};
349365
let file =
350366
VortexFile::new(footer, segment_source, self.session.clone()).with_metadata(metadata);
367+
// The direct range service bypasses SegmentSource and its cache. Let V2 use the
368+
// segment adapter when a caller explicitly configured a cache.
369+
let file = if has_segment_cache {
370+
file
371+
} else {
372+
file.with_scan_io(scan_io)
373+
};
351374
Ok(if self.cache_layout_reader {
352375
file.with_caching()
353376
} else {

0 commit comments

Comments
 (0)