Skip to content

Commit 5ff7af6

Browse files
adriangbclaude
andcommitted
POC: streaming (batch-granular) parquet scan via a persistent sync reader
Adds DF_FETCH_POLICY=streaming: instead of buffering whole row groups, one long-lived sync ParquetRecordBatchReader pulls bytes through a shared in-memory ChunkReader. The stream driver computes from the offset index exactly which page ranges the next batch needs, awaits their fetch, and keeps up to DF_FETCH_BUDGET bytes of background readahead in flight; page bytes are dropped as soon as the decode cursor passes them. Because the reader persists for the whole file, dictionary pages are fetched and decoded exactly once, there is no per-row-group reader rebuild, and resident memory is bounded by the readahead window rather than row-group size. Supports projections, RowSelections (page skipping preserved via selected-row prefix sums), limits, and multi-RG files; falls back to the push-decoder path when row filters are active or the offset index is unavailable. Local simulated-latency results (300MB single-row-group file, 50ms latency, 100MB window, streaming scan): time-to-first-batch 112ms vs ~700ms for all row-group-granular policies, total 708ms vs ~845ms, peak staged 100MB vs 283MB. Co-Authored-By: Claude Fable 5 <[email protected]>
1 parent d38a4ee commit 5ff7af6

2 files changed

Lines changed: 602 additions & 0 deletions

File tree

‎datafusion/datasource-parquet/src/opener/mod.rs‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1387,6 +1387,54 @@ impl RowGroupsPrunedParquetOpen {
13871387
prepared.virtual_state.as_deref(),
13881388
)?;
13891389

1390+
// EXPERIMENT: streaming (batch-granular) scan path, selected via
1391+
// DF_FETCH_POLICY=streaming. Bypasses the push decoder entirely: a
1392+
// long-lived sync reader pulls from a shared buffer that the stream
1393+
// driver fills with exactly the page ranges each batch needs (plus
1394+
// bounded readahead). Falls back to the push-decoder path when row
1395+
// filters are active or the offset index is unavailable.
1396+
if let FetchPolicy::Streaming { window } = FetchPolicy::from_env() {
1397+
let pushdown_active =
1398+
prepared.pushdown_filters && prepared.predicate.is_some();
1399+
if !pushdown_active {
1400+
let streaming_access_plan = prepare_access_plan(access_plan.clone())?;
1401+
if let Some(streaming_plan) = crate::push_decoder::build_streaming_plan(
1402+
&file_metadata,
1403+
&streaming_access_plan.row_group_indexes,
1404+
decoder_projection.projection_mask(),
1405+
streaming_access_plan.row_selection.as_ref(),
1406+
) {
1407+
let stream = crate::push_decoder::build_streaming_stream(
1408+
streaming_plan,
1409+
crate::push_decoder::StreamingScanConfig {
1410+
reader_metadata: reader_metadata.clone(),
1411+
row_group_indexes: streaming_access_plan.row_group_indexes,
1412+
row_selection: streaming_access_plan.row_selection,
1413+
decoder_projection,
1414+
batch_size: prepared.batch_size,
1415+
limit: prepared.limit,
1416+
reader: prepared.async_file_reader,
1417+
baseline_metrics: prepared.baseline_metrics,
1418+
window,
1419+
},
1420+
)?;
1421+
let files_ranges_pruned_statistics =
1422+
prepared.file_metrics.files_ranges_pruned_statistics.clone();
1423+
return match prepared.file_pruner {
1424+
Some(file_pruner) if file_pruner.is_watching() => {
1425+
Ok(EarlyStoppingStream::new(
1426+
stream,
1427+
file_pruner,
1428+
files_ranges_pruned_statistics,
1429+
)
1430+
.boxed())
1431+
}
1432+
_ => Ok(stream),
1433+
};
1434+
}
1435+
}
1436+
}
1437+
13901438
let (decoder, rg_plan) = {
13911439
let pushdown_predicate = prepared
13921440
.pushdown_filters

0 commit comments

Comments
 (0)