Skip to content

Commit 5c6a473

Browse files
authored
perf: Group record batch fetches of remote IPC scans (#29750)
1 parent c630040 commit 5c6a473

3 files changed

Lines changed: 320 additions & 63 deletions

File tree

‎crates/polars-stream/src/nodes/io_sources/ipc/record_batch_data_fetch.rs‎

Lines changed: 207 additions & 56 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,9 @@
11
use std::io::Cursor;
2+
use std::num::NonZeroUsize;
23
use std::ops::Range;
34
use std::sync::Arc;
45

6+
use futures::FutureExt;
57
use polars_async::primitives::wait_group::WaitToken;
68
use polars_buffer::Buffer;
79
use polars_config::config;
@@ -28,6 +30,37 @@ pub(super) struct RecordBatchData {
2830
pub(super) slice: Option<(usize, usize)>,
2931
}
3032

33+
/// Largest gap between record batches that are fetched together.
34+
const MAX_GROUP_GAP: usize = 4096;
35+
36+
/// A group takes at most 1 / `GROUP_BUDGET_SHARE` of the pipeline budget, so it can always be
37+
/// filled.
38+
const GROUP_BUDGET_SHARE: usize = 4;
39+
40+
/// Default largest span of a group. Larger requests leave too few in flight under the current
41+
/// in-flight budget.
42+
const DEFAULT_GROUP_MAX_BYTES: usize = 2 << 20;
43+
44+
/// A record batch waiting for its group to be fetched.
45+
struct PendingRecordBatch {
46+
record_batch_idx: usize,
47+
num_rows: Option<IdxSize>,
48+
row_offset: Option<IdxSize>,
49+
slice: Option<(usize, usize)>,
50+
range: Range<usize>,
51+
fetch_permit: Option<PipelinePermit>,
52+
}
53+
54+
/// Where the record batches of a group get their bytes from.
55+
#[derive(Clone)]
56+
enum GroupBytes<F> {
57+
/// Memory-mapped or in-memory file, sliced per record batch.
58+
Memory(Buffer<u8>),
59+
/// One `get_ranges` call for the group, shared by its record batches and dropped, cancelling
60+
/// the fetch, with the last of them.
61+
Fetched(F),
62+
}
63+
3164
pub(super) struct RecordBatchDataFetcher {
3265
pub(super) file_metadata: Arc<FileMetadata>,
3366
pub(super) record_batch_cum_len: Option<Buffer<IdxSize>>,
@@ -76,6 +109,26 @@ impl RecordBatchDataFetcher {
76109
);
77110
let mut rb_fetch_count: u64 = 0;
78111

112+
// Consecutive record batches are fetched together, like Parquet row groups. Groups depend
113+
// only on the file and these limits, not on load.
114+
// Only remote sources have a chunk size, so only they group.
115+
// TODO: Revisit the group size once the in-flight budget is sized for large requests.
116+
let max_group_span = byte_source.chunk_size().map_or(0, |chunk_size| {
117+
let default = chunk_size.min(DEFAULT_GROUP_MAX_BYTES);
118+
std::env::var("POLARS_RECORD_BATCH_GROUP_MAX_BYTES").map_or(default, |x| {
119+
x.parse::<NonZeroUsize>()
120+
.unwrap_or_else(|_| {
121+
panic!("invalid value for POLARS_RECORD_BATCH_GROUP_MAX_BYTES: {x}")
122+
})
123+
.get()
124+
})
125+
});
126+
let max_group_len = (pipeline_budget.count_limit() / GROUP_BUDGET_SHARE).max(1);
127+
let max_group_kbytes = pipeline_budget.kbytes_limit() / GROUP_BUDGET_SHARE;
128+
let mut group: Vec<PendingRecordBatch> = Vec::new();
129+
let mut group_kbytes: usize = 0;
130+
let mut group_fetch_count: u64 = 0;
131+
79132
for record_batch_idx in 0..file_metadata.blocks.len() {
80133
let mut num_rows_this_rb: Option<IdxSize> = None;
81134
let mut row_offset: Option<IdxSize> = None;
@@ -136,66 +189,62 @@ impl RecordBatchDataFetcher {
136189
..usize::checked_add(block.offset as _, fetch_length)
137190
.ok_or_else(|| polars_err!(ComputeError: "IPC block range overflows usize"))?;
138191

139-
let fetch_permit = match rb_fetch {
140-
RbFetch::All | RbFetch::Metadata => {
141-
Some(pipeline_budget.acquire(fetch_length).await)
142-
},
143-
RbFetch::None => None,
144-
};
145-
146-
let current_byte_source = byte_source.clone();
147-
148-
let fetch_handle = ASYNC.spawn(async move {
149-
let fetched_bytes = if range.is_empty() {
150-
Buffer::new()
151-
} else if let DynByteSource::Buffer(mem_slice) = current_byte_source.as_ref() {
152-
let slice = mem_slice.0.as_ref();
153-
154-
if !std::ptr::eq(
155-
memory_prefetch_func as *const (),
156-
polars_utils::mem::prefetch::no_prefetch as *const (),
157-
) {
158-
debug_assert!(range.end <= slice.len());
159-
memory_prefetch_func(unsafe { slice.get_unchecked(range.clone()) })
160-
}
161-
162-
mem_slice.0.clone().sliced(range)
163-
} else {
164-
// @NOTE. Performance can be optimized by grouping requests and downloading
165-
// through `get_ranges()`.
166-
current_byte_source.get_range(range).await?
192+
let permit_kbytes = pipeline_budget.permit_kbytes(fetch_length);
193+
194+
// Start a new group at a gap or out-of-order block, so `get_ranges` does not fetch the
195+
// bytes in between, when it would span more than one request, or when it would take
196+
// more than its share of the budget.
197+
let starts_new_group = group
198+
.first()
199+
.zip(group.last())
200+
.is_some_and(|(first, last)| {
201+
range
202+
.start
203+
.checked_sub(last.range.end)
204+
.is_none_or(|gap| gap > MAX_GROUP_GAP)
205+
|| range.end - first.range.start > max_group_span
206+
|| group.len() >= max_group_len
207+
|| group_kbytes + permit_kbytes > max_group_kbytes
208+
});
209+
210+
if starts_new_group {
211+
group_kbytes = 0;
212+
let Some(n) = spawn_group(
213+
std::mem::take(&mut group),
214+
&byte_source,
215+
memory_prefetch_func,
216+
&prefetch_send,
217+
)
218+
.await
219+
else {
220+
break;
167221
};
222+
group_fetch_count += n;
223+
}
168224

169-
// Extract the length (i.e., nr of rows) at the earliest possible opportunity.
170-
let num_rows = if let Some(num_rows_this_rb) = num_rows_this_rb {
171-
num_rows_this_rb
172-
} else {
173-
let mut reader = BlockReader::new(Cursor::new(fetched_bytes.as_ref()));
174-
let mut message_scratch = vec![];
175-
reader
176-
.record_batch_num_rows(&mut message_scratch)?
177-
.try_into()
178-
.map_err(|_| polars_err!(ComputeError: LENGTH_LIMIT_MSG))?
179-
};
225+
// Load delays a group but never splits it: it takes at most a share of the budget, and
226+
// all other permits are held by earlier record batches, which are always consumed.
227+
let fetch_permit = if rb_fetch == RbFetch::None {
228+
None
229+
} else {
230+
Some(pipeline_budget.acquire(fetch_length).await)
231+
};
180232

181-
PolarsResult::Ok(RecordBatchData {
182-
fetched_bytes,
183-
record_batch_idx,
184-
num_rows,
185-
row_offset,
186-
slice,
187-
})
233+
group_kbytes += permit_kbytes;
234+
group.push(PendingRecordBatch {
235+
record_batch_idx,
236+
num_rows: num_rows_this_rb,
237+
row_offset,
238+
slice,
239+
range,
240+
fetch_permit,
188241
});
242+
}
189243

190-
let fetch_handle = tokio_handle_ext::AbortOnDropHandle(fetch_handle);
191-
192-
if prefetch_send
193-
.send((fetch_handle, fetch_permit))
194-
.await
195-
.is_err()
196-
{
197-
break;
198-
}
244+
if let Some(n) =
245+
spawn_group(group, &byte_source, memory_prefetch_func, &prefetch_send).await
246+
{
247+
group_fetch_count += n;
199248
}
200249

201250
drop(rb_prefetch_current_all_spawned);
@@ -209,10 +258,112 @@ impl RecordBatchDataFetcher {
209258
"[IpcFileReader]: RecordBatchDataFetcher: \
210259
rb_total_count: {rb_total_count}, \
211260
rb_full_fetch_count: {rb_full_fetch_count}, \
212-
rb_metadata_fetch_count: {rb_metadata_fetch_count}"
261+
rb_metadata_fetch_count: {rb_metadata_fetch_count}, \
262+
group_fetch_count: {group_fetch_count}"
213263
)
214264
}
215265

216266
Ok(())
217267
}
218268
}
269+
270+
/// Spawns the fetch of a group of record batches and sends a handle per record batch in order.
271+
/// Returns the number of `get_ranges` calls, or `None` if the receiver is gone.
272+
async fn spawn_group(
273+
group: Vec<PendingRecordBatch>,
274+
byte_source: &Arc<DynByteSource>,
275+
memory_prefetch_func: fn(&[u8]) -> (),
276+
prefetch_send: &Sender<(
277+
tokio_handle_ext::AbortOnDropHandle<PolarsResult<RecordBatchData>>,
278+
Option<PipelinePermit>,
279+
)>,
280+
) -> Option<u64> {
281+
let group_bytes = match byte_source.as_ref() {
282+
DynByteSource::Buffer(mem_slice) => GroupBytes::Memory(mem_slice.0.clone()),
283+
_ => {
284+
let mut ranges: Vec<Range<usize>> = group
285+
.iter()
286+
.map(|rb| rb.range.clone())
287+
.filter(|range| !range.is_empty())
288+
.collect();
289+
let byte_source = byte_source.clone();
290+
GroupBytes::Fetched(
291+
async move { byte_source.get_ranges(&mut ranges).await.map(Arc::new) }.shared(),
292+
)
293+
},
294+
};
295+
let n_fetches = (matches!(group_bytes, GroupBytes::Fetched(_))
296+
&& group.iter().any(|rb| !rb.range.is_empty())) as u64;
297+
298+
for PendingRecordBatch {
299+
record_batch_idx,
300+
num_rows,
301+
row_offset,
302+
slice,
303+
range,
304+
fetch_permit,
305+
} in group
306+
{
307+
let group_bytes = group_bytes.clone();
308+
309+
let fetch_handle = ASYNC.spawn(async move {
310+
let fetched_bytes = if range.is_empty() {
311+
Buffer::new()
312+
} else {
313+
match group_bytes {
314+
GroupBytes::Memory(mem) => {
315+
if !std::ptr::eq(
316+
memory_prefetch_func as *const (),
317+
polars_utils::mem::prefetch::no_prefetch as *const (),
318+
) {
319+
debug_assert!(range.end <= mem.len());
320+
memory_prefetch_func(unsafe { mem.as_ref().get_unchecked(range.clone()) })
321+
}
322+
323+
mem.sliced(range)
324+
},
325+
GroupBytes::Fetched(group_fetch) => group_fetch
326+
.await?
327+
.get(&range.start)
328+
.filter(|bytes| bytes.len() == range.len())
329+
.cloned()
330+
.ok_or_else(|| {
331+
polars_err!(ComputeError: "IPC record batch {record_batch_idx} not in its group fetch")
332+
})?,
333+
}
334+
};
335+
336+
// Extract the length (i.e., nr of rows) at the earliest possible opportunity.
337+
let num_rows = if let Some(num_rows) = num_rows {
338+
num_rows
339+
} else {
340+
let mut reader = BlockReader::new(Cursor::new(fetched_bytes.as_ref()));
341+
let mut message_scratch = vec![];
342+
reader
343+
.record_batch_num_rows(&mut message_scratch)?
344+
.try_into()
345+
.map_err(|_| polars_err!(ComputeError: LENGTH_LIMIT_MSG))?
346+
};
347+
348+
PolarsResult::Ok(RecordBatchData {
349+
fetched_bytes,
350+
record_batch_idx,
351+
num_rows,
352+
row_offset,
353+
slice,
354+
})
355+
});
356+
357+
let fetch_handle = tokio_handle_ext::AbortOnDropHandle(fetch_handle);
358+
359+
if prefetch_send
360+
.send((fetch_handle, fetch_permit))
361+
.await
362+
.is_err()
363+
{
364+
return None;
365+
}
366+
}
367+
368+
Some(n_fetches)
369+
}

‎crates/polars-stream/src/nodes/io_sources/shared/pipeline_budget.rs‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -60,7 +60,6 @@ impl PipelineBudget {
6060
self.count_limit
6161
}
6262

63-
#[allow(unused)]
6463
pub(crate) fn kbytes_limit(&self) -> usize {
6564
self.kbytes_limit
6665
}
@@ -74,12 +73,7 @@ impl PipelineBudget {
7473
/// The requested capacity is capped at the kbytes limit, so a fetch larger than the
7574
/// limit still proceeds, and any positive limit makes progress.
7675
pub(crate) async fn acquire(&self, n_bytes: usize) -> PipelinePermit {
77-
// Prevent deadlock.
78-
let n_kbytes: u32 = n_bytes
79-
.div_ceil(1 << 10)
80-
.min(self.kbytes_limit)
81-
.try_into()
82-
.unwrap_or(u32::MAX);
76+
let n_kbytes: u32 = self.permit_kbytes(n_bytes).try_into().unwrap_or(u32::MAX);
8377

8478
// Semaphores are never closed, so acquire cannot fail.
8579
let _kbytes = self
@@ -118,6 +112,12 @@ impl PipelineBudget {
118112

119113
PipelinePermit { _count, _kbytes }
120114
}
115+
116+
/// Kilobytes taken by the permit for a fetch of `n_bytes`.
117+
pub(crate) fn permit_kbytes(&self, n_bytes: usize) -> usize {
118+
// Capped at the limit to prevent deadlock.
119+
n_bytes.div_ceil(1 << 10).min(self.kbytes_limit)
120+
}
121121
}
122122

123123
/// RAII permit pair; releases both budgets on drop.

0 commit comments

Comments
 (0)