11use std:: io:: Cursor ;
2+ use std:: num:: NonZeroUsize ;
23use std:: ops:: Range ;
34use std:: sync:: Arc ;
45
6+ use futures:: FutureExt ;
57use polars_async:: primitives:: wait_group:: WaitToken ;
68use polars_buffer:: Buffer ;
79use polars_config:: config;
@@ -28,6 +30,37 @@ pub(super) struct RecordBatchData {
2830 pub ( super ) slice : Option < ( usize , usize ) > ,
2931}
3032
33+ /// Largest gap between record batches that are fetched together.
34+ const MAX_GROUP_GAP : usize = 4096 ;
35+
36+ /// A group takes at most 1 / `GROUP_BUDGET_SHARE` of the pipeline budget, so it can always be
37+ /// filled.
38+ const GROUP_BUDGET_SHARE : usize = 4 ;
39+
40+ /// Default largest span of a group. Larger requests leave too few in flight under the current
41+ /// in-flight budget.
42+ const DEFAULT_GROUP_MAX_BYTES : usize = 2 << 20 ;
43+
44+ /// A record batch waiting for its group to be fetched.
45+ struct PendingRecordBatch {
46+ record_batch_idx : usize ,
47+ num_rows : Option < IdxSize > ,
48+ row_offset : Option < IdxSize > ,
49+ slice : Option < ( usize , usize ) > ,
50+ range : Range < usize > ,
51+ fetch_permit : Option < PipelinePermit > ,
52+ }
53+
54+ /// Where the record batches of a group get their bytes from.
55+ #[ derive( Clone ) ]
56+ enum GroupBytes < F > {
57+ /// Memory-mapped or in-memory file, sliced per record batch.
58+ Memory ( Buffer < u8 > ) ,
59+ /// One `get_ranges` call for the group, shared by its record batches and dropped, cancelling
60+ /// the fetch, with the last of them.
61+ Fetched ( F ) ,
62+ }
63+
3164pub ( super ) struct RecordBatchDataFetcher {
3265 pub ( super ) file_metadata : Arc < FileMetadata > ,
3366 pub ( super ) record_batch_cum_len : Option < Buffer < IdxSize > > ,
@@ -76,6 +109,26 @@ impl RecordBatchDataFetcher {
76109 ) ;
77110 let mut rb_fetch_count: u64 = 0 ;
78111
112+ // Consecutive record batches are fetched together, like Parquet row groups. Groups depend
113+ // only on the file and these limits, not on load.
114+ // Only remote sources have a chunk size, so only they group.
115+ // TODO: Revisit the group size once the in-flight budget is sized for large requests.
116+ let max_group_span = byte_source. chunk_size ( ) . map_or ( 0 , |chunk_size| {
117+ let default = chunk_size. min ( DEFAULT_GROUP_MAX_BYTES ) ;
118+ std:: env:: var ( "POLARS_RECORD_BATCH_GROUP_MAX_BYTES" ) . map_or ( default, |x| {
119+ x. parse :: < NonZeroUsize > ( )
120+ . unwrap_or_else ( |_| {
121+ panic ! ( "invalid value for POLARS_RECORD_BATCH_GROUP_MAX_BYTES: {x}" )
122+ } )
123+ . get ( )
124+ } )
125+ } ) ;
126+ let max_group_len = ( pipeline_budget. count_limit ( ) / GROUP_BUDGET_SHARE ) . max ( 1 ) ;
127+ let max_group_kbytes = pipeline_budget. kbytes_limit ( ) / GROUP_BUDGET_SHARE ;
128+ let mut group: Vec < PendingRecordBatch > = Vec :: new ( ) ;
129+ let mut group_kbytes: usize = 0 ;
130+ let mut group_fetch_count: u64 = 0 ;
131+
79132 for record_batch_idx in 0 ..file_metadata. blocks . len ( ) {
80133 let mut num_rows_this_rb: Option < IdxSize > = None ;
81134 let mut row_offset: Option < IdxSize > = None ;
@@ -136,66 +189,62 @@ impl RecordBatchDataFetcher {
136189 ..usize:: checked_add ( block. offset as _ , fetch_length)
137190 . ok_or_else ( || polars_err ! ( ComputeError : "IPC block range overflows usize" ) ) ?;
138191
139- let fetch_permit = match rb_fetch {
140- RbFetch :: All | RbFetch :: Metadata => {
141- Some ( pipeline_budget. acquire ( fetch_length) . await )
142- } ,
143- RbFetch :: None => None ,
144- } ;
145-
146- let current_byte_source = byte_source. clone ( ) ;
147-
148- let fetch_handle = ASYNC . spawn ( async move {
149- let fetched_bytes = if range. is_empty ( ) {
150- Buffer :: new ( )
151- } else if let DynByteSource :: Buffer ( mem_slice) = current_byte_source. as_ref ( ) {
152- let slice = mem_slice. 0 . as_ref ( ) ;
153-
154- if !std:: ptr:: eq (
155- memory_prefetch_func as * const ( ) ,
156- polars_utils:: mem:: prefetch:: no_prefetch as * const ( ) ,
157- ) {
158- debug_assert ! ( range. end <= slice. len( ) ) ;
159- memory_prefetch_func ( unsafe { slice. get_unchecked ( range. clone ( ) ) } )
160- }
161-
162- mem_slice. 0 . clone ( ) . sliced ( range)
163- } else {
164- // @NOTE. Performance can be optimized by grouping requests and downloading
165- // through `get_ranges()`.
166- current_byte_source. get_range ( range) . await ?
192+ let permit_kbytes = pipeline_budget. permit_kbytes ( fetch_length) ;
193+
194+ // Start a new group at a gap or out-of-order block, so `get_ranges` does not fetch the
195+ // bytes in between, when it would span more than one request, or when it would take
196+ // more than its share of the budget.
197+ let starts_new_group = group
198+ . first ( )
199+ . zip ( group. last ( ) )
200+ . is_some_and ( |( first, last) | {
201+ range
202+ . start
203+ . checked_sub ( last. range . end )
204+ . is_none_or ( |gap| gap > MAX_GROUP_GAP )
205+ || range. end - first. range . start > max_group_span
206+ || group. len ( ) >= max_group_len
207+ || group_kbytes + permit_kbytes > max_group_kbytes
208+ } ) ;
209+
210+ if starts_new_group {
211+ group_kbytes = 0 ;
212+ let Some ( n) = spawn_group (
213+ std:: mem:: take ( & mut group) ,
214+ & byte_source,
215+ memory_prefetch_func,
216+ & prefetch_send,
217+ )
218+ . await
219+ else {
220+ break ;
167221 } ;
222+ group_fetch_count += n;
223+ }
168224
169- // Extract the length (i.e., nr of rows) at the earliest possible opportunity.
170- let num_rows = if let Some ( num_rows_this_rb) = num_rows_this_rb {
171- num_rows_this_rb
172- } else {
173- let mut reader = BlockReader :: new ( Cursor :: new ( fetched_bytes. as_ref ( ) ) ) ;
174- let mut message_scratch = vec ! [ ] ;
175- reader
176- . record_batch_num_rows ( & mut message_scratch) ?
177- . try_into ( )
178- . map_err ( |_| polars_err ! ( ComputeError : LENGTH_LIMIT_MSG ) ) ?
179- } ;
225+ // Load delays a group but never splits it: it takes at most a share of the budget, and
226+ // all other permits are held by earlier record batches, which are always consumed.
227+ let fetch_permit = if rb_fetch == RbFetch :: None {
228+ None
229+ } else {
230+ Some ( pipeline_budget. acquire ( fetch_length) . await )
231+ } ;
180232
181- PolarsResult :: Ok ( RecordBatchData {
182- fetched_bytes,
183- record_batch_idx,
184- num_rows,
185- row_offset,
186- slice,
187- } )
233+ group_kbytes += permit_kbytes;
234+ group. push ( PendingRecordBatch {
235+ record_batch_idx,
236+ num_rows : num_rows_this_rb,
237+ row_offset,
238+ slice,
239+ range,
240+ fetch_permit,
188241 } ) ;
242+ }
189243
190- let fetch_handle = tokio_handle_ext:: AbortOnDropHandle ( fetch_handle) ;
191-
192- if prefetch_send
193- . send ( ( fetch_handle, fetch_permit) )
194- . await
195- . is_err ( )
196- {
197- break ;
198- }
244+ if let Some ( n) =
245+ spawn_group ( group, & byte_source, memory_prefetch_func, & prefetch_send) . await
246+ {
247+ group_fetch_count += n;
199248 }
200249
201250 drop ( rb_prefetch_current_all_spawned) ;
@@ -209,10 +258,112 @@ impl RecordBatchDataFetcher {
209258 "[IpcFileReader]: RecordBatchDataFetcher: \
210259 rb_total_count: {rb_total_count}, \
211260 rb_full_fetch_count: {rb_full_fetch_count}, \
212- rb_metadata_fetch_count: {rb_metadata_fetch_count}"
261+ rb_metadata_fetch_count: {rb_metadata_fetch_count}, \
262+ group_fetch_count: {group_fetch_count}"
213263 )
214264 }
215265
216266 Ok ( ( ) )
217267 }
218268}
269+
270+ /// Spawns the fetch of a group of record batches and sends a handle per record batch in order.
271+ /// Returns the number of `get_ranges` calls, or `None` if the receiver is gone.
272+ async fn spawn_group (
273+ group : Vec < PendingRecordBatch > ,
274+ byte_source : & Arc < DynByteSource > ,
275+ memory_prefetch_func : fn ( & [ u8 ] ) -> ( ) ,
276+ prefetch_send : & Sender < (
277+ tokio_handle_ext:: AbortOnDropHandle < PolarsResult < RecordBatchData > > ,
278+ Option < PipelinePermit > ,
279+ ) > ,
280+ ) -> Option < u64 > {
281+ let group_bytes = match byte_source. as_ref ( ) {
282+ DynByteSource :: Buffer ( mem_slice) => GroupBytes :: Memory ( mem_slice. 0 . clone ( ) ) ,
283+ _ => {
284+ let mut ranges: Vec < Range < usize > > = group
285+ . iter ( )
286+ . map ( |rb| rb. range . clone ( ) )
287+ . filter ( |range| !range. is_empty ( ) )
288+ . collect ( ) ;
289+ let byte_source = byte_source. clone ( ) ;
290+ GroupBytes :: Fetched (
291+ async move { byte_source. get_ranges ( & mut ranges) . await . map ( Arc :: new) } . shared ( ) ,
292+ )
293+ } ,
294+ } ;
295+ let n_fetches = ( matches ! ( group_bytes, GroupBytes :: Fetched ( _) )
296+ && group. iter ( ) . any ( |rb| !rb. range . is_empty ( ) ) ) as u64 ;
297+
298+ for PendingRecordBatch {
299+ record_batch_idx,
300+ num_rows,
301+ row_offset,
302+ slice,
303+ range,
304+ fetch_permit,
305+ } in group
306+ {
307+ let group_bytes = group_bytes. clone ( ) ;
308+
309+ let fetch_handle = ASYNC . spawn ( async move {
310+ let fetched_bytes = if range. is_empty ( ) {
311+ Buffer :: new ( )
312+ } else {
313+ match group_bytes {
314+ GroupBytes :: Memory ( mem) => {
315+ if !std:: ptr:: eq (
316+ memory_prefetch_func as * const ( ) ,
317+ polars_utils:: mem:: prefetch:: no_prefetch as * const ( ) ,
318+ ) {
319+ debug_assert ! ( range. end <= mem. len( ) ) ;
320+ memory_prefetch_func ( unsafe { mem. as_ref ( ) . get_unchecked ( range. clone ( ) ) } )
321+ }
322+
323+ mem. sliced ( range)
324+ } ,
325+ GroupBytes :: Fetched ( group_fetch) => group_fetch
326+ . await ?
327+ . get ( & range. start )
328+ . filter ( |bytes| bytes. len ( ) == range. len ( ) )
329+ . cloned ( )
330+ . ok_or_else ( || {
331+ polars_err ! ( ComputeError : "IPC record batch {record_batch_idx} not in its group fetch" )
332+ } ) ?,
333+ }
334+ } ;
335+
336+ // Extract the length (i.e., nr of rows) at the earliest possible opportunity.
337+ let num_rows = if let Some ( num_rows) = num_rows {
338+ num_rows
339+ } else {
340+ let mut reader = BlockReader :: new ( Cursor :: new ( fetched_bytes. as_ref ( ) ) ) ;
341+ let mut message_scratch = vec ! [ ] ;
342+ reader
343+ . record_batch_num_rows ( & mut message_scratch) ?
344+ . try_into ( )
345+ . map_err ( |_| polars_err ! ( ComputeError : LENGTH_LIMIT_MSG ) ) ?
346+ } ;
347+
348+ PolarsResult :: Ok ( RecordBatchData {
349+ fetched_bytes,
350+ record_batch_idx,
351+ num_rows,
352+ row_offset,
353+ slice,
354+ } )
355+ } ) ;
356+
357+ let fetch_handle = tokio_handle_ext:: AbortOnDropHandle ( fetch_handle) ;
358+
359+ if prefetch_send
360+ . send ( ( fetch_handle, fetch_permit) )
361+ . await
362+ . is_err ( )
363+ {
364+ return None ;
365+ }
366+ }
367+
368+ Some ( n_fetches)
369+ }
0 commit comments