@@ -322,6 +322,9 @@ pub(super) struct ParquetMorselizer {
322322 /// Read-ahead window in bytes. If set, decode a batch at a time. Sourced
323323 /// from `datafusion.execution.parquet.read_ahead_bytes`.
324324 pub read_ahead_bytes : Option < u64 > ,
325+ /// Bytes each file stream fetches when it is built. Sourced from
326+ /// `datafusion.execution.read_ahead_eager_bytes`. 0 turns it off.
327+ pub read_ahead_eager_bytes : u64 ,
325328 /// Whether to read row groups in reverse order
326329 pub reverse_row_groups : bool ,
327330 /// Optional sort order used to reorder row groups by their min/max statistics.
@@ -529,6 +532,7 @@ struct PreparedParquetOpen {
529532 max_in_list_size : usize ,
530533 row_group_range_assignment : RowGroupRangeAssignment ,
531534 read_ahead_bytes : Option < u64 > ,
535+ read_ahead_eager_bytes : u64 ,
532536 reverse_row_groups : bool ,
533537 sort_order_for_reorder : Option < LexOrdering > ,
534538 preserve_order : bool ,
@@ -1178,6 +1182,7 @@ impl ParquetMorselizer {
11781182 max_in_list_size : self . max_in_list_size ,
11791183 row_group_range_assignment : self . row_group_range_assignment ,
11801184 read_ahead_bytes : self . read_ahead_bytes ,
1185+ read_ahead_eager_bytes : self . read_ahead_eager_bytes ,
11811186 reverse_row_groups : self . reverse_row_groups ,
11821187 sort_order_for_reorder : self . sort_order_for_reorder . clone ( ) ,
11831188 preserve_order : self . preserve_order ,
@@ -1919,6 +1924,7 @@ impl RowGroupsPrunedParquetOpen {
19191924 . sum ( ) ;
19201925
19211926 let read_ahead_bytes = prepared. read_ahead_bytes ;
1927+ let read_ahead_eager_bytes = prepared. read_ahead_eager_bytes ;
19221928
19231929 // Lazily-registered suppression counter shared by the open-time first-RG
19241930 // skip below and the stream's per-RG toggle (registered on first use so
@@ -2111,7 +2117,7 @@ impl RowGroupsPrunedParquetOpen {
21112117 ReadAheadMemory :: new ( prepared. memory_pool , prepared. partition_index ) ;
21122118 ReadAhead :: new ( window, decoder. scan_plan ( ) , memory)
21132119 } ) ;
2114- let stream = PushDecoderStreamState {
2120+ let mut stream_state = PushDecoderStreamState {
21152121 decoder : Some ( decoder) ,
21162122 active_reader : None ,
21172123 rg_plan,
@@ -2145,8 +2151,9 @@ impl RowGroupsPrunedParquetOpen {
21452151 flush_at_row_group_boundary : watches_dynamic_filter,
21462152 projection_builder,
21472153 read_ahead,
2148- }
2149- . into_stream ( ) ;
2154+ } ;
2155+ stream_state. start_eager_read_ahead ( read_ahead_eager_bytes) ;
2156+ let stream = stream_state. into_stream ( ) ;
21502157
21512158 // Wrap the stream so a dynamic filter can stop the file scan early, but
21522159 // only when the pruner is still watching a filter that can change
@@ -2462,6 +2469,7 @@ mod test {
24622469 max_in_list_size : usize ,
24632470 row_group_range_assignment : RowGroupRangeAssignment ,
24642471 read_ahead_bytes : Option < u64 > ,
2472+ read_ahead_eager_bytes : u64 ,
24652473 memory_pool : Option < Arc < dyn MemoryPool > > ,
24662474 reverse_row_groups : bool ,
24672475 preserve_order : bool ,
@@ -2686,6 +2694,7 @@ mod test {
26862694 max_in_list_size : MAX_IN_LIST_SIZE ,
26872695 row_group_range_assignment : RowGroupRangeAssignment :: default ( ) ,
26882696 read_ahead_bytes : None ,
2697+ read_ahead_eager_bytes : 0 ,
26892698 memory_pool : None ,
26902699 reverse_row_groups : false ,
26912700 preserve_order : false ,
@@ -2783,6 +2792,12 @@ mod test {
27832792 self
27842793 }
27852794
2795+ /// Set the eager read-ahead bytes (`read_ahead_eager_bytes`).
2796+ fn with_read_ahead_eager_bytes ( mut self , bytes : u64 ) -> Self {
2797+ self . read_ahead_eager_bytes = bytes;
2798+ self
2799+ }
2800+
27862801 /// Set the read-ahead window (`read_ahead_bytes`).
27872802 fn with_read_ahead_bytes ( mut self , bytes : u64 ) -> Self {
27882803 self . read_ahead_bytes = Some ( bytes) ;
@@ -2926,6 +2941,7 @@ mod test {
29262941 max_in_list_size : self . max_in_list_size ,
29272942 row_group_range_assignment : self . row_group_range_assignment ,
29282943 read_ahead_bytes : self . read_ahead_bytes ,
2944+ read_ahead_eager_bytes : self . read_ahead_eager_bytes ,
29292945 reverse_row_groups : self . reverse_row_groups ,
29302946 sort_order_for_reorder : None ,
29312947 virtual_state,
@@ -7180,6 +7196,57 @@ mod test {
71807196 }
71817197 }
71827198
7199+ /// Open the file with read-ahead and `eager` bytes of eager
7200+ /// read-ahead. Returns the data fetches made before the first poll of
7201+ /// the stream, and the rows that the stream then reads.
7202+ async fn fetches_before_first_poll ( eager : u64 ) -> ( Vec < u64 > , usize ) {
7203+ let store = Arc :: new ( InMemory :: new ( ) ) as Arc < dyn ObjectStore > ;
7204+ let ( schema, len) = write_file ( & store) . await ;
7205+ let fetches = Arc :: new ( Mutex :: new ( vec ! [ ] ) ) ;
7206+ let factory = RecordingReaderFactory {
7207+ inner : DefaultParquetFileReaderFactory :: new ( Arc :: clone ( & store) ) ,
7208+ fetches : Arc :: clone ( & fetches) ,
7209+ } ;
7210+ let pool: Arc < dyn MemoryPool > =
7211+ Arc :: new ( GreedyMemoryPool :: new ( 64 * 1024 * 1024 ) ) ;
7212+ let morselizer = ParquetMorselizerBuilder :: new ( )
7213+ . with_store ( Arc :: clone ( & store) )
7214+ . with_schema ( schema)
7215+ . with_projection_indices ( & [ 0 , 1 ] )
7216+ . with_parquet_file_reader_factory ( Arc :: new ( factory) )
7217+ . with_read_ahead_bytes ( WINDOW )
7218+ . with_read_ahead_eager_bytes ( eager)
7219+ . with_memory_pool ( Arc :: clone ( & pool) )
7220+ . build ( ) ;
7221+ let file = PartitionedFile :: new ( "read_ahead.parquet" . to_string ( ) , len as u64 ) ;
7222+ let stream = open_file ( & morselizer, file) . await . unwrap ( ) ;
7223+ // The spawned fetch runs without the stream being polled.
7224+ tokio:: time:: sleep ( std:: time:: Duration :: from_millis ( 50 ) ) . await ;
7225+ let before = fetches. lock ( ) . unwrap ( ) . clone ( ) ;
7226+ let ( _batches, rows) = count_batches_and_rows ( stream) . await ;
7227+ drop ( morselizer) ;
7228+ assert_eq ! ( pool. reserved( ) , 0 , "the stream returns all it reserved" ) ;
7229+ ( before, rows)
7230+ }
7231+
7232+ /// Verifies that eager read-ahead fetches data before the stream is
7233+ /// polled, within its limit, and that the scan still reads every row.
7234+ #[ tokio:: test]
7235+ async fn eager_read_ahead_fetches_before_the_first_poll ( ) {
7236+ let ( before, rows) = fetches_before_first_poll ( 0 ) . await ;
7237+ assert ! (
7238+ before. is_empty( ) ,
7239+ "no data fetch without eager read-ahead: {before:?}"
7240+ ) ;
7241+ assert_eq ! ( rows, ROWS as usize ) ;
7242+
7243+ let eager = WINDOW / 2 ;
7244+ let ( before, rows) = fetches_before_first_poll ( eager) . await ;
7245+ assert_eq ! ( before. len( ) , 1 , "one eager fetch: {before:?}" ) ;
7246+ assert ! ( before[ 0 ] > 0 && before[ 0 ] <= eager, "{before:?}" ) ;
7247+ assert_eq ! ( rows, ROWS as usize ) ;
7248+ }
7249+
71837250 fn concat ( batches : & [ RecordBatch ] ) -> RecordBatch {
71847251 concat_batches ( & batches[ 0 ] . schema ( ) , batches) . unwrap ( )
71857252 }
0 commit comments