@@ -119,16 +119,16 @@ pub(super) struct ParquetOpener {
119119}
120120
121121/// Represents a prepared access plan with optional row selection
122- struct PreparedAccessPlan {
122+ pub ( crate ) struct PreparedAccessPlan {
123123 /// Row group indexes to read
124- row_group_indexes : Vec < usize > ,
124+ pub ( crate ) row_group_indexes : Vec < usize > ,
125125 /// Optional row selection for filtering within row groups
126- row_selection : Option < parquet:: arrow:: arrow_reader:: RowSelection > ,
126+ pub ( crate ) row_selection : Option < parquet:: arrow:: arrow_reader:: RowSelection > ,
127127}
128128
129129impl PreparedAccessPlan {
130130 /// Create a new prepared access plan from a ParquetAccessPlan
131- fn from_access_plan (
131+ pub ( crate ) fn from_access_plan (
132132 access_plan : ParquetAccessPlan ,
133133 rg_metadata : & [ RowGroupMetaData ] ,
134134 ) -> Result < Self > {
@@ -142,17 +142,23 @@ impl PreparedAccessPlan {
142142 }
143143
144144 /// Reverse the access plan for reverse scanning
145- fn reverse (
145+ pub ( crate ) fn reverse (
146146 mut self ,
147147 file_metadata : & parquet:: file:: metadata:: ParquetMetaData ,
148148 ) -> Result < Self > {
149+ // Get the row group indexes before reversing
150+ let row_groups_to_scan = self . row_group_indexes . clone ( ) ;
151+
149152 // Reverse the row group indexes
150153 self . row_group_indexes = self . row_group_indexes . into_iter ( ) . rev ( ) . collect ( ) ;
151154
152155 // If we have a row selection, reverse it to match the new row group order
153156 if let Some ( row_selection) = self . row_selection {
154- self . row_selection =
155- Some ( reverse_row_selection ( & row_selection, file_metadata) ?) ;
157+ self . row_selection = Some ( reverse_row_selection (
158+ & row_selection,
159+ file_metadata,
160+ & row_groups_to_scan, // Pass the original (non-reversed) row group indexes
161+ ) ?) ;
156162 }
157163
158164 Ok ( self )
@@ -826,8 +832,9 @@ fn should_enable_page_index(
826832
827833#[ cfg( test) ]
828834mod test {
829- use crate :: { opener :: ParquetOpener , DefaultParquetFileReaderFactory } ;
835+ use std :: sync :: Arc ;
830836 use arrow:: compute:: cast;
837+ use crate :: { DefaultParquetFileReaderFactory , RowGroupAccess , opener:: ParquetOpener } ;
831838 use arrow:: datatypes:: { DataType , Field , Schema , SchemaRef } ;
832839 use bytes:: { BufMut , BytesMut } ;
833840 use datafusion_common:: {
@@ -851,7 +858,6 @@ mod test {
851858 use object_store:: { memory:: InMemory , path:: Path , ObjectStore } ;
852859 use parquet:: arrow:: ArrowWriter ;
853860 use parquet:: file:: properties:: WriterProperties ;
854- use std:: sync:: Arc ;
855861
856862 async fn count_batches_and_rows (
857863 mut stream : std:: pin:: Pin <
@@ -1726,4 +1732,122 @@ mod test {
17261732 "Reverse scan should reverse row group order while maintaining correct RowSelection for each group"
17271733 ) ;
17281734 }
1735+
1736+ #[ tokio:: test]
1737+ async fn test_reverse_scan_with_non_contiguous_row_groups ( ) {
1738+ use parquet:: file:: properties:: WriterProperties ;
1739+
1740+ let store = Arc :: new ( InMemory :: new ( ) ) as Arc < dyn ObjectStore > ;
1741+
1742+ // Create 4 batches (4 row groups)
1743+ let batch0 = record_batch ! ( ( "a" , Int32 , vec![ Some ( 1 ) , Some ( 2 ) ] ) ) . unwrap ( ) ;
1744+ let batch1 = record_batch ! ( ( "a" , Int32 , vec![ Some ( 3 ) , Some ( 4 ) ] ) ) . unwrap ( ) ;
1745+ let batch2 = record_batch ! ( ( "a" , Int32 , vec![ Some ( 5 ) , Some ( 6 ) ] ) ) . unwrap ( ) ;
1746+ let batch3 = record_batch ! ( ( "a" , Int32 , vec![ Some ( 7 ) , Some ( 8 ) ] ) ) . unwrap ( ) ;
1747+
1748+ let props = WriterProperties :: builder ( )
1749+ . set_max_row_group_size ( 2 )
1750+ . build ( ) ;
1751+
1752+ let data_len = write_parquet_batches (
1753+ Arc :: clone ( & store) ,
1754+ "test.parquet" ,
1755+ vec ! [ batch0. clone( ) , batch1, batch2, batch3] ,
1756+ Some ( props) ,
1757+ )
1758+ . await ;
1759+
1760+ let schema = batch0. schema ( ) ;
1761+
1762+ use crate :: ParquetAccessPlan ;
1763+ use parquet:: arrow:: arrow_reader:: { RowSelection , RowSelector } ;
1764+
1765+ // KEY: Skip RG1 (non-contiguous!)
1766+ // Only scan row groups: [0, 2, 3]
1767+ let mut access_plan = ParquetAccessPlan :: new ( vec ! [
1768+ RowGroupAccess :: Scan , // RG0
1769+ RowGroupAccess :: Skip , // RG1 - SKIPPED!
1770+ RowGroupAccess :: Scan , // RG2
1771+ RowGroupAccess :: Scan , // RG3
1772+ ] ) ;
1773+
1774+ // Add RowSelection for each scanned row group
1775+ // RG0: select first row (1), skip second (2)
1776+ access_plan. scan_selection (
1777+ 0 ,
1778+ RowSelection :: from ( vec ! [ RowSelector :: select( 1 ) , RowSelector :: skip( 1 ) ] ) ,
1779+ ) ;
1780+ // RG1: skipped, no selection needed
1781+ // RG2: select first row (5), skip second (6)
1782+ access_plan. scan_selection (
1783+ 2 ,
1784+ RowSelection :: from ( vec ! [ RowSelector :: select( 1 ) , RowSelector :: skip( 1 ) ] ) ,
1785+ ) ;
1786+ // RG3: select first row (7), skip second (8)
1787+ access_plan. scan_selection (
1788+ 3 ,
1789+ RowSelection :: from ( vec ! [ RowSelector :: select( 1 ) , RowSelector :: skip( 1 ) ] ) ,
1790+ ) ;
1791+
1792+ let file = PartitionedFile :: new (
1793+ "test.parquet" . to_string ( ) ,
1794+ u64:: try_from ( data_len) . unwrap ( ) ,
1795+ )
1796+ . with_extensions ( Arc :: new ( access_plan) ) ;
1797+
1798+ let make_opener = |reverse_scan : bool | ParquetOpener {
1799+ projection : Arc :: new ( [ 0 ] ) ,
1800+ partition_index : 0 ,
1801+ batch_size : 1024 ,
1802+ limit : None ,
1803+ predicate : None ,
1804+ metadata_size_hint : None ,
1805+ metrics : ExecutionPlanMetricsSet :: new ( ) ,
1806+ parquet_file_reader_factory : Arc :: new ( DefaultParquetFileReaderFactory :: new (
1807+ Arc :: clone ( & store) ,
1808+ ) ) ,
1809+ partition_fields : vec ! [ ] ,
1810+ pushdown_filters : true ,
1811+ reorder_filters : false ,
1812+ enable_page_index : false ,
1813+ enable_bloom_filter : false ,
1814+ enable_limit_pruning : false ,
1815+ schema_adapter_factory : Arc :: new ( DefaultSchemaAdapterFactory ) ,
1816+ enable_row_group_stats_pruning : false ,
1817+ coerce_int96 : None ,
1818+ #[ cfg( feature = "parquet_encryption" ) ]
1819+ file_decryption_properties : None ,
1820+ expr_adapter_factory : Some ( Arc :: new ( DefaultPhysicalExprAdapterFactory ) ) ,
1821+ #[ cfg( feature = "parquet_encryption" ) ]
1822+ encryption_factory : None ,
1823+ max_predicate_cache_size : None ,
1824+ reverse_row_groups : reverse_scan,
1825+ logical_file_schema : schema. clone ( ) ,
1826+ } ;
1827+
1828+ // Forward scan: RG0(1), RG2(5), RG3(7)
1829+ // Note: RG1 is completely skipped
1830+ let opener = make_opener ( false ) ;
1831+ let stream = opener. open ( file. clone ( ) ) . unwrap ( ) . await . unwrap ( ) ;
1832+ let forward_values = collect_int32_values ( stream) . await ;
1833+
1834+ assert_eq ! (
1835+ forward_values,
1836+ vec![ 1 , 5 , 7 ] ,
1837+ "Forward scan with non-contiguous row groups"
1838+ ) ;
1839+
1840+ // Reverse scan: RG3(7), RG2(5), RG0(1)
1841+ // WITHOUT the bug fix, this would return WRONG values
1842+ // because the RowSelection would be incorrectly mapped
1843+ let opener = make_opener ( true ) ;
1844+ let stream = opener. open ( file) . unwrap ( ) . await . unwrap ( ) ;
1845+ let reverse_values = collect_int32_values ( stream) . await ;
1846+
1847+ assert_eq ! (
1848+ reverse_values,
1849+ vec![ 7 , 5 , 1 ] ,
1850+ "Reverse scan with non-contiguous row groups should correctly map RowSelection"
1851+ ) ;
1852+ }
17291853}
0 commit comments