@@ -926,24 +926,30 @@ impl MetadataLoadedParquetOpen {
926926 & prepared. predicate_creation_errors ,
927927 ) ;
928928
929- // Only build page pruning predicate if page index is enabled
930- let page_pruning_predicate = if prepared. enable_page_index {
931- prepared. predicate . as_ref ( ) . and_then ( |predicate| {
932- let p = build_page_pruning_predicate ( predicate, & physical_file_schema) ;
933- ( p. filter_number ( ) > 0 ) . then_some ( p)
934- } )
935- } else {
936- None
937- } ;
938-
939929 Ok ( FiltersPreparedParquetOpen {
940930 loaded : MetadataLoadedParquetOpen {
941931 prepared,
942932 reader_metadata,
943933 options,
944934 } ,
945935 pruning_predicate,
946- page_pruning_predicate,
936+ page_pruning_predicate : None ,
937+ } )
938+ }
939+ }
940+
941+ impl MetadataLoadedParquetOpen {
942+ fn has_page_index ( & self ) -> bool {
943+ self . reader_metadata . metadata ( ) . column_index ( ) . is_some ( )
944+ && self . reader_metadata . metadata ( ) . offset_index ( ) . is_some ( )
945+ }
946+ }
947+
948+ impl PreparedParquetOpen {
949+ fn build_page_pruning_predicate ( & self ) -> Option < Arc < PagePruningAccessPlanFilter > > {
950+ self . predicate . as_ref ( ) . and_then ( |predicate| {
951+ let p = build_page_pruning_predicate ( predicate, & self . physical_file_schema ) ;
952+ ( p. filter_number ( ) > 0 ) . then_some ( p)
947953 } )
948954 }
949955}
@@ -953,9 +959,9 @@ impl FiltersPreparedParquetOpen {
953959 async fn load_page_index ( mut self ) -> Result < Self > {
954960 // The page index is not stored inline in the parquet footer so the
955961 // metadata load above may not have read the page index structures yet.
956- // If we need them for reading and they aren't yet loaded, we need to
962+ // If we need them for reading, and they aren't yet loaded, we need to
957963 // load them now.
958- if self . page_pruning_predicate . is_some ( ) {
964+ if self . loaded . prepared . enable_page_index {
959965 self . loaded . reader_metadata = load_page_index (
960966 self . loaded . reader_metadata ,
961967 & mut self . loaded . prepared . async_file_reader ,
@@ -965,6 +971,13 @@ impl FiltersPreparedParquetOpen {
965971 . with_page_index_policy ( PageIndexPolicy :: Optional ) ,
966972 )
967973 . await ?;
974+
975+ // Since creating a page pruning predicate can be non trivial
976+ // only do it when we are sure we have enough metadata to use it
977+ if self . loaded . has_page_index ( ) {
978+ self . page_pruning_predicate =
979+ self . loaded . prepared . build_page_pruning_predicate ( ) ;
980+ }
968981 }
969982
970983 Ok ( self )
@@ -2708,70 +2721,122 @@ mod test {
27082721 ) ;
27092722 }
27102723
2711- /// Test that page pruning predicates are only built and applied when `enable_page_index` is true.
2712- ///
2713- /// The file has a single row group with 10 pages (10 rows each, values 1..100).
2714- /// With page index enabled, pages whose max value <= 90 are pruned, returning only
2715- /// the last page (rows 91..100). With page index disabled, all 100 rows are returned
2716- /// since neither pushdown nor row-group pruning is active.
2724+ /// Test that page pruning occurs when enabled and page index metadata is available.
27172725 #[ tokio:: test]
27182726 async fn test_page_pruning_predicate_respects_enable_page_index ( ) {
2719- use parquet:: file:: properties:: WriterProperties ;
2727+ let fixture = PagePruningFixture :: new ( "test-with-page-index.parquet" )
2728+ . with_write_page_index ( true ) ;
2729+ let rows = fixture. test ( true ) . await ;
27202730
2721- let store = Arc :: new ( InMemory :: new ( ) ) as Arc < dyn ObjectStore > ;
2731+ assert_eq ! ( rows, 10 , "page index should prune 9 of 10 pages" ) ;
2732+ }
27222733
2723- // 100 rows with values 1..=100, written as a single row group with 10 rows per page
2724- let values: Vec < i32 > = ( 1 ..=100 ) . collect ( ) ;
2725- let batch = record_batch ! ( (
2726- "a" ,
2727- Int32 ,
2728- values. iter( ) . map( |v| Some ( * v) ) . collect:: <Vec <_>>( )
2729- ) )
2730- . unwrap ( ) ;
2731- let props = WriterProperties :: builder ( )
2732- . set_data_page_row_count_limit ( 10 )
2733- . set_write_batch_size ( 10 )
2734- . build ( ) ;
2735- let schema = batch. schema ( ) ;
2736- let data_size = write_parquet_batches (
2737- Arc :: clone ( & store) ,
2738- "test.parquet" ,
2739- vec ! [ batch] ,
2740- Some ( props) ,
2741- )
2742- . await ;
2734+ /// Test that page pruning does not occur when page index pruning is disabled.
2735+ #[ tokio:: test]
2736+ async fn test_page_pruning_predicate_disabled_without_option ( ) {
2737+ let fixture = PagePruningFixture :: new ( "test-page-index-disabled.parquet" )
2738+ . with_write_page_index ( true ) ;
2739+ let rows = fixture. test ( false ) . await ;
27432740
2744- let file = PartitionedFile :: new ( "test.parquet" . to_string ( ) , data_size as u64 ) ;
2741+ assert_eq ! ( rows, 100 , "without page index all rows are returned" ) ;
2742+ }
27452743
2746- // predicate: a > 90 — should allow page index to prune first 9 pages
2747- let predicate = logical2physical ( & col ( "a" ) . gt ( lit ( 90i32 ) ) , & schema) ;
2744+ /// Test that page pruning does not occur when page index metadata is unavailable.
2745+ #[ tokio:: test]
2746+ async fn test_page_pruning_predicate_disabled_without_page_index ( ) {
2747+ let fixture = PagePruningFixture :: new ( "test-without-page-index.parquet" )
2748+ . with_write_page_index ( false ) ;
2749+ let rows = fixture. test ( true ) . await ;
27482750
2749- let make_opener = |enable_page_index| {
2751+ assert_eq ! (
2752+ rows, 100 ,
2753+ "when page index metadata is unavailable all rows are returned"
2754+ ) ;
2755+ }
2756+
2757+ struct PagePruningFixture {
2758+ store : Arc < dyn ObjectStore > ,
2759+ schema : SchemaRef ,
2760+ path : String ,
2761+ predicate : Arc < dyn PhysicalExpr > ,
2762+ write_page_index : bool ,
2763+ }
2764+
2765+ impl PagePruningFixture {
2766+ // Creates a page pruning fixture with default settings.
2767+ fn new ( path : & str ) -> Self {
2768+ let store = Arc :: new ( InMemory :: new ( ) ) as Arc < dyn ObjectStore > ;
2769+ let values: Vec < i32 > = ( 1 ..=100 ) . collect ( ) ;
2770+ let batch = record_batch ! ( (
2771+ "a" ,
2772+ Int32 ,
2773+ values. iter( ) . map( |v| Some ( * v) ) . collect:: <Vec <_>>( )
2774+ ) )
2775+ . unwrap ( ) ;
2776+ let schema = batch. schema ( ) ;
2777+ let predicate = logical2physical ( & col ( "a" ) . gt ( lit ( 90i32 ) ) , & schema) ;
2778+
2779+ Self {
2780+ store,
2781+ schema,
2782+ path : path. to_string ( ) ,
2783+ predicate,
2784+ write_page_index : false ,
2785+ }
2786+ }
2787+
2788+ // Configures whether the generated parquet file should include page indexes.
2789+ fn with_write_page_index ( mut self , write_page_index : bool ) -> Self {
2790+ self . write_page_index = write_page_index;
2791+ self
2792+ }
2793+
2794+ // Writes the parquet test file and returns the matching partitioned file.
2795+ async fn create_file ( & self ) -> PartitionedFile {
2796+ use parquet:: file:: properties:: WriterProperties ;
2797+
2798+ let values: Vec < i32 > = ( 1 ..=100 ) . collect ( ) ;
2799+ let batch = record_batch ! ( (
2800+ "a" ,
2801+ Int32 ,
2802+ values. iter( ) . map( |v| Some ( * v) ) . collect:: <Vec <_>>( )
2803+ ) )
2804+ . unwrap ( ) ;
2805+ let props = WriterProperties :: builder ( )
2806+ . set_data_page_row_count_limit ( 10 )
2807+ . set_write_batch_size ( 10 )
2808+ . set_offset_index_disabled ( !self . write_page_index )
2809+ . build ( ) ;
2810+ let size = write_parquet_batches (
2811+ Arc :: clone ( & self . store ) ,
2812+ & self . path ,
2813+ vec ! [ batch] ,
2814+ Some ( props) ,
2815+ )
2816+ . await ;
2817+
2818+ PartitionedFile :: new ( self . path . clone ( ) , size as u64 )
2819+ }
2820+
2821+ // Builds a parquet opener configured so page pruning is the only pruning path.
2822+ fn opener ( & self , enable_page_index : bool ) -> ParquetOpener {
27502823 ParquetOpenerBuilder :: new ( )
2751- . with_store ( Arc :: clone ( & store) )
2752- . with_schema ( Arc :: clone ( & schema) )
2753- . with_predicate ( Arc :: clone ( & predicate) )
2824+ . with_store ( Arc :: clone ( & self . store ) )
2825+ . with_schema ( Arc :: clone ( & self . schema ) )
2826+ . with_predicate ( Arc :: clone ( & self . predicate ) )
27542827 . with_enable_page_index ( enable_page_index)
2755- // disable pushdown and row-group pruning so the only pruning path is page index
27562828 . with_pushdown_filters ( false )
27572829 . with_row_group_stats_pruning ( false )
27582830 . build ( )
2759- } ;
2760- let ( _, rows_with_page_index) = count_batches_and_rows (
2761- make_opener ( true ) . open ( file. clone ( ) ) . unwrap ( ) . await . unwrap ( ) ,
2762- )
2763- . await ;
2764- let ( _, rows_without_page_index) =
2765- count_batches_and_rows ( make_opener ( false ) . open ( file) . unwrap ( ) . await . unwrap ( ) )
2766- . await ;
2831+ }
27672832
2768- assert_eq ! (
2769- rows_with_page_index , 10 ,
2770- "page index should prune 9 of 10 pages"
2771- ) ;
2772- assert_eq ! (
2773- rows_without_page_index , 100 ,
2774- "without page index all rows are returned"
2775- ) ;
2833+ // Writes the test file, opens it, and returns only the produced row count.
2834+ async fn test ( & self , enable_page_index : bool ) -> usize {
2835+ let file = self . create_file ( ) . await ;
2836+ let opener = self . opener ( enable_page_index ) ;
2837+ let ( _ , rows ) =
2838+ count_batches_and_rows ( opener . open ( file ) . unwrap ( ) . await . unwrap ( ) ) . await ;
2839+ rows
2840+ }
27762841 }
27772842}
0 commit comments