@@ -1947,17 +1947,11 @@ impl RowGroupsPrunedParquetOpen {
19471947 // installed filter is owned by the decoder and is not
19481948 // recoverable once replaced.
19491949 let first_rg_fully_matched = rg_plan. front ( ) . is_some_and ( |e| e. fully_matched ) ;
1950- let mut row_filter_context = precomputed_context;
1951- // A rebuild at a row group boundary cannot keep a live row
1952- // selection (#24355), thus the adaptive filter placement keeps
1953- // its first decision for this file.
1954- if has_row_selection
1955- && let Some ( placement) = row_filter_context
1956- . as_mut ( )
1957- . and_then ( |ctx| ctx. placement . as_mut ( ) )
1958- {
1959- placement. freeze ( ) ;
1960- }
1950+ // The adaptive filter placement can change at a row group
1951+ // boundary also when a row selection is live: the rebuild keeps
1952+ // the selections of the remaining row groups (see
1953+ // `PushDecoderStreamState::rebuild_decoder_at_boundary`).
1954+ let row_filter_context = precomputed_context;
19611955
19621956 let mut builder = decoder_config. build ( selections, reader_metadata. clone ( ) ) ;
19631957 let mut filter_installed = false ;
@@ -5950,6 +5944,16 @@ mod test {
59505944 async fn write_file (
59515945 store : & Arc < dyn ObjectStore > ,
59525946 a : impl Fn ( usize ) -> i32 ,
5947+ ) -> ( SchemaRef , PartitionedFile ) {
5948+ write_file_with_pages ( store, a, None ) . await
5949+ }
5950+
5951+ /// Like [`write_file`], with at most `page_rows` rows in each data
5952+ /// page if it is `Some`.
5953+ async fn write_file_with_pages (
5954+ store : & Arc < dyn ObjectStore > ,
5955+ a : impl Fn ( usize ) -> i32 ,
5956+ page_rows : Option < usize > ,
59535957 ) -> ( SchemaRef , PartitionedFile ) {
59545958 let schema = Arc :: new ( Schema :: new ( vec ! [
59555959 Field :: new( "a" , DataType :: Int32 , false ) ,
@@ -5963,9 +5967,14 @@ mod test {
59635967 ] ,
59645968 )
59655969 . unwrap ( ) ;
5966- let props = WriterProperties :: builder ( )
5967- . set_max_row_group_row_count ( Some ( ROWS_PER_ROW_GROUP ) )
5968- . build ( ) ;
5970+ let mut props = WriterProperties :: builder ( )
5971+ . set_max_row_group_row_count ( Some ( ROWS_PER_ROW_GROUP ) ) ;
5972+ if let Some ( page_rows) = page_rows {
5973+ props = props
5974+ . set_data_page_row_count_limit ( page_rows)
5975+ . set_write_batch_size ( page_rows) ;
5976+ }
5977+ let props = props. build ( ) ;
59695978 let size = write_parquet_batches (
59705979 Arc :: clone ( store) ,
59715980 "placement.parquet" ,
@@ -6068,6 +6077,115 @@ mod test {
60686077 assert_eq ! ( post_scan. post_scan_rows( ) , TOTAL_ROWS ) ;
60696078 }
60706079
6080+ /// The values of `v` that a scan with the page index returns, and
6081+ /// its metrics.
6082+ async fn scan_values_with_page_index (
6083+ store : & Arc < dyn ObjectStore > ,
6084+ schema : & SchemaRef ,
6085+ file : & PartitionedFile ,
6086+ predicate : & Arc < dyn PhysicalExpr > ,
6087+ pushdown_filters : bool ,
6088+ filter_placement : bool ,
6089+ ) -> ( Vec < i64 > , ExecutionPlanMetricsSet ) {
6090+ let metrics = ExecutionPlanMetricsSet :: new ( ) ;
6091+ let morselizer = ParquetMorselizerBuilder :: new ( )
6092+ . with_store ( Arc :: clone ( store) )
6093+ . with_schema ( Arc :: clone ( schema) )
6094+ . with_predicate ( Arc :: clone ( predicate) )
6095+ . with_pushdown_filters ( pushdown_filters)
6096+ . with_filter_placement ( filter_placement)
6097+ . with_enable_page_index ( true )
6098+ . with_metrics ( metrics. clone ( ) )
6099+ . build ( ) ;
6100+ let stream = open_file ( & morselizer, file. clone ( ) ) . await . unwrap ( ) ;
6101+ let batches = stream. try_collect :: < Vec < _ > > ( ) . await . unwrap ( ) ;
6102+ let values = batches
6103+ . iter ( )
6104+ . flat_map ( |batch| {
6105+ batch
6106+ . column ( 1 )
6107+ . as_any ( )
6108+ . downcast_ref :: < Int64Array > ( )
6109+ . unwrap ( )
6110+ . values ( )
6111+ . to_vec ( )
6112+ } )
6113+ . collect ( ) ;
6114+ ( values, metrics)
6115+ }
6116+
6117+ /// The page index skips the first pages of the first row group
6118+ /// (`v >= 1000`) and pages of the second and third row groups
6119+ /// (`v < 20000 OR v >= 40000`), thus the scan has a live row
6120+ /// selection. The placement still changes at the first row group
6121+ /// boundary, the rebuilt decoder keeps the selections of the
6122+ /// remaining row groups, and the result is the same as with a row
6123+ /// filter only and with a post-scan filter only.
6124+ #[ tokio:: test]
6125+ async fn placement_changes_with_page_index_selection ( ) {
6126+ use datafusion_physical_plan:: metrics:: MetricValue ;
6127+
6128+ let store = Arc :: new ( InMemory :: new ( ) ) as Arc < dyn ObjectStore > ;
6129+ let ( schema, file) =
6130+ write_file_with_pages ( & store, |i| ( i % 2 ) as i32 , Some ( 100 ) ) . await ;
6131+ let predicate = logical2physical (
6132+ & col ( "a" ) . eq ( lit ( 0 ) ) . and ( col ( "v" ) . gt_eq ( lit ( 1000_i64 ) ) ) . and (
6133+ col ( "v" )
6134+ . lt ( lit ( 20_000_i64 ) )
6135+ . or ( col ( "v" ) . gt_eq ( lit ( 40_000_i64 ) ) ) ,
6136+ ) ,
6137+ & schema,
6138+ ) ;
6139+
6140+ let ( adaptive, adaptive_metrics) = scan_values_with_page_index (
6141+ & store, & schema, & file, & predicate, true , true ,
6142+ )
6143+ . await ;
6144+ let ( row_filter, row_filter_metrics) = scan_values_with_page_index (
6145+ & store, & schema, & file, & predicate, true , false ,
6146+ )
6147+ . await ;
6148+ let ( post_scan, post_scan_metrics) = scan_values_with_page_index (
6149+ & store, & schema, & file, & predicate, false , false ,
6150+ )
6151+ . await ;
6152+
6153+ let expected: Vec < i64 > = ( 1000 ..20_000 )
6154+ . chain ( 40_000 ..TOTAL_ROWS as i64 )
6155+ . step_by ( 2 )
6156+ . collect ( ) ;
6157+ assert_eq ! ( adaptive, expected) ;
6158+ assert_eq ! ( row_filter, expected) ;
6159+ assert_eq ! ( post_scan, expected) ;
6160+
6161+ // The page index skipped rows: the selection is not select-all.
6162+ let MetricValue :: PruningMetrics {
6163+ pruning_metrics, ..
6164+ } = adaptive_metrics
6165+ . clone_inner ( )
6166+ . sum_by_name ( "page_index_rows_pruned" )
6167+ . unwrap ( )
6168+ else {
6169+ panic ! ( "expected page pruning metrics" ) ;
6170+ } ;
6171+ // Whole pages only: a page that has a row in the result is kept.
6172+ assert ! ( pruning_metrics. pruned( ) > 20_000 ) ;
6173+ assert ! (
6174+ counter_metric_value( & adaptive_metrics, "filter_placement_changes" ) > 0
6175+ ) ;
6176+ // The rebuilt decoder fetches only the pages that the selections
6177+ // keep, as the scans that do not change the placement. A rebuild
6178+ // that dropped the selections of the remaining row groups fetches
6179+ // the pruned pages of the second and third row groups.
6180+ let bytes = |metrics : & ExecutionPlanMetricsSet | {
6181+ counter_metric_value ( metrics, "bytes_scanned" )
6182+ } ;
6183+ assert ! (
6184+ bytes( & adaptive_metrics)
6185+ <= bytes( & row_filter_metrics) . max( bytes( & post_scan_metrics) )
6186+ ) ;
6187+ }
6188+
60716189 /// A filter that removes long runs of rows saves decode time in a row
60726190 /// filter, thus it stays a row filter.
60736191 #[ tokio:: test]
0 commit comments