@@ -1916,17 +1916,11 @@ impl RowGroupsPrunedParquetOpen {
19161916 // installed filter is owned by the decoder and is not
19171917 // recoverable once replaced.
19181918 let first_rg_fully_matched = rg_plan. front ( ) . is_some_and ( |e| e. fully_matched ) ;
1919- let mut row_filter_context = precomputed_context;
1920- // A rebuild at a row group boundary cannot keep a live row
1921- // selection (#24355), thus the adaptive filter placement keeps
1922- // its first decision for this file.
1923- if has_row_selection
1924- && let Some ( placement) = row_filter_context
1925- . as_mut ( )
1926- . and_then ( |ctx| ctx. placement . as_mut ( ) )
1927- {
1928- placement. freeze ( ) ;
1929- }
1919+ // The adaptive filter placement can change at a row group
1920+ // boundary also when a row selection is live: the rebuild keeps
1921+ // the selections of the remaining row groups (see
1922+ // `PushDecoderStreamState::rebuild_decoder_at_boundary`).
1923+ let row_filter_context = precomputed_context;
19301924
19311925 let mut builder = decoder_config. build ( selections, reader_metadata. clone ( ) ) ;
19321926 let mut filter_installed = false ;
@@ -5692,6 +5686,16 @@ mod test {
56925686 async fn write_file (
56935687 store : & Arc < dyn ObjectStore > ,
56945688 a : impl Fn ( usize ) -> i32 ,
5689+ ) -> ( SchemaRef , PartitionedFile ) {
5690+ write_file_with_pages ( store, a, None ) . await
5691+ }
5692+
5693+ /// Like [`write_file`], with at most `page_rows` rows in each data
5694+ /// page if it is `Some`.
5695+ async fn write_file_with_pages (
5696+ store : & Arc < dyn ObjectStore > ,
5697+ a : impl Fn ( usize ) -> i32 ,
5698+ page_rows : Option < usize > ,
56955699 ) -> ( SchemaRef , PartitionedFile ) {
56965700 let schema = Arc :: new ( Schema :: new ( vec ! [
56975701 Field :: new( "a" , DataType :: Int32 , false ) ,
@@ -5705,9 +5709,14 @@ mod test {
57055709 ] ,
57065710 )
57075711 . unwrap ( ) ;
5708- let props = WriterProperties :: builder ( )
5709- . set_max_row_group_row_count ( Some ( ROWS_PER_ROW_GROUP ) )
5710- . build ( ) ;
5712+ let mut props = WriterProperties :: builder ( )
5713+ . set_max_row_group_row_count ( Some ( ROWS_PER_ROW_GROUP ) ) ;
5714+ if let Some ( page_rows) = page_rows {
5715+ props = props
5716+ . set_data_page_row_count_limit ( page_rows)
5717+ . set_write_batch_size ( page_rows) ;
5718+ }
5719+ let props = props. build ( ) ;
57115720 let size = write_parquet_batches (
57125721 Arc :: clone ( store) ,
57135722 "placement.parquet" ,
@@ -5810,6 +5819,115 @@ mod test {
58105819 assert_eq ! ( post_scan. post_scan_rows( ) , TOTAL_ROWS ) ;
58115820 }
58125821
5822+ /// The values of `v` that a scan with the page index returns, and
5823+ /// its metrics.
5824+ async fn scan_values_with_page_index (
5825+ store : & Arc < dyn ObjectStore > ,
5826+ schema : & SchemaRef ,
5827+ file : & PartitionedFile ,
5828+ predicate : & Arc < dyn PhysicalExpr > ,
5829+ pushdown_filters : bool ,
5830+ filter_placement : bool ,
5831+ ) -> ( Vec < i64 > , ExecutionPlanMetricsSet ) {
5832+ let metrics = ExecutionPlanMetricsSet :: new ( ) ;
5833+ let morselizer = ParquetMorselizerBuilder :: new ( )
5834+ . with_store ( Arc :: clone ( store) )
5835+ . with_schema ( Arc :: clone ( schema) )
5836+ . with_predicate ( Arc :: clone ( predicate) )
5837+ . with_pushdown_filters ( pushdown_filters)
5838+ . with_filter_placement ( filter_placement)
5839+ . with_enable_page_index ( true )
5840+ . with_metrics ( metrics. clone ( ) )
5841+ . build ( ) ;
5842+ let stream = open_file ( & morselizer, file. clone ( ) ) . await . unwrap ( ) ;
5843+ let batches = stream. try_collect :: < Vec < _ > > ( ) . await . unwrap ( ) ;
5844+ let values = batches
5845+ . iter ( )
5846+ . flat_map ( |batch| {
5847+ batch
5848+ . column ( 1 )
5849+ . as_any ( )
5850+ . downcast_ref :: < Int64Array > ( )
5851+ . unwrap ( )
5852+ . values ( )
5853+ . to_vec ( )
5854+ } )
5855+ . collect ( ) ;
5856+ ( values, metrics)
5857+ }
5858+
5859+ /// The page index skips the first pages of the first row group
5860+ /// (`v >= 1000`) and pages of the second and third row groups
5861+ /// (`v < 20000 OR v >= 40000`), thus the scan has a live row
5862+ /// selection. The placement still changes at the first row group
5863+ /// boundary, the rebuilt decoder keeps the selections of the
5864+ /// remaining row groups, and the result is the same as with a row
5865+ /// filter only and with a post-scan filter only.
5866+ #[ tokio:: test]
5867+ async fn placement_changes_with_page_index_selection ( ) {
5868+ use datafusion_physical_plan:: metrics:: MetricValue ;
5869+
5870+ let store = Arc :: new ( InMemory :: new ( ) ) as Arc < dyn ObjectStore > ;
5871+ let ( schema, file) =
5872+ write_file_with_pages ( & store, |i| ( i % 2 ) as i32 , Some ( 100 ) ) . await ;
5873+ let predicate = logical2physical (
5874+ & col ( "a" ) . eq ( lit ( 0 ) ) . and ( col ( "v" ) . gt_eq ( lit ( 1000_i64 ) ) ) . and (
5875+ col ( "v" )
5876+ . lt ( lit ( 20_000_i64 ) )
5877+ . or ( col ( "v" ) . gt_eq ( lit ( 40_000_i64 ) ) ) ,
5878+ ) ,
5879+ & schema,
5880+ ) ;
5881+
5882+ let ( adaptive, adaptive_metrics) = scan_values_with_page_index (
5883+ & store, & schema, & file, & predicate, true , true ,
5884+ )
5885+ . await ;
5886+ let ( row_filter, row_filter_metrics) = scan_values_with_page_index (
5887+ & store, & schema, & file, & predicate, true , false ,
5888+ )
5889+ . await ;
5890+ let ( post_scan, post_scan_metrics) = scan_values_with_page_index (
5891+ & store, & schema, & file, & predicate, false , false ,
5892+ )
5893+ . await ;
5894+
5895+ let expected: Vec < i64 > = ( 1000 ..20_000 )
5896+ . chain ( 40_000 ..TOTAL_ROWS as i64 )
5897+ . step_by ( 2 )
5898+ . collect ( ) ;
5899+ assert_eq ! ( adaptive, expected) ;
5900+ assert_eq ! ( row_filter, expected) ;
5901+ assert_eq ! ( post_scan, expected) ;
5902+
5903+ // The page index skipped rows: the selection is not select-all.
5904+ let MetricValue :: PruningMetrics {
5905+ pruning_metrics, ..
5906+ } = adaptive_metrics
5907+ . clone_inner ( )
5908+ . sum_by_name ( "page_index_rows_pruned" )
5909+ . unwrap ( )
5910+ else {
5911+ panic ! ( "expected page pruning metrics" ) ;
5912+ } ;
5913+ // Whole pages only: a page that has a row in the result is kept.
5914+ assert ! ( pruning_metrics. pruned( ) > 20_000 ) ;
5915+ assert ! (
5916+ counter_metric_value( & adaptive_metrics, "filter_placement_changes" ) > 0
5917+ ) ;
5918+ // The rebuilt decoder fetches only the pages that the selections
5919+ // keep, as the scans that do not change the placement. A rebuild
5920+ // that dropped the selections of the remaining row groups fetches
5921+ // the pruned pages of the second and third row groups.
5922+ let bytes = |metrics : & ExecutionPlanMetricsSet | {
5923+ counter_metric_value ( metrics, "bytes_scanned" )
5924+ } ;
5925+ assert ! (
5926+ bytes( & adaptive_metrics)
5927+ <= bytes( & row_filter_metrics) . max( bytes( & post_scan_metrics) )
5928+ ) ;
5929+ }
5930+
58135931 /// A filter that removes long runs of rows saves decode time in a row
58145932 /// filter, thus it stays a row filter.
58155933 #[ tokio:: test]
0 commit comments