@@ -1927,17 +1927,11 @@ impl RowGroupsPrunedParquetOpen {
19271927 // installed filter is owned by the decoder and is not
19281928 // recoverable once replaced.
19291929 let first_rg_fully_matched = rg_plan. front ( ) . is_some_and ( |e| e. fully_matched ) ;
1930- let mut row_filter_context = precomputed_context;
1931- // A rebuild at a row group boundary cannot keep a live row
1932- // selection (#24355), thus the adaptive filter placement keeps
1933- // its first decision for this file.
1934- if has_row_selection
1935- && let Some ( placement) = row_filter_context
1936- . as_mut ( )
1937- . and_then ( |ctx| ctx. placement . as_mut ( ) )
1938- {
1939- placement. freeze ( ) ;
1940- }
1930+ // The adaptive filter placement can change at a row group
1931+ // boundary also when a row selection is live: the rebuild keeps
1932+ // the selections of the remaining row groups (see
1933+ // `PushDecoderStreamState::rebuild_decoder_at_boundary`).
1934+ let row_filter_context = precomputed_context;
19411935
19421936 let mut builder = decoder_config. build ( selections, reader_metadata. clone ( ) ) ;
19431937 let mut filter_installed = false ;
@@ -5704,6 +5698,16 @@ mod test {
57045698 async fn write_file (
57055699 store : & Arc < dyn ObjectStore > ,
57065700 a : impl Fn ( usize ) -> i32 ,
5701+ ) -> ( SchemaRef , PartitionedFile ) {
5702+ write_file_with_pages ( store, a, None ) . await
5703+ }
5704+
5705+ /// Like [`write_file`], with at most `page_rows` rows in each data
5706+ /// page if it is `Some`.
5707+ async fn write_file_with_pages (
5708+ store : & Arc < dyn ObjectStore > ,
5709+ a : impl Fn ( usize ) -> i32 ,
5710+ page_rows : Option < usize > ,
57075711 ) -> ( SchemaRef , PartitionedFile ) {
57085712 let schema = Arc :: new ( Schema :: new ( vec ! [
57095713 Field :: new( "a" , DataType :: Int32 , false ) ,
@@ -5717,9 +5721,14 @@ mod test {
57175721 ] ,
57185722 )
57195723 . unwrap ( ) ;
5720- let props = WriterProperties :: builder ( )
5721- . set_max_row_group_row_count ( Some ( ROWS_PER_ROW_GROUP ) )
5722- . build ( ) ;
5724+ let mut props = WriterProperties :: builder ( )
5725+ . set_max_row_group_row_count ( Some ( ROWS_PER_ROW_GROUP ) ) ;
5726+ if let Some ( page_rows) = page_rows {
5727+ props = props
5728+ . set_data_page_row_count_limit ( page_rows)
5729+ . set_write_batch_size ( page_rows) ;
5730+ }
5731+ let props = props. build ( ) ;
57235732 let size = write_parquet_batches (
57245733 Arc :: clone ( store) ,
57255734 "placement.parquet" ,
@@ -5822,6 +5831,115 @@ mod test {
58225831 assert_eq ! ( post_scan. post_scan_rows( ) , TOTAL_ROWS ) ;
58235832 }
58245833
5834+ /// The values of `v` that a scan with the page index returns, and
5835+ /// its metrics.
5836+ async fn scan_values_with_page_index (
5837+ store : & Arc < dyn ObjectStore > ,
5838+ schema : & SchemaRef ,
5839+ file : & PartitionedFile ,
5840+ predicate : & Arc < dyn PhysicalExpr > ,
5841+ pushdown_filters : bool ,
5842+ filter_placement : bool ,
5843+ ) -> ( Vec < i64 > , ExecutionPlanMetricsSet ) {
5844+ let metrics = ExecutionPlanMetricsSet :: new ( ) ;
5845+ let morselizer = ParquetMorselizerBuilder :: new ( )
5846+ . with_store ( Arc :: clone ( store) )
5847+ . with_schema ( Arc :: clone ( schema) )
5848+ . with_predicate ( Arc :: clone ( predicate) )
5849+ . with_pushdown_filters ( pushdown_filters)
5850+ . with_filter_placement ( filter_placement)
5851+ . with_enable_page_index ( true )
5852+ . with_metrics ( metrics. clone ( ) )
5853+ . build ( ) ;
5854+ let stream = open_file ( & morselizer, file. clone ( ) ) . await . unwrap ( ) ;
5855+ let batches = stream. try_collect :: < Vec < _ > > ( ) . await . unwrap ( ) ;
5856+ let values = batches
5857+ . iter ( )
5858+ . flat_map ( |batch| {
5859+ batch
5860+ . column ( 1 )
5861+ . as_any ( )
5862+ . downcast_ref :: < Int64Array > ( )
5863+ . unwrap ( )
5864+ . values ( )
5865+ . to_vec ( )
5866+ } )
5867+ . collect ( ) ;
5868+ ( values, metrics)
5869+ }
5870+
5871+ /// The page index skips the first pages of the first row group
5872+ /// (`v >= 1000`) and pages of the second and third row groups
5873+ /// (`v < 20000 OR v >= 40000`), thus the scan has a live row
5874+ /// selection. The placement still changes at the first row group
5875+ /// boundary, the rebuilt decoder keeps the selections of the
5876+ /// remaining row groups, and the result is the same as with a row
5877+ /// filter only and with a post-scan filter only.
5878+ #[ tokio:: test]
5879+ async fn placement_changes_with_page_index_selection ( ) {
5880+ use datafusion_physical_plan:: metrics:: MetricValue ;
5881+
5882+ let store = Arc :: new ( InMemory :: new ( ) ) as Arc < dyn ObjectStore > ;
5883+ let ( schema, file) =
5884+ write_file_with_pages ( & store, |i| ( i % 2 ) as i32 , Some ( 100 ) ) . await ;
5885+ let predicate = logical2physical (
5886+ & col ( "a" ) . eq ( lit ( 0 ) ) . and ( col ( "v" ) . gt_eq ( lit ( 1000_i64 ) ) ) . and (
5887+ col ( "v" )
5888+ . lt ( lit ( 20_000_i64 ) )
5889+ . or ( col ( "v" ) . gt_eq ( lit ( 40_000_i64 ) ) ) ,
5890+ ) ,
5891+ & schema,
5892+ ) ;
5893+
5894+ let ( adaptive, adaptive_metrics) = scan_values_with_page_index (
5895+ & store, & schema, & file, & predicate, true , true ,
5896+ )
5897+ . await ;
5898+ let ( row_filter, row_filter_metrics) = scan_values_with_page_index (
5899+ & store, & schema, & file, & predicate, true , false ,
5900+ )
5901+ . await ;
5902+ let ( post_scan, post_scan_metrics) = scan_values_with_page_index (
5903+ & store, & schema, & file, & predicate, false , false ,
5904+ )
5905+ . await ;
5906+
5907+ let expected: Vec < i64 > = ( 1000 ..20_000 )
5908+ . chain ( 40_000 ..TOTAL_ROWS as i64 )
5909+ . step_by ( 2 )
5910+ . collect ( ) ;
5911+ assert_eq ! ( adaptive, expected) ;
5912+ assert_eq ! ( row_filter, expected) ;
5913+ assert_eq ! ( post_scan, expected) ;
5914+
5915+ // The page index skipped rows: the selection is not select-all.
5916+ let MetricValue :: PruningMetrics {
5917+ pruning_metrics, ..
5918+ } = adaptive_metrics
5919+ . clone_inner ( )
5920+ . sum_by_name ( "page_index_rows_pruned" )
5921+ . unwrap ( )
5922+ else {
5923+ panic ! ( "expected page pruning metrics" ) ;
5924+ } ;
5925+ // Whole pages only: a page that has a row in the result is kept.
5926+ assert ! ( pruning_metrics. pruned( ) > 20_000 ) ;
5927+ assert ! (
5928+ counter_metric_value( & adaptive_metrics, "filter_placement_changes" ) > 0
5929+ ) ;
5930+ // The rebuilt decoder fetches only the pages that the selections
5931+ // keep, as the scans that do not change the placement. A rebuild
5932+ // that dropped the selections of the remaining row groups fetches
5933+ // the pruned pages of the second and third row groups.
5934+ let bytes = |metrics : & ExecutionPlanMetricsSet | {
5935+ counter_metric_value ( metrics, "bytes_scanned" )
5936+ } ;
5937+ assert ! (
5938+ bytes( & adaptive_metrics)
5939+ <= bytes( & row_filter_metrics) . max( bytes( & post_scan_metrics) )
5940+ ) ;
5941+ }
5942+
58255943 /// A filter that removes long runs of rows saves decode time in a row
58265944 /// filter, thus it stays a row filter.
58275945 #[ tokio:: test]
0 commit comments