@@ -1933,17 +1933,11 @@ impl RowGroupsPrunedParquetOpen {
19331933 // installed filter is owned by the decoder and is not
19341934 // recoverable once replaced.
19351935 let first_rg_fully_matched = rg_plan. front ( ) . is_some_and ( |e| e. fully_matched ) ;
1936- let mut row_filter_context = precomputed_context;
1937- // A rebuild at a row group boundary cannot keep a live row
1938- // selection (#24355), thus the adaptive filter placement keeps
1939- // its first decision for this file.
1940- if has_row_selection
1941- && let Some ( placement) = row_filter_context
1942- . as_mut ( )
1943- . and_then ( |ctx| ctx. placement . as_mut ( ) )
1944- {
1945- placement. freeze ( ) ;
1946- }
1936+ // The adaptive filter placement can change at a row group
1937+ // boundary also when a row selection is live: the rebuild keeps
1938+ // the selections of the remaining row groups (see
1939+ // `PushDecoderStreamState::rebuild_decoder_at_boundary`).
1940+ let row_filter_context = precomputed_context;
19471941
19481942 let mut builder = decoder_config. build ( selections, reader_metadata. clone ( ) ) ;
19491943 let mut filter_installed = false ;
@@ -5715,6 +5709,16 @@ mod test {
57155709 async fn write_file (
57165710 store : & Arc < dyn ObjectStore > ,
57175711 a : impl Fn ( usize ) -> i32 ,
5712+ ) -> ( SchemaRef , PartitionedFile ) {
5713+ write_file_with_pages ( store, a, None ) . await
5714+ }
5715+
5716+ /// Like [`write_file`], with at most `page_rows` rows in each data
5717+ /// page if it is `Some`.
5718+ async fn write_file_with_pages (
5719+ store : & Arc < dyn ObjectStore > ,
5720+ a : impl Fn ( usize ) -> i32 ,
5721+ page_rows : Option < usize > ,
57185722 ) -> ( SchemaRef , PartitionedFile ) {
57195723 let schema = Arc :: new ( Schema :: new ( vec ! [
57205724 Field :: new( "a" , DataType :: Int32 , false ) ,
@@ -5728,9 +5732,14 @@ mod test {
57285732 ] ,
57295733 )
57305734 . unwrap ( ) ;
5731- let props = WriterProperties :: builder ( )
5732- . set_max_row_group_row_count ( Some ( ROWS_PER_ROW_GROUP ) )
5733- . build ( ) ;
5735+ let mut props = WriterProperties :: builder ( )
5736+ . set_max_row_group_row_count ( Some ( ROWS_PER_ROW_GROUP ) ) ;
5737+ if let Some ( page_rows) = page_rows {
5738+ props = props
5739+ . set_data_page_row_count_limit ( page_rows)
5740+ . set_write_batch_size ( page_rows) ;
5741+ }
5742+ let props = props. build ( ) ;
57345743 let size = write_parquet_batches (
57355744 Arc :: clone ( store) ,
57365745 "placement.parquet" ,
@@ -5833,6 +5842,115 @@ mod test {
58335842 assert_eq ! ( post_scan. post_scan_rows( ) , TOTAL_ROWS ) ;
58345843 }
58355844
5845+ /// The values of `v` that a scan with the page index returns, and
5846+ /// its metrics.
5847+ async fn scan_values_with_page_index (
5848+ store : & Arc < dyn ObjectStore > ,
5849+ schema : & SchemaRef ,
5850+ file : & PartitionedFile ,
5851+ predicate : & Arc < dyn PhysicalExpr > ,
5852+ pushdown_filters : bool ,
5853+ filter_placement : bool ,
5854+ ) -> ( Vec < i64 > , ExecutionPlanMetricsSet ) {
5855+ let metrics = ExecutionPlanMetricsSet :: new ( ) ;
5856+ let morselizer = ParquetMorselizerBuilder :: new ( )
5857+ . with_store ( Arc :: clone ( store) )
5858+ . with_schema ( Arc :: clone ( schema) )
5859+ . with_predicate ( Arc :: clone ( predicate) )
5860+ . with_pushdown_filters ( pushdown_filters)
5861+ . with_filter_placement ( filter_placement)
5862+ . with_enable_page_index ( true )
5863+ . with_metrics ( metrics. clone ( ) )
5864+ . build ( ) ;
5865+ let stream = open_file ( & morselizer, file. clone ( ) ) . await . unwrap ( ) ;
5866+ let batches = stream. try_collect :: < Vec < _ > > ( ) . await . unwrap ( ) ;
5867+ let values = batches
5868+ . iter ( )
5869+ . flat_map ( |batch| {
5870+ batch
5871+ . column ( 1 )
5872+ . as_any ( )
5873+ . downcast_ref :: < Int64Array > ( )
5874+ . unwrap ( )
5875+ . values ( )
5876+ . to_vec ( )
5877+ } )
5878+ . collect ( ) ;
5879+ ( values, metrics)
5880+ }
5881+
5882+ /// The page index skips the first pages of the first row group
5883+ /// (`v >= 1000`) and pages of the second and third row groups
5884+ /// (`v < 20000 OR v >= 40000`), thus the scan has a live row
5885+ /// selection. The placement still changes at the first row group
5886+ /// boundary, the rebuilt decoder keeps the selections of the
5887+ /// remaining row groups, and the result is the same as with a row
5888+ /// filter only and with a post-scan filter only.
5889+ #[ tokio:: test]
5890+ async fn placement_changes_with_page_index_selection ( ) {
5891+ use datafusion_physical_plan:: metrics:: MetricValue ;
5892+
5893+ let store = Arc :: new ( InMemory :: new ( ) ) as Arc < dyn ObjectStore > ;
5894+ let ( schema, file) =
5895+ write_file_with_pages ( & store, |i| ( i % 2 ) as i32 , Some ( 100 ) ) . await ;
5896+ let predicate = logical2physical (
5897+ & col ( "a" ) . eq ( lit ( 0 ) ) . and ( col ( "v" ) . gt_eq ( lit ( 1000_i64 ) ) ) . and (
5898+ col ( "v" )
5899+ . lt ( lit ( 20_000_i64 ) )
5900+ . or ( col ( "v" ) . gt_eq ( lit ( 40_000_i64 ) ) ) ,
5901+ ) ,
5902+ & schema,
5903+ ) ;
5904+
5905+ let ( adaptive, adaptive_metrics) = scan_values_with_page_index (
5906+ & store, & schema, & file, & predicate, true , true ,
5907+ )
5908+ . await ;
5909+ let ( row_filter, row_filter_metrics) = scan_values_with_page_index (
5910+ & store, & schema, & file, & predicate, true , false ,
5911+ )
5912+ . await ;
5913+ let ( post_scan, post_scan_metrics) = scan_values_with_page_index (
5914+ & store, & schema, & file, & predicate, false , false ,
5915+ )
5916+ . await ;
5917+
5918+ let expected: Vec < i64 > = ( 1000 ..20_000 )
5919+ . chain ( 40_000 ..TOTAL_ROWS as i64 )
5920+ . step_by ( 2 )
5921+ . collect ( ) ;
5922+ assert_eq ! ( adaptive, expected) ;
5923+ assert_eq ! ( row_filter, expected) ;
5924+ assert_eq ! ( post_scan, expected) ;
5925+
5926+ // The page index skipped rows: the selection is not select-all.
5927+ let MetricValue :: PruningMetrics {
5928+ pruning_metrics, ..
5929+ } = adaptive_metrics
5930+ . clone_inner ( )
5931+ . sum_by_name ( "page_index_rows_pruned" )
5932+ . unwrap ( )
5933+ else {
5934+ panic ! ( "expected page pruning metrics" ) ;
5935+ } ;
5936+ // Whole pages only: a page that has a row in the result is kept.
5937+ assert ! ( pruning_metrics. pruned( ) > 20_000 ) ;
5938+ assert ! (
5939+ counter_metric_value( & adaptive_metrics, "filter_placement_changes" ) > 0
5940+ ) ;
5941+ // The rebuilt decoder fetches only the pages that the selections
5942+ // keep, as the scans that do not change the placement. A rebuild
5943+ // that dropped the selections of the remaining row groups fetches
5944+ // the pruned pages of the second and third row groups.
5945+ let bytes = |metrics : & ExecutionPlanMetricsSet | {
5946+ counter_metric_value ( metrics, "bytes_scanned" )
5947+ } ;
5948+ assert ! (
5949+ bytes( & adaptive_metrics)
5950+ <= bytes( & row_filter_metrics) . max( bytes( & post_scan_metrics) )
5951+ ) ;
5952+ }
5953+
58365954 /// A filter that removes long runs of rows saves decode time in a row
58375955 /// filter, thus it stays a row filter.
58385956 #[ tokio:: test]
0 commit comments