@@ -1019,6 +1019,181 @@ async fn query_single_parquet_file_multi_row_groups_multiple_predicates() {
10191019 ) ;
10201020}
10211021
1022+ #[ tokio:: test]
1023+ async fn query_single_parquet_file_pushdown_filters_one_shot_io ( ) {
1024+ let test = Test :: new ( ) . with_single_file_parquet ( ) . await ;
1025+ test. set ( "datafusion.execution.parquet.pushdown_filters" , "true" )
1026+ . await ;
1027+
1028+ // With filter pushdown enabled and `progressive_io` disabled (the
1029+ // default), all data needed for each row group (the columns needed by
1030+ // the filter and by the projection) is fetched with a single request per
1031+ // row group, the same I/O pattern as when pushdown_filters is disabled
1032+ // (compare `query_single_parquet_file_multi_row_groups_multiple_predicates`):
1033+ //
1034+ // * Row group 1 (`a` in 0..100): `a` (4-534) and `b` (534-1064) are
1035+ // fetched together, even though `a` is only needed to evaluate the
1036+ // filter.
1037+ // * Row group 2 (`a` in 100..200): the statistics prove every row
1038+ // matches, so the filter is skipped and only `b` (1594-2124) is read.
1039+ assert_snapshot ! (
1040+ test. query( "select min(b) from parquet_table WHERE a > 50" ) . await ,
1041+ @r"
1042+ ------- Query Output (1 rows) -------
1043+ +----------------------+
1044+ | min(parquet_table.b) |
1045+ +----------------------+
1046+ | 1051 |
1047+ +----------------------+
1048+ ------- Object Store Request Summary -------
1049+ RequestCountingObjectStore()
1050+ Total Requests: 3
1051+ - GET (opts) path=parquet_table.parquet head=true
1052+ - GET (ranges) path=parquet_table.parquet ranges=4-534,534-1064
1053+ - GET (ranges) path=parquet_table.parquet ranges=1594-2124
1054+ "
1055+ ) ;
1056+ }
1057+
1058+ #[ tokio:: test]
1059+ async fn query_single_parquet_file_pushdown_filters_progressive_io ( ) {
1060+ let test = Test :: new ( ) . with_single_file_parquet ( ) . await ;
1061+ test. set ( "datafusion.execution.parquet.pushdown_filters" , "true" )
1062+ . await ;
1063+ test. set ( "datafusion.execution.parquet.progressive_io" , "true" )
1064+ . await ;
1065+
1066+ // With `progressive_io` enabled (and an offset index present in the
1067+ // file), a row group that needs the filter is fetched progressively:
1068+ // first the column needed to evaluate the filter (`a`, 4-534), then the
1069+ // remaining projected column (`b`, 534-1064) for the rows that passed,
1070+ // resulting in multiple requests for row group 1. Row group 2 is fully
1071+ // matched by its statistics, so the filter is skipped and only `b` is
1072+ // read, as in the one-shot case.
1073+ assert_snapshot ! (
1074+ test. query( "select min(b) from parquet_table WHERE a > 50" ) . await ,
1075+ @r"
1076+ ------- Query Output (1 rows) -------
1077+ +----------------------+
1078+ | min(parquet_table.b) |
1079+ +----------------------+
1080+ | 1051 |
1081+ +----------------------+
1082+ ------- Object Store Request Summary -------
1083+ RequestCountingObjectStore()
1084+ Total Requests: 4
1085+ - GET (opts) path=parquet_table.parquet head=true
1086+ - GET (ranges) path=parquet_table.parquet ranges=4-534
1087+ - GET (ranges) path=parquet_table.parquet ranges=534-951,951-1064
1088+ - GET (ranges) path=parquet_table.parquet ranges=1594-2124
1089+ "
1090+ ) ;
1091+ }
1092+
1093+ #[ tokio:: test]
1094+ async fn query_single_parquet_file_multi_page_no_pushdown ( ) {
1095+ let test = Test :: new ( ) . with_single_file_parquet_multi_page ( ) . await ;
1096+
1097+ // Baseline for the `*_page_index` tests below: without filter pushdown
1098+ // the page index still prunes the first two pages of `a` in row group 1
1099+ // (values 0..40 cannot match `a > 50`) and the matching rows of `b`, so
1100+ // only the dictionary page and the selected data pages of each column
1101+ // chunk are fetched, with a single request per row group.
1102+ assert_snapshot ! (
1103+ test. query( "select min(b) from parquet_table WHERE a > 50" ) . await ,
1104+ @r"
1105+ ------- Query Output (1 rows) -------
1106+ +----------------------+
1107+ | min(parquet_table.b) |
1108+ +----------------------+
1109+ | 1051 |
1110+ +----------------------+
1111+ ------- Object Store Request Summary -------
1112+ RequestCountingObjectStore()
1113+ Total Requests: 3
1114+ - GET (opts) path=parquet_table.parquet head=true
1115+ - GET (ranges) path=parquet_table.parquet ranges=4-421,492-529,529-569,569-609,609-1026,1097-1134,1134-1174,1174-1214
1116+ - GET (ranges) path=parquet_table.parquet ranges=1214-1819,1819-2424
1117+ "
1118+ ) ;
1119+ }
1120+
1121+ #[ tokio:: test]
1122+ async fn query_single_parquet_file_pushdown_filters_one_shot_io_page_index ( ) {
1123+ let test = Test :: new ( ) . with_single_file_parquet_multi_page ( ) . await ;
1124+ test. set ( "datafusion.execution.parquet.pushdown_filters" , "true" )
1125+ . await ;
1126+
1127+ // Each row group has five 20-row data pages per column. The page index
1128+ // proves the first two pages of `a` in row group 1 (values 0..40) cannot
1129+ // match `a > 50`, so the resulting row selection skips them. With
1130+ // one-shot I/O the data is still fetched with a single request per row
1131+ // group, but only the pages the decoder will read:
1132+ //
1133+ // * `a` (needed by the filter) becomes two ranges: its dictionary page
1134+ // (4-421) and the run of selected data pages (492-609); the two pruned
1135+ // pages (421-492) are not fetched.
1136+ // * `b` (the output column) is fetched as one range (609-1214). Note the
1137+ // decoder reads every page of `b` once a row filter is installed, even
1138+ // though the first two pages hold no selected rows (compare
1139+ // `query_single_parquet_file_multi_page_no_pushdown`, which skips
1140+ // 1026-1097). The plan reproduces the decoder's own fetch, so one-shot
1141+ // I/O reads the same bytes as progressive I/O does.
1142+ // * Row group 2 is fully matched by its statistics, so the filter is
1143+ // skipped and only `b` (1819-2424) is read.
1144+ assert_snapshot ! (
1145+ test. query( "select min(b) from parquet_table WHERE a > 50" ) . await ,
1146+ @r"
1147+ ------- Query Output (1 rows) -------
1148+ +----------------------+
1149+ | min(parquet_table.b) |
1150+ +----------------------+
1151+ | 1051 |
1152+ +----------------------+
1153+ ------- Object Store Request Summary -------
1154+ RequestCountingObjectStore()
1155+ Total Requests: 3
1156+ - GET (opts) path=parquet_table.parquet head=true
1157+ - GET (ranges) path=parquet_table.parquet ranges=4-421,492-609,609-1214
1158+ - GET (ranges) path=parquet_table.parquet ranges=1819-2424
1159+ "
1160+ ) ;
1161+ }
1162+
1163+ #[ tokio:: test]
1164+ async fn query_single_parquet_file_pushdown_filters_progressive_io_page_index ( ) {
1165+ let test = Test :: new ( ) . with_single_file_parquet_multi_page ( ) . await ;
1166+ test. set ( "datafusion.execution.parquet.pushdown_filters" , "true" )
1167+ . await ;
1168+ test. set ( "datafusion.execution.parquet.progressive_io" , "true" )
1169+ . await ;
1170+
1171+ // Same file and query as
1172+ // `query_single_parquet_file_pushdown_filters_one_shot_io_page_index`,
1173+ // but fetched progressively: the dictionary and selected pages of `a`
1174+ // first, then (after the filter is evaluated) the dictionary and data
1175+ // pages of `b`. The bytes read are the same as with one-shot I/O; only
1176+ // the number of requests differs.
1177+ assert_snapshot ! (
1178+ test. query( "select min(b) from parquet_table WHERE a > 50" ) . await ,
1179+ @r"
1180+ ------- Query Output (1 rows) -------
1181+ +----------------------+
1182+ | min(parquet_table.b) |
1183+ +----------------------+
1184+ | 1051 |
1185+ +----------------------+
1186+ ------- Object Store Request Summary -------
1187+ RequestCountingObjectStore()
1188+ Total Requests: 4
1189+ - GET (opts) path=parquet_table.parquet head=true
1190+ - GET (ranges) path=parquet_table.parquet ranges=4-421,492-529,529-569,569-609
1191+ - GET (ranges) path=parquet_table.parquet ranges=609-1026,1026-1060,1060-1097,1097-1134,1134-1174,1174-1214
1192+ - GET (ranges) path=parquet_table.parquet ranges=1819-2424
1193+ "
1194+ ) ;
1195+ }
1196+
10221197/// Runs tests with a request counting object store
10231198struct Test {
10241199 object_store : Arc < RequestCountingObjectStore > ,
@@ -1060,6 +1235,17 @@ impl Test {
10601235 format ! ( "{}" , self . object_store)
10611236 }
10621237
1238+ /// Set a session configuration value via SQL `SET`
1239+ async fn set ( & self , key : & str , value : & str ) {
1240+ self . session_context
1241+ . sql ( & format ! ( "set {key} = {value}" ) )
1242+ . await
1243+ . unwrap ( )
1244+ . collect ( )
1245+ . await
1246+ . unwrap ( ) ;
1247+ }
1248+
10631249 /// Store the specified bytes at the given path
10641250 async fn with_bytes ( self , path : & str , bytes : impl Into < Bytes > ) -> Self {
10651251 let path = Path :: from ( path) ;
@@ -1320,6 +1506,35 @@ impl Test {
13201506 . await
13211507 }
13221508
1509+ /// Like [`Self::with_single_file_parquet`], but each column chunk is
1510+ /// written as five 20-row data pages (plus a dictionary page) so that
1511+ /// the page index can prune individual pages within a row group.
1512+ async fn with_single_file_parquet_multi_page ( self ) -> Test {
1513+ let a: ArrayRef = Arc :: new ( Int32Array :: from_iter_values ( 0 ..200 ) ) ;
1514+ let b: ArrayRef = Arc :: new ( Int32Array :: from_iter_values ( 1000 ..1200 ) ) ;
1515+ let batch = RecordBatch :: try_from_iter ( [ ( "a" , a) , ( "b" , b) ] ) . unwrap ( ) ;
1516+
1517+ let mut buffer = vec ! [ ] ;
1518+ let props = parquet:: file:: properties:: WriterProperties :: builder ( )
1519+ . set_max_row_group_row_count ( Some ( 100 ) )
1520+ . set_data_page_row_count_limit ( 20 )
1521+ . set_write_batch_size ( 20 )
1522+ . build ( ) ;
1523+ let mut writer = parquet:: arrow:: ArrowWriter :: try_new (
1524+ & mut buffer,
1525+ batch. schema ( ) ,
1526+ Some ( props) ,
1527+ )
1528+ . unwrap ( ) ;
1529+ writer. write ( & batch) . unwrap ( ) ;
1530+ writer. close ( ) . unwrap ( ) ;
1531+
1532+ self . with_bytes ( "/parquet_table.parquet" , buffer)
1533+ . await
1534+ . register_parquet ( "parquet_table" , "/parquet_table.parquet" )
1535+ . await
1536+ }
1537+
13231538 /// Runs the specified query and returns a string representation of the results
13241539 /// suitable for comparison with insta snapshots
13251540 ///
0 commit comments