Skip to content

Commit 648b488

Browse files
alambclaude
andcommitted
Add parquet progressive_io option: one-shot row group I/O with pushdown filters via ParquetPushDecoder::scan_plan
Co-Authored-By: Claude Fable 5.1 <[email protected]>
1 parent 9586aaf commit 648b488

16 files changed

Lines changed: 458 additions & 1 deletion

File tree

‎datafusion/common/src/config.rs‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1383,6 +1383,19 @@ config_namespace! {
13831383
/// pattern of selected rows.
13841384
pub force_filter_selections: bool, default = false
13851385

1386+
/// (reading) Controls the I/O pattern used when `pushdown_filters` is
1387+
/// enabled. If false (the default), all data pages needed to read a row
1388+
/// group (for both filter evaluation and the output projection) are
1389+
/// fetched in a single request, the same I/O pattern used when
1390+
/// `pushdown_filters` is disabled. If true, data is fetched
1391+
/// progressively: first the columns needed by each filter, then, after
1392+
/// the filters are evaluated, the remaining projected columns for the
1393+
/// rows that passed. Progressive fetching can reduce the total bytes
1394+
/// read when the file has a Parquet offset index, at the cost of
1395+
/// additional I/O requests per row group; files without an offset index
1396+
/// are always read with a single request per row group.
1397+
pub progressive_io: bool, default = false
1398+
13861399
/// (reading) If true, parquet reader will read columns of `Utf8/Utf8Large` with `Utf8View`,
13871400
/// and `Binary/BinaryLarge` with `BinaryView`.
13881401
///

‎datafusion/common/src/file_options/parquet_writer.rs‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -239,6 +239,7 @@ impl ParquetOptions {
239239
pushdown_filters: _,
240240
reorder_filters: _,
241241
force_filter_selections: _, // not used for writer props
242+
progressive_io: _, // not used for writer props
242243
allow_single_file_parallelism: _,
243244
maximum_parallel_row_group_writers: _,
244245
maximum_buffered_record_batches_per_stream: _,
@@ -423,6 +424,7 @@ mod tests {
423424
pushdown_filters: defaults.pushdown_filters,
424425
reorder_filters: defaults.reorder_filters,
425426
force_filter_selections: defaults.force_filter_selections,
427+
progressive_io: defaults.progressive_io,
426428
allow_single_file_parallelism: defaults.allow_single_file_parallelism,
427429
maximum_parallel_row_group_writers: defaults
428430
.maximum_parallel_row_group_writers,
@@ -551,6 +553,7 @@ mod tests {
551553
pushdown_filters: global_options_defaults.pushdown_filters,
552554
reorder_filters: global_options_defaults.reorder_filters,
553555
force_filter_selections: global_options_defaults.force_filter_selections,
556+
progressive_io: global_options_defaults.progressive_io,
554557
allow_single_file_parallelism: global_options_defaults
555558
.allow_single_file_parallelism,
556559
maximum_parallel_row_group_writers: global_options_defaults

‎datafusion/core/tests/datasource/object_store_access.rs‎

Lines changed: 215 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1019,6 +1019,181 @@ async fn query_single_parquet_file_multi_row_groups_multiple_predicates() {
10191019
);
10201020
}
10211021

1022+
#[tokio::test]
1023+
async fn query_single_parquet_file_pushdown_filters_one_shot_io() {
1024+
let test = Test::new().with_single_file_parquet().await;
1025+
test.set("datafusion.execution.parquet.pushdown_filters", "true")
1026+
.await;
1027+
1028+
// With filter pushdown enabled and `progressive_io` disabled (the
1029+
// default), all data needed for each row group (the columns needed by
1030+
// the filter and by the projection) is fetched with a single request per
1031+
// row group, the same I/O pattern as when pushdown_filters is disabled
1032+
// (compare `query_single_parquet_file_multi_row_groups_multiple_predicates`):
1033+
//
1034+
// * Row group 1 (`a` in 0..100): `a` (4-534) and `b` (534-1064) are
1035+
// fetched together, even though `a` is only needed to evaluate the
1036+
// filter.
1037+
// * Row group 2 (`a` in 100..200): the statistics prove every row
1038+
// matches, so the filter is skipped and only `b` (1594-2124) is read.
1039+
assert_snapshot!(
1040+
test.query("select min(b) from parquet_table WHERE a > 50").await,
1041+
@r"
1042+
------- Query Output (1 rows) -------
1043+
+----------------------+
1044+
| min(parquet_table.b) |
1045+
+----------------------+
1046+
| 1051 |
1047+
+----------------------+
1048+
------- Object Store Request Summary -------
1049+
RequestCountingObjectStore()
1050+
Total Requests: 3
1051+
- GET (opts) path=parquet_table.parquet head=true
1052+
- GET (ranges) path=parquet_table.parquet ranges=4-534,534-1064
1053+
- GET (ranges) path=parquet_table.parquet ranges=1594-2124
1054+
"
1055+
);
1056+
}
1057+
1058+
#[tokio::test]
1059+
async fn query_single_parquet_file_pushdown_filters_progressive_io() {
1060+
let test = Test::new().with_single_file_parquet().await;
1061+
test.set("datafusion.execution.parquet.pushdown_filters", "true")
1062+
.await;
1063+
test.set("datafusion.execution.parquet.progressive_io", "true")
1064+
.await;
1065+
1066+
// With `progressive_io` enabled (and an offset index present in the
1067+
// file), a row group that needs the filter is fetched progressively:
1068+
// first the column needed to evaluate the filter (`a`, 4-534), then the
1069+
// remaining projected column (`b`, 534-1064) for the rows that passed,
1070+
// resulting in multiple requests for row group 1. Row group 2 is fully
1071+
// matched by its statistics, so the filter is skipped and only `b` is
1072+
// read, as in the one-shot case.
1073+
assert_snapshot!(
1074+
test.query("select min(b) from parquet_table WHERE a > 50").await,
1075+
@r"
1076+
------- Query Output (1 rows) -------
1077+
+----------------------+
1078+
| min(parquet_table.b) |
1079+
+----------------------+
1080+
| 1051 |
1081+
+----------------------+
1082+
------- Object Store Request Summary -------
1083+
RequestCountingObjectStore()
1084+
Total Requests: 4
1085+
- GET (opts) path=parquet_table.parquet head=true
1086+
- GET (ranges) path=parquet_table.parquet ranges=4-534
1087+
- GET (ranges) path=parquet_table.parquet ranges=534-951,951-1064
1088+
- GET (ranges) path=parquet_table.parquet ranges=1594-2124
1089+
"
1090+
);
1091+
}
1092+
1093+
#[tokio::test]
1094+
async fn query_single_parquet_file_multi_page_no_pushdown() {
1095+
let test = Test::new().with_single_file_parquet_multi_page().await;
1096+
1097+
// Baseline for the `*_page_index` tests below: without filter pushdown
1098+
// the page index still prunes the first two pages of `a` in row group 1
1099+
// (values 0..40 cannot match `a > 50`) and the matching rows of `b`, so
1100+
// only the dictionary page and the selected data pages of each column
1101+
// chunk are fetched, with a single request per row group.
1102+
assert_snapshot!(
1103+
test.query("select min(b) from parquet_table WHERE a > 50").await,
1104+
@r"
1105+
------- Query Output (1 rows) -------
1106+
+----------------------+
1107+
| min(parquet_table.b) |
1108+
+----------------------+
1109+
| 1051 |
1110+
+----------------------+
1111+
------- Object Store Request Summary -------
1112+
RequestCountingObjectStore()
1113+
Total Requests: 3
1114+
- GET (opts) path=parquet_table.parquet head=true
1115+
- GET (ranges) path=parquet_table.parquet ranges=4-421,492-529,529-569,569-609,609-1026,1097-1134,1134-1174,1174-1214
1116+
- GET (ranges) path=parquet_table.parquet ranges=1214-1819,1819-2424
1117+
"
1118+
);
1119+
}
1120+
1121+
#[tokio::test]
1122+
async fn query_single_parquet_file_pushdown_filters_one_shot_io_page_index() {
1123+
let test = Test::new().with_single_file_parquet_multi_page().await;
1124+
test.set("datafusion.execution.parquet.pushdown_filters", "true")
1125+
.await;
1126+
1127+
// Each row group has five 20-row data pages per column. The page index
1128+
// proves the first two pages of `a` in row group 1 (values 0..40) cannot
1129+
// match `a > 50`, so the resulting row selection skips them. With
1130+
// one-shot I/O the data is still fetched with a single request per row
1131+
// group, but only the pages the decoder will read:
1132+
//
1133+
// * `a` (needed by the filter) becomes two ranges: its dictionary page
1134+
// (4-421) and the run of selected data pages (492-609); the two pruned
1135+
// pages (421-492) are not fetched.
1136+
// * `b` (the output column) is fetched as one range (609-1214). Note the
1137+
// decoder reads every page of `b` once a row filter is installed, even
1138+
// though the first two pages hold no selected rows (compare
1139+
// `query_single_parquet_file_multi_page_no_pushdown`, which skips
1140+
// 1026-1097). The plan reproduces the decoder's own fetch, so one-shot
1141+
// I/O reads the same bytes as progressive I/O does.
1142+
// * Row group 2 is fully matched by its statistics, so the filter is
1143+
// skipped and only `b` (1819-2424) is read.
1144+
assert_snapshot!(
1145+
test.query("select min(b) from parquet_table WHERE a > 50").await,
1146+
@r"
1147+
------- Query Output (1 rows) -------
1148+
+----------------------+
1149+
| min(parquet_table.b) |
1150+
+----------------------+
1151+
| 1051 |
1152+
+----------------------+
1153+
------- Object Store Request Summary -------
1154+
RequestCountingObjectStore()
1155+
Total Requests: 3
1156+
- GET (opts) path=parquet_table.parquet head=true
1157+
- GET (ranges) path=parquet_table.parquet ranges=4-421,492-609,609-1214
1158+
- GET (ranges) path=parquet_table.parquet ranges=1819-2424
1159+
"
1160+
);
1161+
}
1162+
1163+
#[tokio::test]
1164+
async fn query_single_parquet_file_pushdown_filters_progressive_io_page_index() {
1165+
let test = Test::new().with_single_file_parquet_multi_page().await;
1166+
test.set("datafusion.execution.parquet.pushdown_filters", "true")
1167+
.await;
1168+
test.set("datafusion.execution.parquet.progressive_io", "true")
1169+
.await;
1170+
1171+
// Same file and query as
1172+
// `query_single_parquet_file_pushdown_filters_one_shot_io_page_index`,
1173+
// but fetched progressively: the dictionary and selected pages of `a`
1174+
// first, then (after the filter is evaluated) the dictionary and data
1175+
// pages of `b`. The bytes read are the same as with one-shot I/O; only
1176+
// the number of requests differs.
1177+
assert_snapshot!(
1178+
test.query("select min(b) from parquet_table WHERE a > 50").await,
1179+
@r"
1180+
------- Query Output (1 rows) -------
1181+
+----------------------+
1182+
| min(parquet_table.b) |
1183+
+----------------------+
1184+
| 1051 |
1185+
+----------------------+
1186+
------- Object Store Request Summary -------
1187+
RequestCountingObjectStore()
1188+
Total Requests: 4
1189+
- GET (opts) path=parquet_table.parquet head=true
1190+
- GET (ranges) path=parquet_table.parquet ranges=4-421,492-529,529-569,569-609
1191+
- GET (ranges) path=parquet_table.parquet ranges=609-1026,1026-1060,1060-1097,1097-1134,1134-1174,1174-1214
1192+
- GET (ranges) path=parquet_table.parquet ranges=1819-2424
1193+
"
1194+
);
1195+
}
1196+
10221197
/// Runs tests with a request counting object store
10231198
struct Test {
10241199
object_store: Arc<RequestCountingObjectStore>,
@@ -1060,6 +1235,17 @@ impl Test {
10601235
format!("{}", self.object_store)
10611236
}
10621237

1238+
/// Set a session configuration value via SQL `SET`
1239+
async fn set(&self, key: &str, value: &str) {
1240+
self.session_context
1241+
.sql(&format!("set {key} = {value}"))
1242+
.await
1243+
.unwrap()
1244+
.collect()
1245+
.await
1246+
.unwrap();
1247+
}
1248+
10631249
/// Store the specified bytes at the given path
10641250
async fn with_bytes(self, path: &str, bytes: impl Into<Bytes>) -> Self {
10651251
let path = Path::from(path);
@@ -1320,6 +1506,35 @@ impl Test {
13201506
.await
13211507
}
13221508

1509+
/// Like [`Self::with_single_file_parquet`], but each column chunk is
1510+
/// written as five 20-row data pages (plus a dictionary page) so that
1511+
/// the page index can prune individual pages within a row group.
1512+
async fn with_single_file_parquet_multi_page(self) -> Test {
1513+
let a: ArrayRef = Arc::new(Int32Array::from_iter_values(0..200));
1514+
let b: ArrayRef = Arc::new(Int32Array::from_iter_values(1000..1200));
1515+
let batch = RecordBatch::try_from_iter([("a", a), ("b", b)]).unwrap();
1516+
1517+
let mut buffer = vec![];
1518+
let props = parquet::file::properties::WriterProperties::builder()
1519+
.set_max_row_group_row_count(Some(100))
1520+
.set_data_page_row_count_limit(20)
1521+
.set_write_batch_size(20)
1522+
.build();
1523+
let mut writer = parquet::arrow::ArrowWriter::try_new(
1524+
&mut buffer,
1525+
batch.schema(),
1526+
Some(props),
1527+
)
1528+
.unwrap();
1529+
writer.write(&batch).unwrap();
1530+
writer.close().unwrap();
1531+
1532+
self.with_bytes("/parquet_table.parquet", buffer)
1533+
.await
1534+
.register_parquet("parquet_table", "/parquet_table.parquet")
1535+
.await
1536+
}
1537+
13231538
/// Runs the specified query and returns a string representation of the results
13241539
/// suitable for comparison with insta snapshots
13251540
///

‎datafusion/datasource-parquet/src/file_format.rs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -717,6 +717,7 @@ impl From<&ParquetFormatFactory> for protobuf::TableParquetOptions {
717717
pushdown_filters: global_options.global.pushdown_filters,
718718
reorder_filters: global_options.global.reorder_filters,
719719
force_filter_selections: global_options.global.force_filter_selections,
720+
progressive_io: global_options.global.progressive_io,
720721
data_pagesize_limit: global_options.global.data_pagesize_limit as u64,
721722
write_batch_size: global_options.global.write_batch_size as u64,
722723
writer_version: global_options.global.writer_version.to_string(),

‎datafusion/datasource-parquet/src/opener/mod.rs‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -270,6 +270,11 @@ pub(super) struct ParquetMorselizer {
270270
pub reorder_filters: bool,
271271
/// Should we force the reader to use RowSelections for filtering
272272
pub force_filter_selections: bool,
273+
/// If true (and filters are pushed down), fetch column data
274+
/// progressively as filters are evaluated. If false, fetch all data
275+
/// pages needed for a row group in a single request. See
276+
/// [`ParquetOptions::progressive_io`](datafusion_common::config::ParquetOptions::progressive_io)
277+
pub progressive_io: bool,
273278
/// Should the page index be read from parquet files, if present, to skip
274279
/// data pages
275280
pub enable_page_index: bool,
@@ -469,6 +474,7 @@ struct PreparedParquetOpen {
469474
reorder_predicates: bool,
470475
pushdown_filters: bool,
471476
force_filter_selections: bool,
477+
progressive_io: bool,
472478
enable_page_index: bool,
473479
enable_bloom_filter: bool,
474480
enable_row_group_stats_pruning: bool,
@@ -991,6 +997,7 @@ impl ParquetMorselizer {
991997
reorder_predicates: self.reorder_filters,
992998
pushdown_filters: self.pushdown_filters,
993999
force_filter_selections: self.force_filter_selections,
1000+
progressive_io: self.progressive_io,
9941001
enable_page_index: self.enable_page_index,
9951002
enable_bloom_filter: self.enable_bloom_filter,
9961003
enable_row_group_stats_pruning: self.enable_row_group_stats_pruning,
@@ -1814,6 +1821,18 @@ impl RowGroupsPrunedParquetOpen {
18141821
}
18151822
};
18161823

1824+
// Decide the I/O pattern for the row groups read with a pushed down
1825+
// `RowFilter` (see `PushDecoderStreamState::one_shot_io`). Fetching
1826+
// progressively can only reduce the bytes read when the file has an
1827+
// offset index: without one the decoder fetches whole column chunks
1828+
// either way, and the extra requests are pure overhead.
1829+
let has_offset_index = reader_metadata
1830+
.metadata()
1831+
.page_index()
1832+
.is_some_and(|page_index| page_index.has_offset_indexes());
1833+
let one_shot_io = row_filter_context.is_some()
1834+
&& (!prepared.progressive_io || !has_offset_index);
1835+
18171836
// Track how much of this file range the scan has finished with. Credit
18181837
// up front every row group it will not read: those pruning removed, and
18191838
// — for a file split into ranges for parallelism — those belonging to
@@ -1902,6 +1921,7 @@ impl RowGroupsPrunedParquetOpen {
19021921
filter_installed,
19031922
row_filter_skipped_fully_matched,
19041923
byte_progress,
1924+
one_shot_io,
19051925
}
19061926
.into_stream();
19071927

@@ -2211,6 +2231,7 @@ mod test {
22112231
pushdown_filters: bool,
22122232
reorder_filters: bool,
22132233
force_filter_selections: bool,
2234+
progressive_io: bool,
22142235
enable_page_index: bool,
22152236
enable_bloom_filter: bool,
22162237
enable_row_group_stats_pruning: bool,
@@ -2431,6 +2452,7 @@ mod test {
24312452
pushdown_filters: false,
24322453
reorder_filters: false,
24332454
force_filter_selections: false,
2455+
progressive_io: false,
24342456
enable_page_index: false,
24352457
enable_bloom_filter: false,
24362458
enable_row_group_stats_pruning: false,
@@ -2628,6 +2650,7 @@ mod test {
26282650
pushdown_filters: self.pushdown_filters,
26292651
reorder_filters: self.reorder_filters,
26302652
force_filter_selections: self.force_filter_selections,
2653+
progressive_io: self.progressive_io,
26312654
enable_page_index: self.enable_page_index,
26322655
enable_bloom_filter: self.enable_bloom_filter,
26332656
enable_row_group_stats_pruning: self.enable_row_group_stats_pruning,

0 commit comments

Comments
 (0)