Skip to content

Commit 90b55e3

Browse files
alambmohitgurav20claude
authored
test(parquet): add I/O access tests for parquet pruning, filter pushdown and TopK (apache#27022)
## Which issue does this PR close? - Part of apache#26682 - Follow on to apache#26897 ## Rationale for this change We have several issues / PRs open that contemplate changing the I/O pattern such as - apache#24393 - apache#20324 - apache#26387 - apache#26418 - apache#23492 - apache#24086 - apache#25811 I want to have a test that clearly shows how those proposals effect the scan ## What changes are included in this PR? Add some new tests that show the Parquet I/O patterns. For example I think this is quite clear now the difference in I/O between filter pushdown and not ```rust /// Without filter pushdown, the scan fetches every column of every row group #[tokio::test] async fn test_filter_pushdown_io_disabled() { let test = filter_pushdown_test(false).await; assert_snapshot!(test.query("SELECT * FROM t WHERE val = 7").await, @r" --- Query: SELECT * FROM t WHERE val = 7 (1 rows) --- Request 1 Read Parquet footer (offsets 0..1402) Request 2 Read dictionary page and data page 0 for column 'id' (rg 0) (offsets 4..51) Read dictionary page and data page 0 for column 'val' (rg 0) (offsets 51..98) Request 3 Read dictionary page and data page 0 for column 'id' (rg 1) (offsets 98..145) Read dictionary page and data page 0 for column 'val' (rg 1) (offsets 145..192) Request 4 Read dictionary page and data page 0 for column 'id' (rg 2) (offsets 192..239) Read dictionary page and data page 0 for column 'val' (rg 2) (offsets 239..286) Produce batch (1 rows) "); } /// **With** filter pushdown, the scan first fetches and evaluates the filter /// column (`val`) for each row group, and then fetches the remaining columns /// (`id`) only for the row groups with matching rows (row group 1). #[tokio::test] async fn test_filter_pushdown_io_enabled() { let test = filter_pushdown_test(true).await; assert_snapshot!(test.query("SELECT * FROM t WHERE val = 7").await, @r" --- Query: SELECT * FROM t WHERE val = 7 (1 rows) --- Request 1 Read Parquet footer (offsets 0..1402) Request 2 Read dictionary page and data page 0 for column 'val' (rg 0) (offsets 51..98) Request 3 Read dictionary page and data page 0 for column 'val' (rg 1) (offsets 145..192) Request 4 Read dictionary page for column 'id' (rg 1) (offsets 98..124) Read data page 0 for column 'id' (rg 1) (offsets 124..145) Produce batch (1 rows) Request 5 Read dictionary page and data page 0 for column 'val' (rg 2) (offsets 239..286) "); } ``` ## What is the testing strategy for this PR? CI ## Are there any user-facing changes? No --------- Co-authored-by: mohitgurav20 <[email protected]> Co-authored-by: Claude Opus 5.5 <[email protected]>
1 parent 22aedf0 commit 90b55e3

6 files changed

Lines changed: 880 additions & 65 deletions

File tree

‎datafusion/core/tests/parquet/io/bloom_filter_io.rs‎

Lines changed: 45 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,24 @@
33
// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements
44

55
//! Tests tracking storage I/O events during Parquet Bloom filter evaluation.
6+
//!
7+
//! The file has a single row group with Bloom filters on both columns and no
8+
//! statistics, so only the Bloom filters are used to prune.
9+
//!
10+
//! ```text
11+
//! row group 0
12+
//! ┌──────┬───────┐
13+
//! │ a │ b │
14+
//! ├──────┼───────┤
15+
//! │ 10 │ 100 │
16+
//! │ 20 │ 200 │
17+
//! │ 30 │ 300 │
18+
//! └──────┴───────┘
19+
//! Bloom filter (a) Bloom filter (b) no min/max statistics
20+
//!
21+
//! a = 10 AND b = 100 -> Bloom filters match, row group read
22+
//! a = 999 AND b = 100 -> Bloom filter on `a` misses, row group pruned
23+
//! ```
624
725
use std::sync::Arc;
826

@@ -21,14 +39,25 @@ async fn test_parquet_bloom_filter_io_matching() {
2139
let test = bloom_filter_test().await;
2240
// Loads the Bloom filters, finds a match, then reads the data pages and
2341
// produces a batch.
42+
//
43+
// Note: in practice, the default 512 KiB metadata size hint often covers
44+
// the OffsetIndex and Bloom filters for small files, so they wouldn't
45+
// need separate requests.
2446
assert_snapshot!(test.query("SELECT * FROM t WHERE a = 10 AND b = 100").await, @r"
2547
--- Query: SELECT * FROM t WHERE a = 10 AND b = 100 (1 rows) ---
26-
Read Parquet footer (offsets 0..678)
27-
Read OffsetIndex for column 'a' (rg 0) (offsets 192..213)
28-
Read Bloom filter for column 'a' (rg 0) (offsets 98..145)
29-
Read Bloom filter for column 'b' (rg 0) (offsets 145..192)
30-
Read data page for column 'a' (rg 0) (offsets 4..51)
31-
Read data page for column 'b' (rg 0) (offsets 51..98)
48+
Request 1
49+
Head (object size and metadata)
50+
Request 2
51+
Read Parquet footer (offsets 213..678)
52+
Request 3
53+
Read OffsetIndex for columns 'a', 'b' (rg 0) (offsets 192..213)
54+
Request 4
55+
Read Bloom filter for column 'a' (rg 0) (offsets 98..145)
56+
Request 5
57+
Read Bloom filter for column 'b' (rg 0) (offsets 145..192)
58+
Request 6
59+
Read dictionary page and data page 0 for column 'a' (rg 0) (offsets 4..51)
60+
Read dictionary page and data page 0 for column 'b' (rg 0) (offsets 51..98)
3261
Produce batch (1 rows)
3362
");
3463
}
@@ -43,10 +72,16 @@ async fn test_parquet_bloom_filter_io_pruned() {
4372
// evaluated first and the Bloom filter for `b` would not be loaded.
4473
assert_snapshot!(test.query("SELECT * FROM t WHERE a = 999 AND b = 100").await, @r"
4574
--- Query: SELECT * FROM t WHERE a = 999 AND b = 100 (0 rows) ---
46-
Read Parquet footer (offsets 0..678)
47-
Read OffsetIndex for column 'a' (rg 0) (offsets 192..213)
48-
Read Bloom filter for column 'a' (rg 0) (offsets 98..145)
49-
Read Bloom filter for column 'b' (rg 0) (offsets 145..192)
75+
Request 1
76+
Head (object size and metadata)
77+
Request 2
78+
Read Parquet footer (offsets 213..678)
79+
Request 3
80+
Read OffsetIndex for columns 'a', 'b' (rg 0) (offsets 192..213)
81+
Request 4
82+
Read Bloom filter for column 'a' (rg 0) (offsets 98..145)
83+
Request 5
84+
Read Bloom filter for column 'b' (rg 0) (offsets 145..192)
5085
");
5186
}
5287

Lines changed: 118 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,118 @@
1+
// SPDX-License-Identifier: Apache-2.0
2+
// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership
3+
// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements
4+
5+
//! Tests tracking I/O events with and without Parquet filter pushdown
6+
//! (`datafusion.execution.parquet.pushdown_filters`, see
7+
//! [`ParquetOptions::pushdown_filters`]).
8+
//!
9+
//! Both tests run `val = 7`, which statistics cannot prune, but which only
10+
//! matches a row in row group 1:
11+
//!
12+
//! ```text
13+
//! row group 0 row group 1 row group 2
14+
//! ┌──────┬───────┐ ┌──────┬───────┐ ┌──────┬───────┐
15+
//! │ id │ val │ │ id │ val │ │ id │ val │
16+
//! ├──────┼───────┤ ├──────┼───────┤ ├──────┼───────┤
17+
//! │ 1 │ 0 │ │ 4 │ 0 │ │ 7 │ 0 │
18+
//! │ 2 │ 5 │ │ 5 │ 7 * │ │ 8 │ 5 │
19+
//! │ 3 │ 10 │ │ 6 │ 10 │ │ 9 │ 10 │
20+
//! └──────┴───────┘ └──────┴───────┘ └──────┴───────┘
21+
//! val: min 0 max 10 val: min 0 max 10 val: min 0 max 10
22+
//!
23+
//! * the only row matching `val = 7`
24+
//! ```
25+
//!
26+
//! [`ParquetOptions::pushdown_filters`]: datafusion::config::ParquetOptions::pushdown_filters
27+
28+
use std::sync::Arc;
29+
30+
use arrow::array::{ArrayRef, Int32Array, RecordBatch};
31+
use bytes::Bytes;
32+
use datafusion::prelude::SessionConfig;
33+
use insta::assert_snapshot;
34+
use parquet::arrow::ArrowWriter;
35+
use parquet::file::properties::WriterProperties;
36+
37+
use super::ParquetIoTest;
38+
39+
/// Without filter pushdown, the scan fetches every column of every row group.
40+
#[tokio::test]
41+
async fn test_filter_pushdown_io_disabled() {
42+
let test = filter_pushdown_test(false).await;
43+
// Note: in practice, the default 512 KiB metadata size hint often covers
44+
// the OffsetIndex and Bloom filters for small files, so they wouldn't
45+
// need separate requests.
46+
assert_snapshot!(test.query("SELECT * FROM t WHERE val = 7").await, @r"
47+
--- Query: SELECT * FROM t WHERE val = 7 (1 rows) ---
48+
Request 1
49+
Head (object size and metadata)
50+
Request 2
51+
Read Parquet footer (offsets 489..1402)
52+
Request 3
53+
Read ColumnIndex and OffsetIndex for columns 'id', 'val' (rg 0, 1, 2) (offsets 286..489)
54+
Request 4
55+
Read dictionary page and data page 0 for column 'id' (rg 0) (offsets 4..51)
56+
Read dictionary page and data page 0 for column 'val' (rg 0) (offsets 51..98)
57+
Request 5
58+
Read dictionary page and data page 0 for column 'id' (rg 1) (offsets 98..145)
59+
Read dictionary page and data page 0 for column 'val' (rg 1) (offsets 145..192)
60+
Request 6
61+
Read dictionary page and data page 0 for column 'id' (rg 2) (offsets 192..239)
62+
Read dictionary page and data page 0 for column 'val' (rg 2) (offsets 239..286)
63+
Produce batch (1 rows)
64+
");
65+
}
66+
67+
/// **With** filter pushdown, the scan first fetches and evaluates the filter
68+
/// column (`val`) for each row group, and then fetches the remaining columns
69+
/// (`id`) only for the row groups with matching rows (row group 1).
70+
#[tokio::test]
71+
async fn test_filter_pushdown_io_enabled() {
72+
let test = filter_pushdown_test(true).await;
73+
assert_snapshot!(test.query("SELECT * FROM t WHERE val = 7").await, @r"
74+
--- Query: SELECT * FROM t WHERE val = 7 (1 rows) ---
75+
Request 1
76+
Head (object size and metadata)
77+
Request 2
78+
Read Parquet footer (offsets 489..1402)
79+
Request 3
80+
Read ColumnIndex and OffsetIndex for columns 'id', 'val' (rg 0, 1, 2) (offsets 286..489)
81+
Request 4
82+
Read dictionary page and data page 0 for column 'val' (rg 0) (offsets 51..98)
83+
Request 5
84+
Read dictionary page and data page 0 for column 'val' (rg 1) (offsets 145..192)
85+
Request 6
86+
Read dictionary page for column 'id' (rg 1) (offsets 98..124)
87+
Read data page 0 for column 'id' (rg 1) (offsets 124..145)
88+
Produce batch (1 rows)
89+
Request 7
90+
Read dictionary page and data page 0 for column 'val' (rg 2) (offsets 239..286)
91+
");
92+
}
93+
94+
/// Returns a [`ParquetIoTest`] for the file described in the module docs
95+
async fn filter_pushdown_test(pushdown_filters: bool) -> ParquetIoTest {
96+
let mut config = SessionConfig::new().with_target_partitions(1);
97+
config.options_mut().execution.parquet.pushdown_filters = pushdown_filters;
98+
ParquetIoTest::new(three_row_group_file(), config).await
99+
}
100+
101+
/// Returns the Parquet file described in the module docs: three row groups of
102+
/// three rows each, with `id` sorted across row groups.
103+
fn three_row_group_file() -> Bytes {
104+
let id: ArrayRef = Arc::new(Int32Array::from_iter_values(1..=9));
105+
let val: ArrayRef = Arc::new(Int32Array::from(vec![0, 5, 10, 0, 7, 10, 0, 5, 10]));
106+
let batch = RecordBatch::try_from_iter([("id", id), ("val", val)]).unwrap();
107+
108+
let props = WriterProperties::builder()
109+
.set_max_row_group_row_count(Some(3))
110+
.build();
111+
112+
let mut buffer = Vec::new();
113+
let mut writer =
114+
ArrowWriter::try_new(&mut buffer, batch.schema(), Some(props)).unwrap();
115+
writer.write(&batch).unwrap();
116+
writer.close().unwrap();
117+
Bytes::from(buffer)
118+
}

0 commit comments

Comments
 (0)