Skip to content

Commit 39a66b1

Browse files
mohitgurav20alambclaude
authored
test(parquet): add I/O tracking test for Bloom filter loading baseline (apache#26897)
# Which issue does this PR close? Part of apache#25797. Baseline for apache#25811. # Rationale for this change In apache#25811 we evaluate Parquet bloom filters incrementally during row group pruning to avoid fetching filters for later columns when an earlier column already rejects the row group. Per review discussion on apache#25811, this PR adds a baseline test to `main` using a tracking `AsyncFileReader` (similar to `arrow-rs` async reader tests). It records that `main` currently fetches all candidate column bloom filters upfront (2 requests for a 2-column query where the first column is absent). Once this merges, apache#25811 will rebase on top and show the skipped fetch directly in the test diff. # What changes are included in this PR? - Adds `RangeTracker`, `TrackingAsyncReader`, and `TrackingReaderFactory` in `datafusion/datasource-parquet/src/opener/mod.rs` to record requested byte ranges. - Adds `test_bloom_filter_io_pattern_baseline` asserting current upfront fetch behavior on `main`. # Are these changes tested? Yes, via `test_bloom_filter_io_pattern_baseline`. # Are there any user-facing changes? No. --------- Co-authored-by: Andrew Lamb <[email protected]> Co-authored-by: Claude Opus 5.5 <[email protected]>
1 parent 320a7d2 commit 39a66b1

4 files changed

Lines changed: 415 additions & 0 deletions

File tree

‎datafusion/core/tests/datasource/object_store_access.rs‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,9 @@
99
//! various operations (table creation, queries with and without predicates)
1010
//! to verify the expected object store access patterns.
1111
//!
12+
//! File-level Parquet access tests live in
13+
//! [`datafusion/core/tests/parquet/io`](https://github.com/apache/datafusion/blob/main/datafusion/core/tests/parquet/io/mod.rs).
14+
//!
1215
//! [`ListingTable`]: datafusion::datasource::listing::ListingTable
1316
1417
use arrow::array::{ArrayRef, Int32Array, RecordBatch};
Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,85 @@
1+
// SPDX-License-Identifier: Apache-2.0
2+
// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership
3+
// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements
4+
5+
//! Tests tracking storage I/O events during Parquet Bloom filter evaluation.
6+
7+
use std::sync::Arc;
8+
9+
use arrow::array::{ArrayRef, Int32Array, RecordBatch};
10+
use bytes::Bytes;
11+
use datafusion::prelude::SessionConfig;
12+
use insta::assert_snapshot;
13+
use parquet::arrow::ArrowWriter;
14+
use parquet::file::properties::{EnabledStatistics, WriterProperties};
15+
16+
use super::ParquetIoTest;
17+
18+
/// Matching row group
19+
#[tokio::test]
20+
async fn test_parquet_bloom_filter_io_matching() {
21+
let test = bloom_filter_test().await;
22+
// Loads the Bloom filters, finds a match, then reads the data pages and
23+
// produces a batch.
24+
assert_snapshot!(test.query("SELECT * FROM t WHERE a = 10 AND b = 100").await, @r"
25+
--- Query: SELECT * FROM t WHERE a = 10 AND b = 100 (1 rows) ---
26+
Read Parquet footer (offsets 0..678)
27+
Read OffsetIndex for column 'a' (rg 0) (offsets 192..213)
28+
Read Bloom filter for column 'a' (rg 0) (offsets 98..145)
29+
Read Bloom filter for column 'b' (rg 0) (offsets 145..192)
30+
Read data page for column 'a' (rg 0) (offsets 4..51)
31+
Read data page for column 'b' (rg 0) (offsets 51..98)
32+
Produce batch (1 rows)
33+
");
34+
}
35+
36+
/// Column `a` does not match, so the row group is pruned.
37+
#[tokio::test]
38+
async fn test_parquet_bloom_filter_io_pruned() {
39+
let test = bloom_filter_test().await;
40+
// Currently the Bloom filters for all candidate columns (`a` and `b`) are
41+
// loaded upfront. With incremental pruning
42+
// (https://github.com/apache/datafusion/issues/25811), `a` would be
43+
// evaluated first and the Bloom filter for `b` would not be loaded.
44+
assert_snapshot!(test.query("SELECT * FROM t WHERE a = 999 AND b = 100").await, @r"
45+
--- Query: SELECT * FROM t WHERE a = 999 AND b = 100 (0 rows) ---
46+
Read Parquet footer (offsets 0..678)
47+
Read OffsetIndex for column 'a' (rg 0) (offsets 192..213)
48+
Read Bloom filter for column 'a' (rg 0) (offsets 98..145)
49+
Read Bloom filter for column 'b' (rg 0) (offsets 145..192)
50+
");
51+
}
52+
53+
/// Returns a [`ParquetIoTest`] for a file with one row group of 3 rows:
54+
///
55+
/// | a | b |
56+
/// |----|-----|
57+
/// | 10 | 100 |
58+
/// | 20 | 200 |
59+
/// | 30 | 300 |
60+
///
61+
/// Both columns have Bloom filters. Statistics are disabled so that row group
62+
/// pruning relies entirely on the Bloom filters.
63+
async fn bloom_filter_test() -> ParquetIoTest {
64+
let a: ArrayRef = Arc::new(Int32Array::from(vec![10, 20, 30]));
65+
let b: ArrayRef = Arc::new(Int32Array::from(vec![100, 200, 300]));
66+
let batch = RecordBatch::try_from_iter([("a", a), ("b", b)]).unwrap();
67+
68+
let props = WriterProperties::builder()
69+
.set_max_row_group_row_count(Some(3))
70+
.set_bloom_filter_enabled(true)
71+
.set_statistics_enabled(EnabledStatistics::None)
72+
.build();
73+
74+
let mut buffer = Vec::new();
75+
let mut writer =
76+
ArrowWriter::try_new(&mut buffer, batch.schema(), Some(props)).unwrap();
77+
writer.write(&batch).unwrap();
78+
writer.close().unwrap();
79+
80+
let config = SessionConfig::new()
81+
.with_target_partitions(1)
82+
.with_parquet_bloom_filter_pruning(true);
83+
84+
ParquetIoTest::new(Bytes::from(buffer), config).await
85+
}

0 commit comments

Comments
 (0)