Skip to content

Commit 2bcf4e7

Browse files
authored
Merge branch 'main' into apachegh-24128-pruning-predicate-builder
2 parents 9af8bcc + 24483db commit 2bcf4e7

23 files changed

Lines changed: 1801 additions & 410 deletions

File tree

‎datafusion/core/Cargo.toml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -257,6 +257,11 @@ harness = false
257257
name = "parquet_struct_projection"
258258
required-features = ["parquet"]
259259

260+
[[bench]]
261+
harness = false
262+
name = "parquet_struct_shared_prefix_pushdown"
263+
required-features = ["parquet"]
264+
260265
[[bench]]
261266
harness = false
262267
name = "cse_projection_pushdown"
Lines changed: 329 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,329 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
//! Benchmarks for row-filter pushdown with predicates that reach several struct
19+
//! leaves under a common prefix.
20+
//!
21+
//! The existing `parquet_struct_query` bench exercises single-field struct
22+
//! predicates only; `parquet_struct_projection` has no `WHERE` clause. Neither
23+
//! drives the row-filter planner with multiple accesses under the same struct
24+
//! root.
25+
//!
26+
//! Two properties of the planner shape these queries, and both are easy to get
27+
//! wrong:
28+
//!
29+
//! * `execution.parquet.pushdown_filters` must be enabled. It defaults to
30+
//! `false`, in which case no row filter is built and every case below
31+
//! degenerates to a plain scan.
32+
//! * The predicate must be a *single* conjunct. `build_row_filter` calls
33+
//! `split_conjunction` before building filter candidates, and each candidate
34+
//! collects its own access paths, so `s['a'] = 5 AND s['b'] = 5` becomes two
35+
//! independent single-access candidates and never reaches multi-access
36+
//! planning. The cases below use the `(s['a'] + s['b']) = 10` form so every
37+
//! access lands in one candidate.
38+
//!
39+
//! Nested access is written `s['inner']['x']`, which the planner represents as a
40+
//! single flattened `get_field(s, 'inner', 'x')`. That form is pushdown-eligible;
41+
//! a chained `get_field(get_field(s, 'inner'), 'x')` is not.
42+
//!
43+
//! Dataset schema:
44+
//!
45+
//! ```sql
46+
//! CREATE TABLE t (
47+
//! id INT,
48+
//! s STRUCT<
49+
//! a INT, b INT, c INT, d INT, e INT,
50+
//! inner STRUCT<x INT, y INT, z INT>
51+
//! >
52+
//! );
53+
//! ```
54+
//!
55+
//! All struct leaves mirror the top-level `id`, so a sum of `n` leaves equals
56+
//! `n * id` and every predicate is satisfied by exactly one row (`id = 5`).
57+
//! Holding the match count fixed keeps the cases comparable, and each is
58+
//! asserted to return that single row.
59+
60+
use arrow::array::{ArrayRef, Int32Array, StructArray};
61+
use arrow::datatypes::{DataType, Field, Fields, Schema, SchemaRef};
62+
use arrow::record_batch::RecordBatch;
63+
use criterion::{Criterion, criterion_group, criterion_main};
64+
use datafusion::prelude::{SessionConfig, SessionContext};
65+
use datafusion_common::instant::Instant;
66+
use parquet::arrow::ArrowWriter;
67+
use parquet::file::properties::{WriterProperties, WriterVersion};
68+
use std::hint::black_box;
69+
use std::path::Path;
70+
use std::sync::Arc;
71+
use tempfile::NamedTempFile;
72+
use tokio::runtime::Runtime;
73+
74+
/// The number of batches to write
75+
const NUM_BATCHES: usize = 128;
76+
/// The number of rows in each record batch to write
77+
const WRITE_RECORD_BATCH_SIZE: usize = 4096;
78+
/// The number of rows in a row group
79+
const ROW_GROUP_ROW_COUNT: usize = 65536;
80+
/// The number of row groups expected
81+
const EXPECTED_ROW_GROUPS: usize = 8;
82+
/// Number of rows every predicate is expected to match.
83+
const EXPECTED_MATCHES: usize = 1;
84+
85+
fn inner_struct_fields() -> Fields {
86+
Fields::from(vec![
87+
Field::new("x", DataType::Int32, false),
88+
Field::new("y", DataType::Int32, false),
89+
Field::new("z", DataType::Int32, false),
90+
])
91+
}
92+
93+
fn struct_fields() -> Fields {
94+
Fields::from(vec![
95+
Field::new("a", DataType::Int32, false),
96+
Field::new("b", DataType::Int32, false),
97+
Field::new("c", DataType::Int32, false),
98+
Field::new("d", DataType::Int32, false),
99+
Field::new("e", DataType::Int32, false),
100+
Field::new("inner", DataType::Struct(inner_struct_fields()), false),
101+
])
102+
}
103+
104+
fn schema() -> SchemaRef {
105+
Arc::new(Schema::new(vec![
106+
Field::new("id", DataType::Int32, false),
107+
Field::new("s", DataType::Struct(struct_fields()), false),
108+
]))
109+
}
110+
111+
fn generate_batch(batch_id: usize) -> RecordBatch {
112+
let schema = schema();
113+
let len = WRITE_RECORD_BATCH_SIZE;
114+
115+
// Sequential IDs give distinct per-row values so a predicate like
116+
// `s['a'] = 5` matches exactly one row, mirroring parquet_struct_query.
117+
let base_id = (batch_id * len) as i32;
118+
let id_values: Vec<i32> = (0..len).map(|i| base_id + i as i32).collect();
119+
let id_array = Arc::new(Int32Array::from(id_values.clone()));
120+
121+
let leaf = || Arc::new(Int32Array::from(id_values.clone())) as ArrayRef;
122+
123+
let inner_struct = StructArray::from(vec![
124+
(Arc::new(Field::new("x", DataType::Int32, false)), leaf()),
125+
(Arc::new(Field::new("y", DataType::Int32, false)), leaf()),
126+
(Arc::new(Field::new("z", DataType::Int32, false)), leaf()),
127+
]);
128+
129+
let struct_array = StructArray::from(vec![
130+
(Arc::new(Field::new("a", DataType::Int32, false)), leaf()),
131+
(Arc::new(Field::new("b", DataType::Int32, false)), leaf()),
132+
(Arc::new(Field::new("c", DataType::Int32, false)), leaf()),
133+
(Arc::new(Field::new("d", DataType::Int32, false)), leaf()),
134+
(Arc::new(Field::new("e", DataType::Int32, false)), leaf()),
135+
(
136+
Arc::new(Field::new(
137+
"inner",
138+
DataType::Struct(inner_struct_fields()),
139+
false,
140+
)),
141+
Arc::new(inner_struct) as ArrayRef,
142+
),
143+
]);
144+
145+
RecordBatch::try_new(schema, vec![id_array, Arc::new(struct_array)]).unwrap()
146+
}
147+
148+
fn generate_file() -> NamedTempFile {
149+
let now = Instant::now();
150+
let mut named_file = tempfile::Builder::new()
151+
.prefix("parquet_struct_shared_prefix_pushdown")
152+
.suffix(".parquet")
153+
.tempfile()
154+
.unwrap();
155+
156+
println!("Generating parquet file - {}", named_file.path().display());
157+
let schema = schema();
158+
159+
let properties = WriterProperties::builder()
160+
.set_writer_version(WriterVersion::PARQUET_2_0)
161+
.set_max_row_group_row_count(Some(ROW_GROUP_ROW_COUNT))
162+
.build();
163+
164+
let mut writer =
165+
ArrowWriter::try_new(&mut named_file, schema, Some(properties)).unwrap();
166+
167+
for batch_id in 0..NUM_BATCHES {
168+
let batch = generate_batch(batch_id);
169+
writer.write(&batch).unwrap();
170+
}
171+
172+
let metadata = writer.close().unwrap();
173+
let file_metadata = metadata.file_metadata();
174+
let expected_rows = WRITE_RECORD_BATCH_SIZE * NUM_BATCHES;
175+
assert_eq!(
176+
file_metadata.num_rows() as usize,
177+
expected_rows,
178+
"Expected {expected_rows} rows but got {}",
179+
file_metadata.num_rows()
180+
);
181+
assert_eq!(
182+
metadata.row_groups().len(),
183+
EXPECTED_ROW_GROUPS,
184+
"Expected {EXPECTED_ROW_GROUPS} row groups but got {}",
185+
metadata.row_groups().len()
186+
);
187+
188+
println!(
189+
"Generated parquet file with {} rows and {} row groups in {:.2}s",
190+
file_metadata.num_rows(),
191+
metadata.row_groups().len(),
192+
now.elapsed().as_secs_f32()
193+
);
194+
195+
named_file
196+
}
197+
198+
fn create_context(file_path: &str, rt: &Runtime) -> SessionContext {
199+
let mut config = SessionConfig::new();
200+
// Row-filter pushdown is off by default. Without it no row filter is built,
201+
// and these benchmarks would time a plain scan for every predicate shape.
202+
config.options_mut().execution.parquet.pushdown_filters = true;
203+
204+
let ctx = SessionContext::new_with_config(config);
205+
rt.block_on(ctx.register_parquet("t", file_path, Default::default()))
206+
.unwrap();
207+
ctx
208+
}
209+
210+
fn query(ctx: &SessionContext, rt: &Runtime, sql: &str) {
211+
let ctx = ctx.clone();
212+
let sql = sql.to_string();
213+
let df = rt.block_on(ctx.sql(&sql)).unwrap();
214+
black_box(rt.block_on(df.collect()).unwrap());
215+
}
216+
217+
/// Fails unless `sql` actually pushes a row filter into the Parquet decoder and
218+
/// matches [`EXPECTED_MATCHES`] rows.
219+
///
220+
/// Guards the two silent-failure modes: a disabled `pushdown_filters` and a
221+
/// predicate shape that turns out not to be pushdown-eligible. Either would
222+
/// leave the benchmark timing a plain scan instead of row-filter pushdown.
223+
///
224+
/// Metrics are read off the executed plan rather than scraped from
225+
/// `EXPLAIN ANALYZE` text, so the check does not depend on output formatting.
226+
fn assert_pushdown_active(ctx: &SessionContext, rt: &Runtime, name: &str, sql: &str) {
227+
let (rows, pruned) = rt
228+
.block_on(async {
229+
let plan = ctx.sql(sql).await?.create_physical_plan().await?;
230+
let batches =
231+
datafusion::physical_plan::collect(Arc::clone(&plan), ctx.task_ctx())
232+
.await?;
233+
let rows = batches.iter().map(|b| b.num_rows()).sum::<usize>();
234+
Ok::<_, datafusion_common::DataFusionError>((rows, rows_pruned(&plan)))
235+
})
236+
.unwrap();
237+
238+
assert_eq!(
239+
rows, EXPECTED_MATCHES,
240+
"`{name}` matched {rows} rows, expected {EXPECTED_MATCHES}"
241+
);
242+
assert!(
243+
pruned > 0,
244+
"`{name}` pruned no rows via the Parquet row filter, so it does not \
245+
exercise row-filter pushdown (is `pushdown_filters` enabled, and is \
246+
the predicate a single pushdown-eligible conjunct?)"
247+
);
248+
}
249+
250+
/// Total `pushdown_rows_pruned` reported anywhere in the executed plan.
251+
fn rows_pruned(plan: &Arc<dyn datafusion::physical_plan::ExecutionPlan>) -> usize {
252+
let mut total = plan
253+
.metrics()
254+
.and_then(|metrics| metrics.sum_by_name("pushdown_rows_pruned"))
255+
.map(|value| value.as_usize())
256+
.unwrap_or(0);
257+
258+
for child in plan.children() {
259+
total += rows_pruned(child);
260+
}
261+
262+
total
263+
}
264+
265+
fn criterion_benchmark(c: &mut Criterion) {
266+
let (file_path, temp_file) = match std::env::var("PARQUET_FILE") {
267+
Ok(file) => (file, None),
268+
Err(_) => {
269+
let temp_file = generate_file();
270+
(temp_file.path().display().to_string(), Some(temp_file))
271+
}
272+
};
273+
274+
assert!(Path::new(&file_path).exists(), "path not found");
275+
println!("Using parquet file {file_path}");
276+
277+
let rt = Runtime::new().unwrap();
278+
let ctx = create_context(&file_path, &rt);
279+
280+
// Baseline: one access on a single struct leaf.
281+
let sql = "select id from t where s['a'] = 5";
282+
assert_pushdown_active(&ctx, &rt, "1_access", sql);
283+
c.bench_function("1_access", |b| b.iter(|| query(&ctx, &rt, sql)));
284+
285+
// Two accesses inside one conjunct, sharing the struct root `s`.
286+
let sql = "select id from t where (s['a'] + s['b']) = 10";
287+
assert_pushdown_active(&ctx, &rt, "2_access_shared_root", sql);
288+
c.bench_function("2_access_shared_root", |b| b.iter(|| query(&ctx, &rt, sql)));
289+
290+
// Three accesses sharing the struct root `s`.
291+
let sql = "select id from t where (s['a'] + s['b'] + s['c']) = 15";
292+
assert_pushdown_active(&ctx, &rt, "3_access_shared_root", sql);
293+
c.bench_function("3_access_shared_root", |b| b.iter(|| query(&ctx, &rt, sql)));
294+
295+
// Five accesses sharing the struct root `s`, amplifying planning cost.
296+
let sql = "select id from t \
297+
where (s['a'] + s['b'] + s['c'] + s['d'] + s['e']) = 25";
298+
assert_pushdown_active(&ctx, &rt, "5_access_shared_root", sql);
299+
c.bench_function("5_access_shared_root", |b| b.iter(|| query(&ctx, &rt, sql)));
300+
301+
// Two accesses sharing the deeper prefix `s.inner`.
302+
let sql = "select id from t where (s['inner']['x'] + s['inner']['y']) = 10";
303+
assert_pushdown_active(&ctx, &rt, "2_access_shared_nested_prefix", sql);
304+
c.bench_function("2_access_shared_nested_prefix", |b| {
305+
b.iter(|| query(&ctx, &rt, sql))
306+
});
307+
308+
// Three accesses sharing the deeper prefix `s.inner`.
309+
let sql = "select id from t \
310+
where (s['inner']['x'] + s['inner']['y'] + s['inner']['z']) = 15";
311+
assert_pushdown_active(&ctx, &rt, "3_access_shared_nested_prefix", sql);
312+
c.bench_function("3_access_shared_nested_prefix", |b| {
313+
b.iter(|| query(&ctx, &rt, sql))
314+
});
315+
316+
// Mix: two accesses on `s` leaves and two on `s.inner` leaves.
317+
let sql = "select id from t \
318+
where (s['a'] + s['b'] + s['inner']['x'] + s['inner']['y']) = 20";
319+
assert_pushdown_active(&ctx, &rt, "mixed_depth_shared_prefix", sql);
320+
c.bench_function("mixed_depth_shared_prefix", |b| {
321+
b.iter(|| query(&ctx, &rt, sql))
322+
});
323+
324+
// Temporary file must outlive the benchmarks, it is deleted when dropped
325+
drop(temp_file);
326+
}
327+
328+
criterion_group!(benches, criterion_benchmark);
329+
criterion_main!(benches);

‎datafusion/core/tests/memory_limit/mod.rs‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -101,7 +101,8 @@ async fn group_by_row_hash() {
101101
TestCase::new()
102102
.with_query("select count(*) from t GROUP BY response_bytes")
103103
.with_expected_errors(vec![
104-
"Resources exhausted: Additional allocation failed", "with top memory consumers (across reservations) as:\n GroupedHashAggregateStream"
104+
"Resources exhausted: Additional allocation failed",
105+
"for FinalHashAggregateStream[0]",
105106
])
106107
.with_memory_limit(2_000)
107108
.run()
@@ -114,7 +115,8 @@ async fn group_by_hash() {
114115
// group by dict column
115116
.with_query("select count(*) from t GROUP BY service, host, pod, container")
116117
.with_expected_errors(vec![
117-
"Resources exhausted: Additional allocation failed", "with top memory consumers (across reservations) as:\n GroupedHashAggregateStream"
118+
"Resources exhausted: Additional allocation failed",
119+
"for PartialHashAggregateStream[0]",
118120
])
119121
.with_memory_limit(1_000)
120122
.run()
@@ -425,7 +427,7 @@ async fn oom_grouped_hash_aggregate() {
425427
.with_query("SELECT COUNT(*), SUM(request_bytes) FROM t GROUP BY host")
426428
.with_expected_errors(vec![
427429
"Failed to allocate additional",
428-
"GroupedHashAggregateStream[0] (count(1), sum(t.request_bytes))",
430+
"for PartialHashAggregateStream[0]",
429431
])
430432
.with_memory_limit(1_000)
431433
.run()

‎datafusion/datasource/Cargo.toml‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -34,10 +34,12 @@ all-features = true
3434
backtrace = ["datafusion-common/backtrace"]
3535
compression = ["async-compression", "liblzma", "bzip2", "flate2", "zstd", "tokio-util"]
3636
default = ["compression"]
37-
# Enables the protobuf conversions for the file-scan leaf types owned by this
38-
# crate (`FileRange`, `PartitionedFile`, `FileGroup`). Off by default so
39-
# consumers that never serialize plans pay nothing.
40-
proto = ["dep:datafusion-proto-models"]
37+
# Enables protobuf conversions for datasource types and serialization hooks.
38+
# Off by default so consumers that never serialize plans pay nothing.
39+
proto = [
40+
"dep:datafusion-proto-models",
41+
"datafusion-physical-plan/proto",
42+
]
4143

4244
[dependencies]
4345
arrow = { workspace = true }

‎datafusion/datasource/src/file_sink_config.rs‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,9 @@ use datafusion_expr::dml::InsertOp;
3232
use async_trait::async_trait;
3333
use object_store::ObjectStore;
3434

35+
#[cfg(feature = "proto")]
36+
mod proto;
37+
3538
/// Determines how `FileSink` output paths are interpreted.
3639
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
3740
pub enum FileOutputMode {

0 commit comments

Comments
 (0)