Skip to content

Commit fb4aa45

Browse files
Omega359alamb
andauthored
Add basic sql benchmark runner for running sql benchmarks (apache#23052)
## Which issue does this PR close? <!-- We generally require a GitHub issue to be filed for all bug fixes and enhancements and this helps us generate change logs for our releases. You can link an issue to this PR using the GitHub syntax. For example `Closes apache#123` indicates that this PR will close issue apache#123. --> - Part of apache#21937. ## Rationale for this change Running sql benchmarks using environment variables for configuration is awkward and error prone and strictly using criterion, while statistically much better, is quite slow compared to using simple iterations. This PR is the first version of a benchmark runner for sql benchmarks that will eventually use arguments for all benchmark configuration options. ## What changes are included in this PR? A simple benchmark runner that can list out the sql benchmarks and run a benchmark using iterations or criterion allowing for specifying a single query if desired. Future enhancements will use arguments for benchmark configuration vs just using environment variables as well as providing help and tying this into bench.sh ## Are these changes tested? Yes. I have a script that tests all current sql benchmarks both with and without criterion. Here is an portion of it for the single clickbench benchmark: ``` # clickbench single basic long flags env DATA_DIR=data CLICKBENCH_TYPE=single cargo run -p datafusion-benchmarks --bin benchmark_runner -- clickbench --query 0 --iterations 5 --output results/benchmark_runner/clickbench_single_long.json # clickbench single basic short flags env DATA_DIR=data CLICKBENCH_TYPE=single cargo run -p datafusion-benchmarks --bin benchmark_runner -- clickbench --query 0 -i 5 -o results/benchmark_runner/clickbench_single_short.json # clickbench single basic env iterations env DATA_DIR=data CLICKBENCH_TYPE=single ITERATIONS=5 cargo run -p datafusion-benchmarks --bin benchmark_runner -- clickbench --query 0 --output results/benchmark_runner/clickbench_single_env_iterations.json # clickbench single criterion with baseline env DATA_DIR=data CLICKBENCH_TYPE=single cargo run -p datafusion-benchmarks --bin benchmark_runner -- clickbench --query 0 --criterion --save-baseline benchmark_runner_acceptance # clickbench single criterion without baseline env DATA_DIR=data CLICKBENCH_TYPE=single cargo run -p datafusion-benchmarks --bin benchmark_runner -- clickbench --query 0 --criterion ``` The existing `cargo bench` approach still works the same (criterion only): ``` env DATA_DIR=data CLICKBENCH_TYPE=single BENCH_NAME=clickbench BENCH_QUERY=0 cargo bench -p datafusion-benchmarks --bench sql` ``` ## Are there any user-facing changes? No. --------- Co-authored-by: Andrew Lamb <[email protected]>
1 parent ed37b6c commit fb4aa45

7 files changed

Lines changed: 1558 additions & 248 deletions

File tree

‎Cargo.lock‎

Lines changed: 1 addition & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎benchmarks/Cargo.toml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,7 @@ clap = { version = "4.6.0", features = ["derive", "env"] }
4747
criterion = { workspace = true, features = ["html_reports"] }
4848
datafusion = { workspace = true, default-features = true }
4949
datafusion-common = { workspace = true, default-features = true }
50+
datafusion-common-runtime = { workspace = true }
5051
env_logger = { workspace = true }
5152
futures = { workspace = true }
5253
libmimalloc-sys = { version = "0.1", optional = true }

‎benchmarks/benches/sql.rs‎

Lines changed: 20 additions & 239 deletions
Original file line numberDiff line numberDiff line change
@@ -22,26 +22,13 @@
2222
//! Cargo, for example: `BENCH_NAME=tpch cargo bench --bench sql`.
2323
2424
use clap::Parser;
25-
use criterion::{Criterion, SamplingMode, criterion_group, criterion_main};
26-
use datafusion::error::Result;
27-
use datafusion::prelude::SessionContext;
28-
use datafusion_benchmarks::sql_benchmark::SqlBenchmark;
29-
use datafusion_benchmarks::util::{CommonOpt, print_memory_stats};
25+
use criterion::{Criterion, criterion_group, criterion_main};
26+
use datafusion_benchmarks::sql_benchmark_runner::{
27+
BenchmarkFilter, SqlRunConfig, default_sql_benchmark_directory,
28+
run_criterion_benchmarks_impl,
29+
};
30+
use datafusion_benchmarks::util::CommonOpt;
3031
use datafusion_common::instant::Instant;
31-
use log::{debug, info};
32-
use std::collections::BTreeMap;
33-
use std::fs;
34-
use std::sync::LazyLock;
35-
use tokio::runtime::Runtime;
36-
37-
static SQL_BENCHMARK_DIRECTORY: LazyLock<String> = LazyLock::new(|| {
38-
format!(
39-
"{}{}{}",
40-
env!("CARGO_MANIFEST_DIR"),
41-
std::path::MAIN_SEPARATOR,
42-
"sql_benchmarks"
43-
)
44-
});
4532

4633
#[cfg(feature = "snmalloc")]
4734
#[global_allocator]
@@ -90,234 +77,28 @@ pub fn sql(c: &mut Criterion) {
9077

9178
let start = Instant::now();
9279
let args = EnvParser::parse();
93-
let rt = make_tokio_runtime();
80+
let config = SqlRunConfig {
81+
common: args.options,
82+
filter: BenchmarkFilter {
83+
name: args.name,
84+
subgroup: args.subgroup,
85+
query: args.query,
86+
},
87+
persist_results: args.persist_results,
88+
validate_results: args.validate,
89+
output: None,
90+
};
9491

9592
println!("Loading benchmarks...");
9693

97-
let benchmarks = rt.block_on(async {
98-
let ctx = make_ctx(&args).expect("SessionContext creation failed");
99-
100-
load_benchmarks(&args, &ctx, &SQL_BENCHMARK_DIRECTORY)
101-
.await
102-
.unwrap_or_else(|err| panic!("failed load benchmarks: {err:?}"))
103-
});
94+
run_criterion_benchmarks_impl(&default_sql_benchmark_directory(), &config, c)
95+
.unwrap_or_else(|err| panic!("failed to run SQL benchmarks: {err:?}"));
10496

10597
println!(
106-
"Loaded benchmarks in {} ms ...",
98+
"Completed benchmarks in {} ms ...",
10799
start.elapsed().as_millis()
108100
);
109-
110-
for (group, benchmarks) in benchmarks {
111-
let mut group = c.benchmark_group(group);
112-
group.sample_size(10);
113-
group.sampling_mode(SamplingMode::Flat);
114-
115-
for mut benchmark in benchmarks {
116-
// create a context
117-
let ctx = make_ctx(&args).expect("SessionContext creation failed");
118-
119-
// initialize the benchmark. This parses the benchmark file and does any pre-execution
120-
// work such as loading data into tables
121-
rt.block_on(async {
122-
benchmark
123-
.initialize(&ctx)
124-
.await
125-
.expect("initialization failed");
126-
127-
// run assertions
128-
benchmark.assert(&ctx).await.expect("assertion failed");
129-
});
130-
131-
let mut name = benchmark.name().to_string();
132-
if !benchmark.subgroup().is_empty() {
133-
name.push('_');
134-
name.push_str(benchmark.subgroup());
135-
}
136-
137-
if args.persist_results {
138-
handle_persist(&rt, &ctx, &name, &mut benchmark);
139-
} else if args.validate {
140-
handle_verify(&rt, &ctx, &name, &mut benchmark);
141-
} else {
142-
info!("Running benchmark {name} ...");
143-
144-
let name = name.clone();
145-
group.bench_function(name.clone(), |b| {
146-
b.iter(|| handle_run(&rt, &ctx, &args, &mut benchmark, &name))
147-
});
148-
149-
print_memory_stats();
150-
151-
info!("Benchmark {name} completed");
152-
}
153-
154-
// run cleanup
155-
rt.block_on(async {
156-
benchmark.cleanup(&ctx).await.expect("Cleanup failed");
157-
});
158-
}
159-
160-
group.finish();
161-
}
162-
}
163-
164-
fn handle_run(
165-
rt: &Runtime,
166-
ctx: &SessionContext,
167-
args: &EnvParser,
168-
benchmark: &mut SqlBenchmark,
169-
name: &str,
170-
) {
171-
rt.block_on(async {
172-
benchmark
173-
.run(ctx, args.validate)
174-
.await
175-
.unwrap_or_else(|err| panic!("Failed to run benchmark {name}: {err:?}"))
176-
});
177-
}
178-
179-
fn handle_persist(
180-
rt: &Runtime,
181-
ctx: &SessionContext,
182-
name: &str,
183-
benchmark: &mut SqlBenchmark,
184-
) {
185-
info!("Running benchmark {name} prior to persisting results ...");
186-
187-
rt.block_on(async {
188-
info!("Persisting benchmark {name} ...");
189-
190-
benchmark
191-
.persist(ctx)
192-
.await
193-
.expect("Failed to persist results");
194-
});
195-
196-
info!("Persisted benchmark {name} successfully");
197-
}
198-
199-
fn handle_verify(
200-
rt: &Runtime,
201-
ctx: &SessionContext,
202-
name: &str,
203-
benchmark: &mut SqlBenchmark,
204-
) {
205-
info!("Verifying benchmark {name} results ...");
206-
207-
rt.block_on(async {
208-
benchmark
209-
.run(ctx, true)
210-
.await
211-
.unwrap_or_else(|err| panic!("Failed to run benchmark {name}: {err:?}"));
212-
benchmark
213-
.verify(ctx)
214-
.await
215-
.unwrap_or_else(|err| panic!("Verification failed: {err:?}"));
216-
});
217-
218-
info!("Verified benchmark {name} results successfully");
219101
}
220102

221103
criterion_group!(benches, sql);
222104
criterion_main!(benches);
223-
224-
fn make_tokio_runtime() -> Runtime {
225-
tokio::runtime::Builder::new_multi_thread()
226-
.enable_all()
227-
.build()
228-
.unwrap()
229-
}
230-
231-
fn make_ctx(args: &EnvParser) -> Result<SessionContext> {
232-
let config = args.options.config()?;
233-
let rt = args.options.build_runtime()?;
234-
235-
Ok(SessionContext::new_with_config_rt(config, rt))
236-
}
237-
238-
/// Recursively walks the directory tree starting at `path` and
239-
/// calls the call back function for every file encountered.
240-
pub fn list_files<F>(path: &str, callback: &mut F)
241-
where
242-
F: FnMut(&str),
243-
{
244-
let mut entries: Vec<fs::DirEntry> =
245-
fs::read_dir(path).unwrap().filter_map(Result::ok).collect();
246-
entries.sort_by_key(|entry| entry.path());
247-
248-
for dir_entry in entries {
249-
let path = dir_entry.path();
250-
if path.is_dir() {
251-
// Recurse into the sub‑directory
252-
list_files(&path.to_string_lossy(), callback);
253-
} else {
254-
// For files, invoke the callback with the full path as a string
255-
let full_str = path.to_string_lossy();
256-
callback(&full_str);
257-
}
258-
}
259-
}
260-
261-
/// Loads all benchmark files in the `sql_benchmarks` directory.
262-
/// For each file ending with `.benchmark` it creates a new
263-
/// `SqlBenchmark` instance.
264-
async fn load_benchmarks(
265-
args: &EnvParser,
266-
ctx: &SessionContext,
267-
path: &str,
268-
) -> Result<BTreeMap<String, Vec<SqlBenchmark>>> {
269-
let mut benches = BTreeMap::new();
270-
let mut paths = Vec::new();
271-
272-
list_files(path, &mut |path: &str| {
273-
if path.ends_with(".benchmark") {
274-
paths.push(path.to_string());
275-
}
276-
});
277-
278-
for path in paths {
279-
debug!("Loading benchmark from {path}");
280-
281-
let benchmark = SqlBenchmark::new(ctx, &path, &*SQL_BENCHMARK_DIRECTORY).await?;
282-
let entries = benches
283-
.entry(benchmark.group().to_string())
284-
.or_insert(vec![]);
285-
286-
entries.push(benchmark);
287-
}
288-
289-
benches = filter_benchmarks(args, benches);
290-
benches.iter_mut().for_each(|(_, benchmarks)| {
291-
benchmarks.sort_by(|b1, b2| b1.name().cmp(b2.name()))
292-
});
293-
294-
Ok(benches)
295-
}
296-
297-
fn filter_benchmarks(
298-
args: &EnvParser,
299-
benchmarks: BTreeMap<String, Vec<SqlBenchmark>>,
300-
) -> BTreeMap<String, Vec<SqlBenchmark>> {
301-
match &args.name {
302-
Some(bench_name) => benchmarks
303-
.into_iter()
304-
.filter(|(key, _val)| key.eq_ignore_ascii_case(bench_name))
305-
.map(|(key, mut val)| {
306-
if let Some(subgroup) = &args.subgroup {
307-
val.retain(|bench| bench.subgroup().eq_ignore_ascii_case(subgroup));
308-
}
309-
if let Some(query) = &args.query {
310-
// Accept `1`, `01`, `6a`, `Q06a`, ... case-insensitively.
311-
// Bench names are canonical, e.g. `Q01`, `Q06a`.
312-
let q = query.trim_start_matches(['Q', 'q']);
313-
let split = q.find(|c: char| !c.is_ascii_digit()).unwrap_or(q.len());
314-
let (num, suffix) = q.split_at(split);
315-
let normalized = format!("Q{num:0>2}{suffix}");
316-
val.retain(|bench| bench.name().eq_ignore_ascii_case(&normalized));
317-
}
318-
(key, val)
319-
})
320-
.collect(),
321-
None => benchmarks,
322-
}
323-
}
Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
//! DataFusion SQL benchmark runner.
19+
20+
use datafusion_benchmarks::sql_benchmark_runner;
21+
22+
#[cfg(feature = "snmalloc")]
23+
#[global_allocator]
24+
static ALLOC: snmalloc_rs::SnMalloc = snmalloc_rs::SnMalloc;
25+
26+
// `cargo clippy --all-features` enables both allocator features, so prefer
27+
// `snmalloc` in that case and fall back to `mimalloc` otherwise.
28+
#[cfg(all(not(feature = "snmalloc"), feature = "mimalloc"))]
29+
#[global_allocator]
30+
static ALLOC: mimalloc::MiMalloc = mimalloc::MiMalloc;
31+
32+
#[tokio::main]
33+
async fn main() {
34+
env_logger::init();
35+
if let Err(error) = sql_benchmark_runner::run_cli().await {
36+
eprintln!("Error: {error}");
37+
std::process::exit(1);
38+
}
39+
}

‎benchmarks/src/lib.rs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ pub mod smj;
2727
pub mod sort_pushdown;
2828
pub mod sort_tpch;
2929
pub mod sql_benchmark;
30+
pub mod sql_benchmark_runner;
3031
pub mod tpcds;
3132
pub mod tpch;
3233
pub mod util;

0 commit comments

Comments
 (0)