Skip to content

Commit 20287fc

Browse files
committed
test: run microbenchmarks in extended CI
Execute Criterion benchmarks in test mode with debug checks and a small planner fixture. Repair stale benchmark argument and return types exposed by the new check. Closes apache#24868
1 parent 6c1727f commit 20287fc

8 files changed

Lines changed: 147 additions & 36 deletions

File tree

‎.asf.yaml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,7 @@ github:
6969
- "cargo check datafusion-spark features"
7070
- "cargo test (amd64)"
7171
- "cargo test 'extended_tests' (amd64)"
72+
- "microbenchmark tests (amd64)"
7273
- "cargo test hash collisions (amd64)"
7374
- "cargo test datafusion-cli (amd64)"
7475
- "cargo examples (amd64)"

‎.github/workflows/extended.yml‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -90,6 +90,28 @@ jobs:
9090
- name: Cleanup
9191
run: cargo clean
9292

93+
# Execute Criterion benchmarks once to catch panics without measuring performance.
94+
linux-test-microbenchmarks:
95+
name: microbenchmark tests (amd64)
96+
# Long-running job: run in the merge queue, on release pushes, or manually.
97+
if: github.event_name != 'pull_request'
98+
runs-on: ${{ vars.USE_RUNS_ON == 'true' && format('runs-on={0},family=m8a+m7a+c8a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }}
99+
container:
100+
image: amd64/rust
101+
steps:
102+
- uses: runs-on/action@efac073ea2507ec18797de3a81704201ade11d9d # v2.3.1
103+
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
104+
with:
105+
ref: ${{ github.event.inputs.pr_head_sha }} # otherwise use the event's commit, including the merge group
106+
submodules: true
107+
fetch-depth: 1
108+
- name: Setup Rust toolchain
109+
uses: ./.github/actions/setup-builder
110+
with:
111+
rust-version: stable
112+
- name: Test microbenchmarks
113+
run: cargo xtask ci step test microbenchmarks
114+
93115
# Check answers are correct when hash values collide
94116
hash-collisions:
95117
name: cargo test hash collisions (amd64)

‎Cargo.toml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -408,3 +408,8 @@ debug = false
408408
debug-assertions = false
409409
strip = "debuginfo"
410410
incremental = false
411+
412+
# Execute benchmark tests with debug checks, without release-mode build costs.
413+
[profile.ci-bench]
414+
inherits = "ci"
415+
opt-level = 1

‎datafusion/core/benches/sql_planner.rs‎

Lines changed: 18 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -203,17 +203,10 @@ fn register_defs(ctx: SessionContext, defs: Vec<TableDef>) -> SessionContext {
203203
ctx
204204
}
205205

206-
fn register_clickbench_hits_table(rt: &Runtime) -> SessionContext {
206+
fn register_clickbench_hits_table(rt: &Runtime, path: &str) -> SessionContext {
207207
let ctx = SessionContext::new();
208208

209209
// use an external table for clickbench benchmarks
210-
let path =
211-
if PathBuf::from(format!("{BENCHMARKS_PATH_1}{CLICKBENCH_DATA_PATH}")).exists() {
212-
format!("{BENCHMARKS_PATH_1}{CLICKBENCH_DATA_PATH}")
213-
} else {
214-
format!("{BENCHMARKS_PATH_2}{CLICKBENCH_DATA_PATH}")
215-
};
216-
217210
let sql =
218211
format!("CREATE EXTERNAL TABLE hits_raw STORED AS PARQUET LOCATION '{path}'");
219212

@@ -355,15 +348,23 @@ fn union_orderby_query(n: usize) -> String {
355348
}
356349

357350
fn criterion_benchmark(c: &mut Criterion) {
351+
// CI uses the small checked-in fixture; performance runs use the full dataset.
352+
let clickbench_data_path =
353+
std::env::var("CLICKBENCH_DATA_PATH").unwrap_or_else(|_| {
354+
let path = format!("{BENCHMARKS_PATH_1}{CLICKBENCH_DATA_PATH}");
355+
if PathBuf::from(&path).exists() {
356+
path
357+
} else {
358+
format!("{BENCHMARKS_PATH_2}{CLICKBENCH_DATA_PATH}")
359+
}
360+
});
358361
// verify that we can load the clickbench data prior to running the benchmark
359-
if !PathBuf::from(format!("{BENCHMARKS_PATH_1}{CLICKBENCH_DATA_PATH}")).exists()
360-
&& !PathBuf::from(format!("{BENCHMARKS_PATH_2}{CLICKBENCH_DATA_PATH}")).exists()
361-
{
362-
panic!(
363-
"benchmarks/data/hits_partitioned/ could not be loaded. Please run \
364-
'benchmarks/bench.sh data clickbench_partitioned' prior to running this benchmark"
365-
)
366-
}
362+
assert!(
363+
PathBuf::from(&clickbench_data_path).exists(),
364+
"{clickbench_data_path} could not be loaded. Please run \
365+
'benchmarks/bench.sh data clickbench_partitioned' prior to running this benchmark \
366+
or set CLICKBENCH_DATA_PATH to an existing dataset"
367+
);
367368

368369
let ctx = create_context();
369370
let rt = Runtime::new().unwrap();
@@ -693,7 +694,7 @@ fn criterion_benchmark(c: &mut Criterion) {
693694
)))
694695
.collect::<Vec<_>>();
695696

696-
let clickbench_ctx = register_clickbench_hits_table(&rt);
697+
let clickbench_ctx = register_clickbench_hits_table(&rt, &clickbench_data_path);
697698

698699
// for (i, sql) in clickbench_queries.iter().enumerate() {
699700
// c.bench_function(&format!("logical_plan_clickbench_q{}", i + 1), |b| {

‎datafusion/functions/benches/datetime_expressions/to_timestamp.rs‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ use std::sync::Arc;
2121
use arrow::array::builder::StringBuilder;
2222
use arrow::array::{Array, ArrayRef, StringArray};
2323
use arrow::compute::cast;
24-
use arrow::datatypes::{DataType, Field, TimeUnit};
24+
use arrow::datatypes::{DataType, Field};
2525
use criterion::{Criterion, criterion_group};
2626
use datafusion_common::config::ConfigOptions;
2727
use datafusion_expr::{ColumnarValue, ScalarFunctionArgs};
@@ -108,15 +108,19 @@ fn data_with_formats() -> (StringArray, StringArray, StringArray, StringArray) {
108108
)
109109
}
110110
fn criterion_benchmark(c: &mut Criterion) {
111-
let return_field =
112-
Field::new("f", DataType::Timestamp(TimeUnit::Nanosecond, None), true).into();
113111
let arg_field = Field::new("a", DataType::Utf8, false).into();
114112
let arg_fields = vec![arg_field];
115113
let mut options = ConfigOptions::default();
116114
options.execution.time_zone = Some("UTC".parse().unwrap());
117115
let config_options = Arc::new(options);
118116

119117
let to_timestamp_udf = to_timestamp(config_options.as_ref());
118+
let return_field = Field::new(
119+
"f",
120+
to_timestamp_udf.return_type(&[DataType::Utf8]).unwrap(),
121+
true,
122+
)
123+
.into();
120124

121125
c.bench_function("to_timestamp_no_formats_utf8", |b| {
122126
let to_timestamp_udf = Arc::clone(&to_timestamp_udf);

‎datafusion/functions/benches/string_expressions/repeat.rs‎

Lines changed: 25 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -60,7 +60,6 @@ fn create_args<O: OffsetSizeTrait>(
6060

6161
fn invoke_repeat_with_args(
6262
args: Vec<ColumnarValue>,
63-
repeat_times: i64,
6463
) -> Result<ColumnarValue, DataFusionError> {
6564
let arg_fields = args
6665
.iter()
@@ -69,11 +68,22 @@ fn invoke_repeat_with_args(
6968
.collect::<Vec<_>>();
7069
let config_options = Arc::new(ConfigOptions::default());
7170

71+
let return_type = string::repeat().return_type(
72+
&args
73+
.iter()
74+
.map(ColumnarValue::data_type)
75+
.collect::<Vec<_>>(),
76+
)?;
77+
let number_rows = match &args[0] {
78+
ColumnarValue::Array(array) => array.len(),
79+
ColumnarValue::Scalar(_) => 1,
80+
};
81+
7282
string::repeat().invoke_with_args(ScalarFunctionArgs {
7383
args,
7484
arg_fields,
75-
number_rows: repeat_times as usize,
76-
return_field: Field::new("f", DataType::Utf8, true).into(),
85+
number_rows,
86+
return_field: Field::new("f", return_type, true).into(),
7787
config_options: Arc::clone(&config_options),
7888
})
7989
}
@@ -111,7 +121,7 @@ fn criterion_benchmark(c: &mut Criterion) {
111121
Field::new("b", DataType::Int64, false).into(),
112122
],
113123
number_rows: 1,
114-
return_field: Field::new("f", DataType::Utf8, true).into(),
124+
return_field: Field::new("f", DataType::Utf8View, true).into(),
115125
config_options: Arc::clone(&config_options),
116126
};
117127
b.iter(|| black_box(repeat_fn.invoke_with_args(args.clone()).unwrap()))
@@ -131,7 +141,7 @@ fn criterion_benchmark(c: &mut Criterion) {
131141
|b| {
132142
b.iter(|| {
133143
let args_cloned = args.clone();
134-
black_box(invoke_repeat_with_args(args_cloned, repeat_times))
144+
black_box(invoke_repeat_with_args(args_cloned).unwrap())
135145
})
136146
},
137147
);
@@ -142,7 +152,7 @@ fn criterion_benchmark(c: &mut Criterion) {
142152
|b| {
143153
b.iter(|| {
144154
let args_cloned = args.clone();
145-
black_box(invoke_repeat_with_args(args_cloned, repeat_times))
155+
black_box(invoke_repeat_with_args(args_cloned).unwrap())
146156
})
147157
},
148158
);
@@ -153,7 +163,7 @@ fn criterion_benchmark(c: &mut Criterion) {
153163
|b| {
154164
b.iter(|| {
155165
let args_cloned = args.clone();
156-
black_box(invoke_repeat_with_args(args_cloned, repeat_times))
166+
black_box(invoke_repeat_with_args(args_cloned).unwrap())
157167
})
158168
},
159169
);
@@ -173,7 +183,7 @@ fn criterion_benchmark(c: &mut Criterion) {
173183
|b| {
174184
b.iter(|| {
175185
let args_cloned = args.clone();
176-
black_box(invoke_repeat_with_args(args_cloned, repeat_times))
186+
black_box(invoke_repeat_with_args(args_cloned).unwrap())
177187
})
178188
},
179189
);
@@ -184,7 +194,7 @@ fn criterion_benchmark(c: &mut Criterion) {
184194
|b| {
185195
b.iter(|| {
186196
let args_cloned = args.clone();
187-
black_box(invoke_repeat_with_args(args_cloned, repeat_times))
197+
black_box(invoke_repeat_with_args(args_cloned).unwrap())
188198
})
189199
},
190200
);
@@ -195,7 +205,7 @@ fn criterion_benchmark(c: &mut Criterion) {
195205
|b| {
196206
b.iter(|| {
197207
let args_cloned = args.clone();
198-
black_box(invoke_repeat_with_args(args_cloned, repeat_times))
208+
black_box(invoke_repeat_with_args(args_cloned).unwrap())
199209
})
200210
},
201211
);
@@ -215,7 +225,11 @@ fn criterion_benchmark(c: &mut Criterion) {
215225
|b| {
216226
b.iter(|| {
217227
let args_cloned = args.clone();
218-
black_box(invoke_repeat_with_args(args_cloned, repeat_times))
228+
black_box(
229+
invoke_repeat_with_args(args_cloned).expect_err(
230+
"repeat should reject an overflowing output size",
231+
),
232+
)
219233
})
220234
},
221235
);

‎datafusion/spark/benches/slice.rs‎

Lines changed: 16 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,8 @@ use arrow::datatypes::{DataType, Field, Int64Type};
2323
use criterion::{Criterion, criterion_group, criterion_main};
2424
use datafusion_common::ScalarValue;
2525
use datafusion_common::config::ConfigOptions;
26-
use datafusion_expr::{ColumnarValue, ScalarFunctionArgs};
26+
use datafusion_expr::type_coercion::functions::fields_with_udf;
27+
use datafusion_expr::{ColumnarValue, ReturnFieldArgs, ScalarFunctionArgs};
2728
use datafusion_spark::function::array::slice;
2829
use rand::rngs::StdRng;
2930
use rand::{Rng, SeedableRng};
@@ -115,6 +116,19 @@ fn array_slice_benchmark(
115116
<Arc<Field>>::from(Field::new(format!("arg_{idx}"), arg.data_type(), true))
116117
})
117118
.collect::<Vec<_>>();
119+
// Apply planner coercion (including ListView to List) before timing the UDF.
120+
let arg_fields = fields_with_udf(&arg_fields, array_slice.as_ref()).unwrap();
121+
let args = args
122+
.iter()
123+
.zip(&arg_fields)
124+
.map(|(arg, field)| arg.cast_to(field.data_type(), None).unwrap())
125+
.collect::<Vec<_>>();
126+
let return_field = array_slice
127+
.return_field_from_args(ReturnFieldArgs {
128+
arg_fields: &arg_fields,
129+
scalar_arguments: &vec![None; arg_fields.len()],
130+
})
131+
.unwrap();
118132
c.bench_function(name, |b| {
119133
b.iter(|| {
120134
black_box(
@@ -123,8 +137,7 @@ fn array_slice_benchmark(
123137
args: args.clone(),
124138
arg_fields: arg_fields.clone(),
125139
number_rows: size,
126-
return_field: Field::new_list_field(args[0].data_type(), true)
127-
.into(),
140+
return_field: Arc::clone(&return_field),
128141
config_options: Arc::new(ConfigOptions::default()),
129142
})
130143
.unwrap(),

‎xtask/src/ci_steps.rs‎

Lines changed: 53 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,7 @@ static CI_STEPS: &[StepInfo] = &[
5555
},
5656
StepInfo {
5757
command: "test",
58-
help_usage: "test <workspace|cli|doctest|ffi|benchmark-plan|benchmark-sqllogic|postgres|substrait|substrait-optimized|extended|hash-collisions|sqlite> [--explain]",
58+
help_usage: "test <workspace|cli|doctest|ffi|microbenchmarks|benchmark-plan|benchmark-sqllogic|postgres|substrait|substrait-optimized|extended|hash-collisions|sqlite> [--explain]",
5959
help_examples: &["test cli", "test workspace"],
6060
help_description: "Run a CI test suite",
6161
error_message: "Cargo test step failed",
@@ -288,6 +288,38 @@ impl StepContext {
288288
"integration-tests",
289289
]);
290290
}
291+
"microbenchmarks" => {
292+
// The SQL benchmark harness needs separately generated datasets
293+
// and is covered by the benchmark result verification job.
294+
// Select only bench targets (not lib/bin unit tests), and tell
295+
// Criterion to execute each benchmark once instead of timing it.
296+
// Keep debug assertions and overflow checks enabled without the
297+
// cost of unoptimized execution on the larger benchmark datasets.
298+
command.env(
299+
"CLICKBENCH_DATA_PATH",
300+
self.root
301+
.join("datafusion/core/tests/data/clickbench_hits_10.parquet"),
302+
);
303+
command.args([
304+
"test",
305+
"--profile",
306+
"ci-bench",
307+
"--no-fail-fast",
308+
"--workspace",
309+
"--exclude",
310+
"datafusion-examples",
311+
"--exclude",
312+
"datafusion-benchmarks",
313+
"--exclude",
314+
"datafusion-cli",
315+
"--bench",
316+
"*",
317+
"--features",
318+
"datafusion-physical-plan/test_utils",
319+
"--",
320+
"--test",
321+
]);
322+
}
291323
"benchmark-plan" => {
292324
command
293325
.args([
@@ -614,7 +646,7 @@ fn required_env(name: &str) -> Result<String> {
614646
mod tests {
615647
use super::*;
616648

617-
const TEST_USAGE: &str = "usage: cargo xtask ci step test <workspace|cli|doctest|ffi|benchmark-plan|benchmark-sqllogic|postgres|substrait|substrait-optimized|extended|hash-collisions|sqlite> [--explain]";
649+
const TEST_USAGE: &str = "usage: cargo xtask ci step test <workspace|cli|doctest|ffi|microbenchmarks|benchmark-plan|benchmark-sqllogic|postgres|substrait|substrait-optimized|extended|hash-collisions|sqlite> [--explain]";
618650

619651
fn args(args: &[&str]) -> Vec<String> {
620652
args.iter().map(|arg| (*arg).to_string()).collect()
@@ -793,6 +825,25 @@ cargo test \
793825
--lib \
794826
--tests \
795827
--features integration-tests
828+
")
829+
},
830+
},
831+
Cmd {
832+
command: &["test", "microbenchmarks"],
833+
expected: |actual| {
834+
insta::assert_snapshot!(actual, @r"
835+
cd /workspace && \
836+
CLICKBENCH_DATA_PATH=/workspace/datafusion/core/tests/data/clickbench_hits_10.parquet \
837+
cargo test \
838+
--profile ci-bench \
839+
--no-fail-fast \
840+
--workspace \
841+
--exclude datafusion-examples \
842+
--exclude datafusion-benchmarks \
843+
--exclude datafusion-cli \
844+
--bench '*' \
845+
--features datafusion-physical-plan/test_utils \
846+
-- --test
796847
")
797848
},
798849
},

0 commit comments

Comments
 (0)