Skip to content

Commit 925cca1

Browse files
Add small-batch IN LIST dictionary benchmarks
Cover plain and dictionary Int32 inputs from one row through the standard 8,192-row batch. The cases use both short and longer IN lists so fixed per-batch costs are visible without losing the usual vectorized comparison.
1 parent 46f8df1 commit 925cca1

1 file changed

Lines changed: 56 additions & 1 deletion

File tree

‎datafusion/physical-expr/benches/in_list_strategy.rs‎

Lines changed: 56 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@
4545
//! | Utf8View length-12 cases | Utf8View | 12-byte strings | 16, 64 |
4646
//! | Utf8View long-string cases | Utf8View | 24-byte strings | 4, 16, 64, 256 |
4747
//! | Shared-prefix string cases | Utf8, Utf8View | same prefix, different suffix | 16, 32, 64 |
48+
//! | Dictionary dispatch cases | Int32, Dictionary<Int32> | per-batch dispatch overhead | 4, 64 |
4849
//! | Fixed-size binary direct-comparison case | FixedSizeBinary(1) | direct-comparison cutoff | 16 |
4950
//! | Fixed-size binary direct-comparison case | FixedSizeBinary(16) | direct-comparison cutoff | 4 |
5051
//! | Fixed-size binary bitmap case | FixedSizeBinary(2) | bitmap lookup | 64 |
@@ -871,6 +872,60 @@ fn bench_dictionary(c: &mut Criterion) {
871872
bench_dict_string(c, "utf8_short/dict=500/list=20", 500, 20, 10);
872873
}
873874

875+
/// Measures the fixed per-batch cost of handling dictionary and plain inputs.
876+
/// Small batches make dispatch overhead visible, while the standard 8,192-row
877+
/// batch shows whether it matters in the usual vectorized case.
878+
fn bench_dictionary_dispatch(c: &mut Criterion) {
879+
for list_size in [4_usize, 64] {
880+
let strategy = if list_size == 4 {
881+
"branchless"
882+
} else {
883+
"hash_set"
884+
};
885+
let haystack = (0..list_size as i32).collect::<Vec<_>>();
886+
887+
for batch_size in [1, 8, 64, ARRAY_SIZE] {
888+
let keys = (0..batch_size)
889+
.map(|index| (index % 8) as i32)
890+
.collect::<Vec<_>>();
891+
let values = keys.iter().map(|key| key * 2).collect::<Vec<_>>();
892+
893+
for dictionary in [false, true] {
894+
let array: ArrayRef = if dictionary {
895+
Arc::new(
896+
DictionaryArray::<Int32Type>::try_new(
897+
Int32Array::from(keys.clone()),
898+
Arc::new(Int32Array::from_iter_values((0..8).map(|v| v * 2))),
899+
)
900+
.unwrap(),
901+
)
902+
} else {
903+
Arc::new(Int32Array::from(values.clone()))
904+
};
905+
906+
let schema =
907+
Schema::new(vec![Field::new("a", array.data_type().clone(), false)]);
908+
let exprs = haystack.iter().map(|value| lit(*value)).collect();
909+
let expr =
910+
in_list(col("a", &schema).unwrap(), exprs, &false, &schema).unwrap();
911+
let batch = RecordBatch::try_new(Arc::new(schema), vec![array]).unwrap();
912+
let encoding = if dictionary { "dictionary" } else { "plain" };
913+
914+
c.bench_with_input(
915+
BenchmarkId::new(
916+
"dictionary_dispatch",
917+
format!(
918+
"i32/{strategy}/{encoding}/list={list_size}/batch={batch_size}"
919+
),
920+
),
921+
&batch,
922+
|b, batch| b.iter(|| expr.evaluate(batch).unwrap()),
923+
);
924+
}
925+
}
926+
}
927+
}
928+
874929
// =============================================================================
875930
// NULL HANDLING BENCHMARKS
876931
// =============================================================================
@@ -1204,7 +1259,7 @@ fn bench_fixed_size_binary(c: &mut Criterion) {
12041259
criterion_group! {
12051260
name = benches;
12061261
config = Criterion::default();
1207-
targets = bench_narrow_integer, bench_primitive, bench_f32, bench_timestamp_ns, bench_interval_month_day_nano, bench_utf8, bench_utf8view, bench_dictionary, bench_nulls, bench_fixed_size_binary
1262+
targets = bench_narrow_integer, bench_primitive, bench_f32, bench_timestamp_ns, bench_interval_month_day_nano, bench_utf8, bench_utf8view, bench_dictionary, bench_dictionary_dispatch, bench_nulls, bench_fixed_size_binary
12081263
}
12091264

12101265
criterion_main!(benches);

0 commit comments

Comments
 (0)