Skip to content

Commit 9ede065

Browse files
tohuya6kosiew
authored andcommitted
feat: add GroupColumn support for Decimal256 in multi-column GROUP BY (apache#23849)
## Which issue does this PR close? - Part of apache#22715. ## Rationale for this change A `Decimal256` GROUP BY key currently forces the whole grouping onto the row-encoded `GroupValuesRows` fallback. `i256` is `Copy` and already implements `ArrowNativeTypeOp` + `HashValue`, so it reuses the existing `PrimitiveGroupValueBuilder`, exactly like `Decimal128`. ## What changes are included in this PR? - Support `Decimal256` in `make_group_column` and `group_column_supported_type`. - Update the supported-type consistency test and add `Decimal256` benchmark. ## Are these changes tested? Yes, the consistency test + multi-column `Decimal256` GROUP BY (with NULL key) in `group_by.slt`. ## Are there any user-facing changes? No. --------- Co-authored-by: tohuya6 <[email protected]>
1 parent cdccc5a commit 9ede065

3 files changed

Lines changed: 113 additions & 3 deletions

File tree

‎datafusion/physical-plan/benches/multi_group_by.rs‎

Lines changed: 91 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,12 +28,13 @@
2828
//! `FixedSizeBinaryGroupValueBuilder`.
2929
3030
use arrow::array::{
31-
ArrayRef, DurationMicrosecondArray, Float16Array, Int32Array,
31+
ArrayRef, Decimal256Array, DurationMicrosecondArray, Float16Array, Int32Array,
3232
IntervalMonthDayNanoArray, UInt32Array,
3333
};
3434
use arrow::compute::take;
3535
use arrow::datatypes::{
3636
DataType, Field, IntervalMonthDayNano, IntervalUnit, Schema, SchemaRef, TimeUnit,
37+
i256,
3738
};
3839
use arrow::util::bench_util::create_fsb_array;
3940
use criterion::{BenchmarkId, Criterion, criterion_group, criterion_main};
@@ -709,6 +710,94 @@ fn bench_interval(c: &mut Criterion) {
709710
group.finish();
710711
}
711712

713+
fn make_decimal256_schema() -> SchemaRef {
714+
Arc::new(Schema::new(vec![
715+
Field::new("dec", DataType::Decimal256(50, 0), false),
716+
Field::new("id", DataType::Int32, false),
717+
]))
718+
}
719+
720+
/// Generate `(Decimal256(50, 0), Int32)` batches with `num_distinct_groups`
721+
/// distinct keys.
722+
///
723+
/// Each distinct value is `i256::from_i128(g)`, and precision > 38 keeps it a
724+
/// genuine `Decimal256`. The `Int32` column is keyed identically so the combined
725+
/// cardinality equals `num_distinct_groups`.
726+
fn generate_decimal256_batches(
727+
num_distinct_groups: usize,
728+
num_rows: usize,
729+
batch_size: usize,
730+
) -> Vec<Vec<ArrayRef>> {
731+
let num_full_batches = num_rows / batch_size;
732+
let remainder = num_rows % batch_size;
733+
let num_batches = num_full_batches + if remainder > 0 { 1 } else { 0 };
734+
735+
(0..num_batches)
736+
.map(|batch_idx| {
737+
let batch_start = batch_idx * batch_size;
738+
let current_batch_size = if batch_idx == num_batches - 1 && remainder > 0 {
739+
remainder
740+
} else {
741+
batch_size
742+
};
743+
744+
let group_ids = (0..current_batch_size)
745+
.map(|row| (batch_start + row) % num_distinct_groups);
746+
747+
let keys = Decimal256Array::from_iter_values(
748+
group_ids.clone().map(|g| i256::from_i128(g as i128)),
749+
)
750+
.with_precision_and_scale(50, 0)
751+
.unwrap();
752+
let id: Int32Array = group_ids.map(|g| g as i32).collect();
753+
754+
vec![Arc::new(keys) as ArrayRef, Arc::new(id) as ArrayRef]
755+
})
756+
.collect()
757+
}
758+
759+
/// Experiment 11: Group count sweep for a `(Decimal256, Int32)` key.
760+
///
761+
/// Exercises the primitive `GroupColumn` builder for `Decimal256` (32-byte
762+
/// `i256` native) on the multi-column path (previously such a schema fell back
763+
/// to `GroupValuesRows`).
764+
fn bench_decimal256(c: &mut Criterion) {
765+
let mut group = c.benchmark_group("decimal256");
766+
group.sample_size(15);
767+
768+
let schema = make_decimal256_schema();
769+
770+
for num_groups in [1_000, 1_000_000] {
771+
let batches =
772+
generate_decimal256_batches(num_groups, 1_000_000, DEFAULT_BATCH_SIZE);
773+
774+
for vectorized in [true, false] {
775+
let label = if vectorized {
776+
"vectorized"
777+
} else {
778+
"row_based"
779+
};
780+
group.bench_with_input(
781+
BenchmarkId::new(label, format!("grp_{num_groups}")),
782+
&batches,
783+
|b, batches| {
784+
b.iter_batched_ref(
785+
|| {
786+
(
787+
create_group_values(&schema, vectorized),
788+
Vec::<usize>::with_capacity(DEFAULT_BATCH_SIZE),
789+
)
790+
},
791+
|(gv, groups)| bench_intern(gv, batches, groups),
792+
criterion::BatchSize::LargeInput,
793+
);
794+
},
795+
);
796+
}
797+
}
798+
group.finish();
799+
}
800+
712801
criterion_group!(
713802
benches,
714803
bench_issue_17850_regression,
@@ -721,5 +810,6 @@ criterion_group!(
721810
bench_float16,
722811
bench_duration,
723812
bench_interval,
813+
bench_decimal256,
724814
);
725815
criterion_main!(benches);

‎datafusion/physical-plan/src/aggregates/group_values/multi_group_by/mod.rs‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ use crate::aggregates::group_values::multi_group_by::{
3434
use arrow::array::{Array, ArrayRef, BooleanBufferBuilder};
3535
use arrow::compute::cast;
3636
use arrow::datatypes::{
37-
BinaryViewType, DataType, Date32Type, Date64Type, Decimal128Type,
37+
BinaryViewType, DataType, Date32Type, Date64Type, Decimal128Type, Decimal256Type,
3838
DurationMicrosecondType, DurationMillisecondType, DurationNanosecondType,
3939
DurationSecondType, Field, Float16Type, Float32Type, Float64Type, Int8Type,
4040
Int16Type, Int32Type, Int64Type, IntervalDayTimeType, IntervalMonthDayNanoType,
@@ -951,6 +951,7 @@ fn group_column_supported_type(data_type: &DataType) -> bool {
951951
| DataType::Float32
952952
| DataType::Float64
953953
| DataType::Decimal128(_, _)
954+
| DataType::Decimal256(_, _)
954955
| DataType::Utf8
955956
| DataType::LargeUtf8
956957
| DataType::Binary
@@ -1081,6 +1082,9 @@ fn make_group_column(field: &Field) -> Result<Box<dyn GroupColumn>> {
10811082
DataType::Decimal128(_, _) => {
10821083
instantiate_primitive!(v, nullable, Decimal128Type, data_type)
10831084
}
1085+
DataType::Decimal256(_, _) => {
1086+
instantiate_primitive!(v, nullable, Decimal256Type, data_type)
1087+
}
10841088
DataType::Utf8 => {
10851089
v.push(Box::new(ByteGroupValueBuilder::<i32>::new(
10861090
OutputType::Utf8,
@@ -1603,6 +1607,7 @@ mod tests {
16031607
DataType::Float64,
16041608
DataType::Float16,
16051609
DataType::Decimal128(38, 10),
1610+
DataType::Decimal256(76, 10),
16061611
DataType::Utf8,
16071612
DataType::LargeUtf8,
16081613
DataType::Utf8View,
@@ -1640,7 +1645,6 @@ mod tests {
16401645
}
16411646

16421647
let unsupported_cases: Vec<DataType> = vec![
1643-
DataType::Decimal256(76, 10),
16441648
// Invalid Time-unit combinations: Time32 is defined only for
16451649
// Second / Millisecond and Time64 only for Microsecond /
16461650
// Nanosecond. The TimeUnit enum allows constructing the other

‎datafusion/sqllogictest/test_files/group_by.slt‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5765,3 +5765,19 @@ FROM interval_group_test GROUP BY column1, column2 ORDER BY column1, count(*);
57655765

57665766
statement ok
57675767
DROP TABLE interval_group_test;
5768+
5769+
# Test multi group by int + Decimal256
5770+
statement ok
5771+
create table decimal256_multi_group (k int, d decimal(50, 2)) as values
5772+
(1, 100.00), (1, 100.00), (1, 250.00), (2, 100.00), (2, NULL);
5773+
5774+
query IRI
5775+
select k, d, count(*) from decimal256_multi_group group by k, d order by k, d;
5776+
----
5777+
1 100 2
5778+
1 250 1
5779+
2 100 1
5780+
2 NULL 1
5781+
5782+
statement ok
5783+
drop table decimal256_multi_group;

0 commit comments

Comments
 (0)