Skip to content

Commit 34f922d

Browse files
tohuya6kosiew
authored andcommitted
feat: add GroupColumn support for Duration in multi-column GROUP BY (apache#23783)
## Which issue does this PR close? - Part of apache#22715. ## Rationale for this change `multi_group_by::group_column_supported_type` gates which GROUP BY columns may use the column-wise `GroupValuesColumn` fast path, and the gate is all-or-nothing: a single unsupported column forces the **entire** grouping onto the byte-encoded `GroupValuesRows` fallback, even when every other key column would have qualified. A `Duration` key triggers exactly that today, so an otherwise-qualifying multi-column `GROUP BY` pays the row-encoding tax because of one column. `Duration` shares the `i64` native representation already used by `Timestamp`, so supporting it is a pure slot-in of the existing `PrimitiveGroupValueBuilder` — no new builder type and no new comparison/hash logic. ## What changes are included in this PR? - Accept `Duration(_)` in `group_column_supported_type` (all four `TimeUnit`s are valid Arrow types, unlike the restricted `Time32`/`Time64` set). - Dispatch the four `Duration*Type` units in `make_group_column`. - Extend the `group_column_supported_type` ↔ `make_group_column` consistency fuzz with all four Duration units. - Add a `(Duration, Int32)` group-count benchmark to `benches/multi_group_by.rs`. ## Are these changes tested? Yes. - New unit test `test_group_values_column_duration`: a `(Duration(Microsecond), Int64)` key stays on the `GroupValuesColumn` path, dedups on the composite key (including the `(null, null)` pair), and round-trips with the `Duration` output type preserved (not the bare `i64`). - The consistency fuzz now asserts every `Duration` unit routes through the dispatcher. - New single- and multi-column `Duration` `GROUP BY` coverage in `group_by.slt`. ## Are there any user-facing changes? No API changes. `GROUP BY` queries with a `Duration` key now use the column-wise fast path instead of the row-encoded fallback; results are unchanged. --------- Co-authored-by: tohuya6 <[email protected]>
1 parent 1305241 commit 34f922d

3 files changed

Lines changed: 195 additions & 9 deletions

File tree

‎datafusion/physical-plan/benches/multi_group_by.rs‎

Lines changed: 87 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -27,9 +27,9 @@
2727
//! covers a `(FixedSizeBinary, Int32)` key to exercise the
2828
//! `FixedSizeBinaryGroupValueBuilder`.
2929
30-
use arrow::array::{ArrayRef, Int32Array, UInt32Array};
30+
use arrow::array::{ArrayRef, DurationMicrosecondArray, Int32Array, UInt32Array};
3131
use arrow::compute::take;
32-
use arrow::datatypes::{DataType, Field, Schema, SchemaRef};
32+
use arrow::datatypes::{DataType, Field, Schema, SchemaRef, TimeUnit};
3333
use arrow::util::bench_util::create_fsb_array;
3434
use criterion::{BenchmarkId, Criterion, criterion_group, criterion_main};
3535
use datafusion_physical_plan::aggregates::group_values::GroupValues;
@@ -444,6 +444,90 @@ fn bench_fixed_size_binary(c: &mut Criterion) {
444444
group.finish();
445445
}
446446

447+
fn make_duration_schema() -> SchemaRef {
448+
Arc::new(Schema::new(vec![
449+
Field::new("dur", DataType::Duration(TimeUnit::Microsecond), false),
450+
Field::new("id", DataType::Int32, false),
451+
]))
452+
}
453+
454+
/// Generate `(Duration(Microsecond), Int32)` batches with `num_distinct_groups`
455+
/// distinct keys.
456+
///
457+
/// Each distinct duration is `g` microseconds. The `Int32` column is keyed
458+
/// identically so the combined cardinality equals `num_distinct_groups`.
459+
fn generate_duration_batches(
460+
num_distinct_groups: usize,
461+
num_rows: usize,
462+
batch_size: usize,
463+
) -> Vec<Vec<ArrayRef>> {
464+
let num_full_batches = num_rows / batch_size;
465+
let remainder = num_rows % batch_size;
466+
let num_batches = num_full_batches + if remainder > 0 { 1 } else { 0 };
467+
468+
(0..num_batches)
469+
.map(|batch_idx| {
470+
let batch_start = batch_idx * batch_size;
471+
let current_batch_size = if batch_idx == num_batches - 1 && remainder > 0 {
472+
remainder
473+
} else {
474+
batch_size
475+
};
476+
477+
let group_ids = (0..current_batch_size)
478+
.map(|row| (batch_start + row) % num_distinct_groups);
479+
480+
let keys = DurationMicrosecondArray::from_iter_values(
481+
group_ids.clone().map(|g| g as i64),
482+
);
483+
let id: Int32Array = group_ids.map(|g| g as i32).collect();
484+
485+
vec![Arc::new(keys) as ArrayRef, Arc::new(id) as ArrayRef]
486+
})
487+
.collect()
488+
}
489+
490+
/// Experiment 9: Group count sweep for a `(Duration, Int32)` key.
491+
///
492+
/// Exercises the primitive `GroupColumn` builder for `Duration` on the
493+
/// multi-column path (previously such a schema fell back to `GroupValuesRows`).
494+
fn bench_duration(c: &mut Criterion) {
495+
let mut group = c.benchmark_group("duration");
496+
group.sample_size(15);
497+
498+
let schema = make_duration_schema();
499+
500+
for num_groups in [1_000, 1_000_000] {
501+
let batches =
502+
generate_duration_batches(num_groups, 1_000_000, DEFAULT_BATCH_SIZE);
503+
504+
for vectorized in [true, false] {
505+
let label = if vectorized {
506+
"vectorized"
507+
} else {
508+
"row_based"
509+
};
510+
group.bench_with_input(
511+
BenchmarkId::new(label, format!("grp_{num_groups}")),
512+
&batches,
513+
|b, batches| {
514+
b.iter_batched_ref(
515+
|| {
516+
(
517+
create_group_values(&schema, vectorized),
518+
Vec::<usize>::with_capacity(DEFAULT_BATCH_SIZE),
519+
)
520+
},
521+
|(gv, groups)| bench_intern(gv, batches, groups),
522+
criterion::BatchSize::LargeInput,
523+
);
524+
},
525+
);
526+
}
527+
}
528+
group.finish();
529+
}
530+
447531
criterion_group!(
448532
benches,
449533
bench_issue_17850_regression,
@@ -453,5 +537,6 @@ criterion_group!(
453537
bench_high_cardinality_scaling,
454538
bench_group_count_sweep,
455539
bench_fixed_size_binary,
540+
bench_duration,
456541
);
457542
criterion_main!(benches);

‎datafusion/physical-plan/src/aggregates/group_values/multi_group_by/mod.rs‎

Lines changed: 81 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -34,12 +34,13 @@ use crate::aggregates::group_values::multi_group_by::{
3434
use arrow::array::{Array, ArrayRef, BooleanBufferBuilder};
3535
use arrow::compute::cast;
3636
use arrow::datatypes::{
37-
BinaryViewType, DataType, Date32Type, Date64Type, Decimal128Type, Field, Float32Type,
38-
Float64Type, Int8Type, Int16Type, Int32Type, Int64Type, Schema, SchemaRef,
39-
StringViewType, Time32MillisecondType, Time32SecondType, Time64MicrosecondType,
40-
Time64NanosecondType, TimeUnit, TimestampMicrosecondType, TimestampMillisecondType,
41-
TimestampNanosecondType, TimestampSecondType, UInt8Type, UInt16Type, UInt32Type,
42-
UInt64Type,
37+
BinaryViewType, DataType, Date32Type, Date64Type, Decimal128Type,
38+
DurationMicrosecondType, DurationMillisecondType, DurationNanosecondType,
39+
DurationSecondType, Field, Float32Type, Float64Type, Int8Type, Int16Type, Int32Type,
40+
Int64Type, Schema, SchemaRef, StringViewType, Time32MillisecondType,
41+
Time32SecondType, Time64MicrosecondType, Time64NanosecondType, TimeUnit,
42+
TimestampMicrosecondType, TimestampMillisecondType, TimestampNanosecondType,
43+
TimestampSecondType, UInt8Type, UInt16Type, UInt32Type, UInt64Type,
4344
};
4445
use datafusion_common::hash_utils::RandomState;
4546
use datafusion_common::hash_utils::create_hashes;
@@ -963,6 +964,7 @@ fn group_column_supported_type(data_type: &DataType) -> bool {
963964
| DataType::Time64(TimeUnit::Microsecond)
964965
| DataType::Time64(TimeUnit::Nanosecond)
965966
| DataType::Timestamp(_, _)
967+
| DataType::Duration(_)
966968
| DataType::Utf8View
967969
| DataType::BinaryView
968970
| DataType::Boolean
@@ -1042,6 +1044,20 @@ fn make_group_column(field: &Field) -> Result<Box<dyn GroupColumn>> {
10421044
instantiate_primitive!(v, nullable, TimestampNanosecondType, data_type)
10431045
}
10441046
},
1047+
DataType::Duration(t) => match t {
1048+
TimeUnit::Second => {
1049+
instantiate_primitive!(v, nullable, DurationSecondType, data_type)
1050+
}
1051+
TimeUnit::Millisecond => {
1052+
instantiate_primitive!(v, nullable, DurationMillisecondType, data_type)
1053+
}
1054+
TimeUnit::Microsecond => {
1055+
instantiate_primitive!(v, nullable, DurationMicrosecondType, data_type)
1056+
}
1057+
TimeUnit::Nanosecond => {
1058+
instantiate_primitive!(v, nullable, DurationNanosecondType, data_type)
1059+
}
1060+
},
10451061
DataType::Decimal128(_, _) => {
10461062
instantiate_primitive!(v, nullable, Decimal128Type, data_type)
10471063
}
@@ -1278,7 +1294,10 @@ enum Nulls {
12781294
mod tests {
12791295
use std::{collections::HashMap, sync::Arc};
12801296

1281-
use arrow::array::{ArrayRef, Int64Array, RecordBatch, StringArray, StringViewArray};
1297+
use arrow::array::{
1298+
Array, ArrayRef, DurationMicrosecondArray, Int64Array, RecordBatch, StringArray,
1299+
StringViewArray,
1300+
};
12821301
use arrow::datatypes::{DataType, Field, Schema, SchemaRef};
12831302
use arrow::{compute::concat_batches, util::pretty::pretty_format_batches};
12841303
use datafusion_common::utils::proxy::HashTableAllocExt;
@@ -1577,6 +1596,10 @@ mod tests {
15771596
DataType::Time64(arrow::datatypes::TimeUnit::Microsecond),
15781597
DataType::Time64(arrow::datatypes::TimeUnit::Nanosecond),
15791598
DataType::Timestamp(arrow::datatypes::TimeUnit::Nanosecond, None),
1599+
DataType::Duration(arrow::datatypes::TimeUnit::Second),
1600+
DataType::Duration(arrow::datatypes::TimeUnit::Millisecond),
1601+
DataType::Duration(arrow::datatypes::TimeUnit::Microsecond),
1602+
DataType::Duration(arrow::datatypes::TimeUnit::Nanosecond),
15801603
];
15811604

15821605
for dt in &supported_cases {
@@ -1620,6 +1643,57 @@ mod tests {
16201643
}
16211644
}
16221645

1646+
// `Duration` group keys stay on the `GroupValuesColumn` fast path, dedup
1647+
// (including nulls), and round-trip with the `Duration` type preserved.
1648+
#[test]
1649+
fn test_group_values_column_duration() {
1650+
use arrow::datatypes::TimeUnit;
1651+
1652+
let schema = Arc::new(Schema::new(vec![
1653+
Field::new("d", DataType::Duration(TimeUnit::Microsecond), true),
1654+
Field::new("i", DataType::Int64, true),
1655+
]));
1656+
assert!(supported_schema(&schema));
1657+
let mut group_values =
1658+
GroupValuesColumn::<false>::try_new(Arc::clone(&schema)).unwrap();
1659+
1660+
// (d, i) rows, where row 3 repeats row 0 and row 4 repeats the null pair.
1661+
let d: ArrayRef = Arc::new(DurationMicrosecondArray::from(vec![
1662+
Some(10),
1663+
None,
1664+
Some(20),
1665+
Some(10),
1666+
None,
1667+
]));
1668+
let i: ArrayRef = Arc::new(Int64Array::from(vec![
1669+
Some(1),
1670+
None,
1671+
Some(2),
1672+
Some(1),
1673+
None,
1674+
]));
1675+
let mut groups = Vec::new();
1676+
group_values.intern(&[d, i], &mut groups).unwrap();
1677+
assert_eq!(groups, vec![0, 1, 2, 0, 1]);
1678+
1679+
let emitted = group_values.emit(EmitTo::All).unwrap();
1680+
assert_eq!(emitted.len(), 2);
1681+
// The Duration column round-trips as Duration on emit, not bare i64.
1682+
assert_eq!(
1683+
emitted[0].data_type(),
1684+
&DataType::Duration(TimeUnit::Microsecond)
1685+
);
1686+
let actual = emitted[0]
1687+
.as_any()
1688+
.downcast_ref::<DurationMicrosecondArray>()
1689+
.expect("emitted column should be a DurationMicrosecondArray");
1690+
// Three groups in first-seen order: 10, null, 20.
1691+
assert_eq!(actual.len(), 3);
1692+
assert_eq!(actual.value(0), 10);
1693+
assert!(actual.is_null(1));
1694+
assert_eq!(actual.value(2), 20);
1695+
}
1696+
16231697
#[test]
16241698
fn supported_schema_rejects_mix_of_supported_and_unsupported() {
16251699
// One Float16 column among supported columns flips the whole

‎datafusion/sqllogictest/test_files/group_by.slt‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5694,3 +5694,30 @@ drop table users_with_pk;
56945694

56955695
statement ok
56965696
drop table user_orders;
5697+
5698+
# Test multi group by int + Duration
5699+
statement ok
5700+
CREATE TABLE duration_group_test AS VALUES
5701+
(1, arrow_cast(5, 'Duration(Second)')),
5702+
(1, arrow_cast(5, 'Duration(Second)')),
5703+
(1, arrow_cast(7, 'Duration(Second)')),
5704+
(2, arrow_cast(5, 'Duration(Second)'));
5705+
5706+
# Single Duration group key ({5s, 7s}, 5s x3) via the GroupValuesPrimitive path.
5707+
query ?I
5708+
SELECT column2, count(*) FROM duration_group_test GROUP BY column2 ORDER BY column2;
5709+
----
5710+
0 days 0 hours 0 mins 5 secs 3
5711+
0 days 0 hours 0 mins 7 secs 1
5712+
5713+
# Multi-column GROUP BY: a primitive key and a Duration key on the same path.
5714+
query I?I
5715+
SELECT column1, column2, count(*)
5716+
FROM duration_group_test GROUP BY column1, column2 ORDER BY column1, column2;
5717+
----
5718+
1 0 days 0 hours 0 mins 5 secs 2
5719+
1 0 days 0 hours 0 mins 7 secs 1
5720+
2 0 days 0 hours 0 mins 5 secs 1
5721+
5722+
statement ok
5723+
DROP TABLE duration_group_test;

0 commit comments

Comments
 (0)