Skip to content

Commit d2c76b0

Browse files
committed
feat(physical-plan): add GroupColumn support for Duration in multi-column GROUP BY
`multi_group_by::group_column_supported_type` gates which GROUP BY columns can use the column-wise `GroupValuesColumn` fast path. Any unsupported column forces the entire grouping onto the byte-encoded `GroupValuesRows` fallback, so a single `Duration` key dragged an otherwise-qualifying multi-column GROUP BY onto the slow path. `Duration` shares the `i64` native representation of `Timestamp`, so it reuses the existing `PrimitiveGroupValueBuilder` with no new builder type: - dispatch the four `Duration*Type` units in `make_group_column` - accept `Duration(_)` in `group_column_supported_type` (all four units are valid Arrow types, unlike Time32/Time64) - extend the `group_column_supported_type` <-> `make_group_column` consistency fuzz with the four Duration units - add an end-to-end unit test (Duration GROUP BY dedups including nulls and preserves the Duration output type) and a Duration GROUP BY block (single- and multi-column keys) in aggregate.slt Part of apache#22715
1 parent 67947b6 commit d2c76b0

2 files changed

Lines changed: 113 additions & 7 deletions

File tree

  • datafusion
    • physical-plan/src/aggregates/group_values/multi_group_by
    • sqllogictest/test_files

‎datafusion/physical-plan/src/aggregates/group_values/multi_group_by/mod.rs‎

Lines changed: 82 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -32,12 +32,13 @@ use crate::aggregates::group_values::multi_group_by::{
3232
use arrow::array::{Array, ArrayRef, BooleanBufferBuilder};
3333
use arrow::compute::cast;
3434
use arrow::datatypes::{
35-
BinaryViewType, DataType, Date32Type, Date64Type, Decimal128Type, Field, Float32Type,
36-
Float64Type, Int8Type, Int16Type, Int32Type, Int64Type, Schema, SchemaRef,
37-
StringViewType, Time32MillisecondType, Time32SecondType, Time64MicrosecondType,
38-
Time64NanosecondType, TimeUnit, TimestampMicrosecondType, TimestampMillisecondType,
39-
TimestampNanosecondType, TimestampSecondType, UInt8Type, UInt16Type, UInt32Type,
40-
UInt64Type,
35+
BinaryViewType, DataType, Date32Type, Date64Type, Decimal128Type,
36+
DurationMicrosecondType, DurationMillisecondType, DurationNanosecondType,
37+
DurationSecondType, Field, Float32Type, Float64Type, Int8Type, Int16Type, Int32Type,
38+
Int64Type, Schema, SchemaRef, StringViewType, Time32MillisecondType,
39+
Time32SecondType, Time64MicrosecondType, Time64NanosecondType, TimeUnit,
40+
TimestampMicrosecondType, TimestampMillisecondType, TimestampNanosecondType,
41+
TimestampSecondType, UInt8Type, UInt16Type, UInt32Type, UInt64Type,
4142
};
4243
use datafusion_common::hash_utils::RandomState;
4344
use datafusion_common::hash_utils::create_hashes;
@@ -952,6 +953,9 @@ fn group_column_supported_type(data_type: &DataType) -> bool {
952953
| DataType::Time64(TimeUnit::Microsecond)
953954
| DataType::Time64(TimeUnit::Nanosecond)
954955
| DataType::Timestamp(_, _)
956+
// All four Duration units are valid Arrow types (unlike Time32 /
957+
// Time64), so every unit is handled by the dispatcher below.
958+
| DataType::Duration(_)
955959
| DataType::Utf8View
956960
| DataType::BinaryView
957961
| DataType::Boolean
@@ -1031,6 +1035,20 @@ fn make_group_column(field: &Field) -> Result<Box<dyn GroupColumn>> {
10311035
instantiate_primitive!(v, nullable, TimestampNanosecondType, data_type)
10321036
}
10331037
},
1038+
DataType::Duration(t) => match t {
1039+
TimeUnit::Second => {
1040+
instantiate_primitive!(v, nullable, DurationSecondType, data_type)
1041+
}
1042+
TimeUnit::Millisecond => {
1043+
instantiate_primitive!(v, nullable, DurationMillisecondType, data_type)
1044+
}
1045+
TimeUnit::Microsecond => {
1046+
instantiate_primitive!(v, nullable, DurationMicrosecondType, data_type)
1047+
}
1048+
TimeUnit::Nanosecond => {
1049+
instantiate_primitive!(v, nullable, DurationNanosecondType, data_type)
1050+
}
1051+
},
10341052
DataType::Decimal128(_, _) => {
10351053
instantiate_primitive!(v, nullable, Decimal128Type, data_type)
10361054
}
@@ -1259,7 +1277,10 @@ enum Nulls {
12591277
mod tests {
12601278
use std::{collections::HashMap, sync::Arc};
12611279

1262-
use arrow::array::{ArrayRef, Int64Array, RecordBatch, StringArray, StringViewArray};
1280+
use arrow::array::{
1281+
Array, ArrayRef, DurationMicrosecondArray, Int64Array, RecordBatch, StringArray,
1282+
StringViewArray,
1283+
};
12631284
use arrow::datatypes::{DataType, Field, Schema, SchemaRef};
12641285
use arrow::{compute::concat_batches, util::pretty::pretty_format_batches};
12651286
use datafusion_common::utils::proxy::HashTableAllocExt;
@@ -1309,6 +1330,10 @@ mod tests {
13091330
DataType::Time64(arrow::datatypes::TimeUnit::Microsecond),
13101331
DataType::Time64(arrow::datatypes::TimeUnit::Nanosecond),
13111332
DataType::Timestamp(arrow::datatypes::TimeUnit::Nanosecond, None),
1333+
DataType::Duration(arrow::datatypes::TimeUnit::Second),
1334+
DataType::Duration(arrow::datatypes::TimeUnit::Millisecond),
1335+
DataType::Duration(arrow::datatypes::TimeUnit::Microsecond),
1336+
DataType::Duration(arrow::datatypes::TimeUnit::Nanosecond),
13121337
];
13131338

13141339
for dt in &supported_cases {
@@ -1352,6 +1377,56 @@ mod tests {
13521377
}
13531378
}
13541379

1380+
/// End-to-end coverage for `Duration` group keys: a `Duration` column stays
1381+
/// on the `GroupValuesColumn` fast path, deduplicates equal durations
1382+
/// (including nulls), and round-trips with its `Duration` output type
1383+
/// preserved (not the bare `i64` native). The four units share one builder,
1384+
/// so a single unit exercises the dispatcher; the fuzz above covers routing
1385+
/// for every unit.
1386+
#[test]
1387+
fn test_group_values_column_duration() {
1388+
use arrow::datatypes::TimeUnit;
1389+
1390+
let schema = Arc::new(Schema::new(vec![Field::new(
1391+
"d",
1392+
DataType::Duration(TimeUnit::Microsecond),
1393+
true,
1394+
)]));
1395+
assert!(supported_schema(&schema));
1396+
let mut group_values =
1397+
GroupValuesColumn::<false>::try_new(Arc::clone(&schema)).unwrap();
1398+
1399+
// Two distinct durations and a null with repeats: row 3 repeats row 0,
1400+
// row 4 repeats the null of row 1.
1401+
let input: ArrayRef = Arc::new(DurationMicrosecondArray::from(vec![
1402+
Some(10),
1403+
None,
1404+
Some(20),
1405+
Some(10),
1406+
None,
1407+
]));
1408+
let mut groups = Vec::new();
1409+
group_values.intern(&[input], &mut groups).unwrap();
1410+
assert_eq!(groups, vec![0, 1, 2, 0, 1]);
1411+
1412+
let emitted = group_values.emit(EmitTo::All).unwrap();
1413+
assert_eq!(emitted.len(), 1);
1414+
// The emitted key keeps its Duration type, not the bare i64 native.
1415+
assert_eq!(
1416+
emitted[0].data_type(),
1417+
&DataType::Duration(TimeUnit::Microsecond)
1418+
);
1419+
let actual = emitted[0]
1420+
.as_any()
1421+
.downcast_ref::<DurationMicrosecondArray>()
1422+
.expect("emitted column should be a DurationMicrosecondArray");
1423+
// Three groups in first-seen order: 10, null, 20.
1424+
assert_eq!(actual.len(), 3);
1425+
assert_eq!(actual.value(0), 10);
1426+
assert!(actual.is_null(1));
1427+
assert_eq!(actual.value(2), 20);
1428+
}
1429+
13551430
#[test]
13561431
fn supported_schema_rejects_mix_of_supported_and_unsupported() {
13571432
// One Float16 column among supported columns flips the whole

‎datafusion/sqllogictest/test_files/aggregate.slt‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2268,6 +2268,37 @@ statement ok
22682268
DROP TABLE approx_distinct_duration_test;
22692269

22702270

2271+
# GROUP BY over Duration group keys. Duration columns route through the
2272+
# GroupValuesColumn column-wise fast path, so a Duration key no longer forces
2273+
# the row-encoded fallback. Assert equal durations collapse and a Duration key
2274+
# groups alongside a primitive on the multi-column path.
2275+
statement ok
2276+
CREATE TABLE duration_group_test AS VALUES
2277+
(1, arrow_cast(5, 'Duration(Second)')),
2278+
(1, arrow_cast(5, 'Duration(Second)')),
2279+
(1, arrow_cast(7, 'Duration(Second)')),
2280+
(2, arrow_cast(5, 'Duration(Second)'));
2281+
2282+
# Single Duration group key: {5s, 7s}, with 5s appearing three times.
2283+
query ?I
2284+
SELECT column2, count(*) FROM duration_group_test GROUP BY column2 ORDER BY column2;
2285+
----
2286+
0 days 0 hours 0 mins 5 secs 3
2287+
0 days 0 hours 0 mins 7 secs 1
2288+
2289+
# Multi-column GROUP BY: a primitive key and a Duration key on the same path.
2290+
query I?I
2291+
SELECT column1, column2, count(*)
2292+
FROM duration_group_test GROUP BY column1, column2 ORDER BY column1, column2;
2293+
----
2294+
1 0 days 0 hours 0 mins 5 secs 2
2295+
1 0 days 0 hours 0 mins 7 secs 1
2296+
2 0 days 0 hours 0 mins 5 secs 1
2297+
2298+
statement ok
2299+
DROP TABLE duration_group_test;
2300+
2301+
22712302
# This test runs approx_distinct over the intervals YearMonth,
22722303
# DayTime, MonthDayNano for the scalar and the grouped path.
22732304
statement ok

0 commit comments

Comments
 (0)