Skip to content

Commit 7b4ccd0

Browse files
tohuya6kosiew
authored andcommitted
feat: add GroupColumn support for Interval in multi-column GROUP BY (apache#23786)
## Which issue does this PR close? - Part of apache#22715. ## Rationale for this change `multi_group_by::group_column_supported_type` gates which GROUP BY columns may use the column-wise `GroupValuesColumn` fast path, and the gate is all-or-nothing: a single unsupported column forces the **entire** grouping onto the byte-encoded `GroupValuesRows` fallback, even when every other key column would have qualified. An `Interval` key triggers exactly that today. All three `Interval` units reuse the existing `PrimitiveGroupValueBuilder` with no new builder type and no new `HashValue` impl — the `i32` / `IntervalDayTime` / `IntervalMonthDayNano` natives already implement it. ## What changes are included in this PR? - Dispatch the three `Interval*Type` units in `make_group_column`. `IntervalUnit` has exactly three variants, so the match is exhaustive with no fallback arm (unlike `Time32`/`Time64`, which must reject invalid unit combinations). - Accept `Interval(_)` in `group_column_supported_type`. - Extend the `group_column_supported_type` ↔ `make_group_column` consistency fuzz with the three Interval units. - Add an `(Interval, Int32)` group-count benchmark to `benches/multi_group_by.rs`. ## Are these changes tested? Yes. - New unit test: an `Interval` key stays on the `GroupValuesColumn` path, dedups including nulls, keeps "1 month" and "30 days" as **distinct** groups (no cross-unit folding), and round-trips with the `Interval` output type preserved. - The consistency fuzz now asserts all three Interval units route through the dispatcher. - New single- and multi-column `Interval` `GROUP BY` coverage in `group_by.slt`. ## Are there any user-facing changes? No API changes. `GROUP BY` queries with an `Interval` key now use the column-wise fast path instead of the row-encoded fallback; results are unchanged. --------- Co-authored-by: tohuya6 <[email protected]>
1 parent 9ab4509 commit 7b4ccd0

3 files changed

Lines changed: 198 additions & 4 deletions

File tree

‎datafusion/physical-plan/benches/multi_group_by.rs‎

Lines changed: 92 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -28,10 +28,13 @@
2828
//! `FixedSizeBinaryGroupValueBuilder`.
2929
3030
use arrow::array::{
31-
ArrayRef, DurationMicrosecondArray, Float16Array, Int32Array, UInt32Array,
31+
ArrayRef, DurationMicrosecondArray, Float16Array, Int32Array,
32+
IntervalMonthDayNanoArray, UInt32Array,
3233
};
3334
use arrow::compute::take;
34-
use arrow::datatypes::{DataType, Field, Schema, SchemaRef, TimeUnit};
35+
use arrow::datatypes::{
36+
DataType, Field, IntervalMonthDayNano, IntervalUnit, Schema, SchemaRef, TimeUnit,
37+
};
3538
use arrow::util::bench_util::create_fsb_array;
3639
use criterion::{BenchmarkId, Criterion, criterion_group, criterion_main};
3740
use datafusion_physical_plan::aggregates::group_values::GroupValues;
@@ -620,6 +623,92 @@ fn bench_duration(c: &mut Criterion) {
620623
group.finish();
621624
}
622625

626+
fn make_interval_schema() -> SchemaRef {
627+
Arc::new(Schema::new(vec![
628+
Field::new("iv", DataType::Interval(IntervalUnit::MonthDayNano), false),
629+
Field::new("id", DataType::Int32, false),
630+
]))
631+
}
632+
633+
/// Generate `(Interval(MonthDayNano), Int32)` batches with `num_distinct_groups`
634+
/// distinct keys.
635+
///
636+
/// Each distinct interval is `MonthDayNano(g, 0, 0)`. The `Int32` column is
637+
/// keyed identically so the combined cardinality equals `num_distinct_groups`.
638+
fn generate_interval_batches(
639+
num_distinct_groups: usize,
640+
num_rows: usize,
641+
batch_size: usize,
642+
) -> Vec<Vec<ArrayRef>> {
643+
let num_full_batches = num_rows / batch_size;
644+
let remainder = num_rows % batch_size;
645+
let num_batches = num_full_batches + if remainder > 0 { 1 } else { 0 };
646+
647+
(0..num_batches)
648+
.map(|batch_idx| {
649+
let batch_start = batch_idx * batch_size;
650+
let current_batch_size = if batch_idx == num_batches - 1 && remainder > 0 {
651+
remainder
652+
} else {
653+
batch_size
654+
};
655+
656+
let group_ids = (0..current_batch_size)
657+
.map(|row| (batch_start + row) % num_distinct_groups);
658+
659+
let keys = IntervalMonthDayNanoArray::from_iter_values(
660+
group_ids
661+
.clone()
662+
.map(|g| IntervalMonthDayNano::new(g as i32, 0, 0)),
663+
);
664+
let id: Int32Array = group_ids.map(|g| g as i32).collect();
665+
666+
vec![Arc::new(keys) as ArrayRef, Arc::new(id) as ArrayRef]
667+
})
668+
.collect()
669+
}
670+
671+
/// Experiment 10: Group count sweep for an `(Interval, Int32)` key.
672+
///
673+
/// Exercises the primitive `GroupColumn` builder for `Interval` on the
674+
/// multi-column path (previously such a schema fell back to `GroupValuesRows`).
675+
fn bench_interval(c: &mut Criterion) {
676+
let mut group = c.benchmark_group("interval");
677+
group.sample_size(15);
678+
679+
let schema = make_interval_schema();
680+
681+
for num_groups in [1_000, 1_000_000] {
682+
let batches =
683+
generate_interval_batches(num_groups, 1_000_000, DEFAULT_BATCH_SIZE);
684+
685+
for vectorized in [true, false] {
686+
let label = if vectorized {
687+
"vectorized"
688+
} else {
689+
"row_based"
690+
};
691+
group.bench_with_input(
692+
BenchmarkId::new(label, format!("grp_{num_groups}")),
693+
&batches,
694+
|b, batches| {
695+
b.iter_batched_ref(
696+
|| {
697+
(
698+
create_group_values(&schema, vectorized),
699+
Vec::<usize>::with_capacity(DEFAULT_BATCH_SIZE),
700+
)
701+
},
702+
|(gv, groups)| bench_intern(gv, batches, groups),
703+
criterion::BatchSize::LargeInput,
704+
);
705+
},
706+
);
707+
}
708+
}
709+
group.finish();
710+
}
711+
623712
criterion_group!(
624713
benches,
625714
bench_issue_17850_regression,
@@ -631,5 +720,6 @@ criterion_group!(
631720
bench_fixed_size_binary,
632721
bench_float16,
633722
bench_duration,
723+
bench_interval,
634724
);
635725
criterion_main!(benches);

‎datafusion/physical-plan/src/aggregates/group_values/multi_group_by/mod.rs‎

Lines changed: 79 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,8 @@ use arrow::datatypes::{
3737
BinaryViewType, DataType, Date32Type, Date64Type, Decimal128Type,
3838
DurationMicrosecondType, DurationMillisecondType, DurationNanosecondType,
3939
DurationSecondType, Field, Float16Type, Float32Type, Float64Type, Int8Type,
40-
Int16Type, Int32Type, Int64Type, Schema, SchemaRef, StringViewType,
40+
Int16Type, Int32Type, Int64Type, IntervalDayTimeType, IntervalMonthDayNanoType,
41+
IntervalUnit, IntervalYearMonthType, Schema, SchemaRef, StringViewType,
4142
Time32MillisecondType, Time32SecondType, Time64MicrosecondType, Time64NanosecondType,
4243
TimeUnit, TimestampMicrosecondType, TimestampMillisecondType,
4344
TimestampNanosecondType, TimestampSecondType, UInt8Type, UInt16Type, UInt32Type,
@@ -967,6 +968,7 @@ fn group_column_supported_type(data_type: &DataType) -> bool {
967968
| DataType::Time64(TimeUnit::Nanosecond)
968969
| DataType::Timestamp(_, _)
969970
| DataType::Duration(_)
971+
| DataType::Interval(_)
970972
| DataType::Utf8View
971973
| DataType::BinaryView
972974
| DataType::Boolean
@@ -1063,6 +1065,19 @@ fn make_group_column(field: &Field) -> Result<Box<dyn GroupColumn>> {
10631065
instantiate_primitive!(v, nullable, DurationNanosecondType, data_type)
10641066
}
10651067
},
1068+
// `IntervalUnit` has exactly three variants, so this match is exhaustive
1069+
// with no fallback arm (unlike Time32 / Time64).
1070+
DataType::Interval(u) => match u {
1071+
IntervalUnit::YearMonth => {
1072+
instantiate_primitive!(v, nullable, IntervalYearMonthType, data_type)
1073+
}
1074+
IntervalUnit::DayTime => {
1075+
instantiate_primitive!(v, nullable, IntervalDayTimeType, data_type)
1076+
}
1077+
IntervalUnit::MonthDayNano => {
1078+
instantiate_primitive!(v, nullable, IntervalMonthDayNanoType, data_type)
1079+
}
1080+
},
10661081
DataType::Decimal128(_, _) => {
10671082
instantiate_primitive!(v, nullable, Decimal128Type, data_type)
10681083
}
@@ -1301,7 +1316,7 @@ mod tests {
13011316

13021317
use arrow::array::{
13031318
Array, ArrayRef, DurationMicrosecondArray, Float16Array, Int32Array, Int64Array,
1304-
RecordBatch, StringArray, StringViewArray,
1319+
PrimitiveArray, RecordBatch, StringArray, StringViewArray,
13051320
};
13061321
use arrow::datatypes::{DataType, Field, Schema, SchemaRef};
13071322
use arrow::{compute::concat_batches, util::pretty::pretty_format_batches};
@@ -1606,6 +1621,9 @@ mod tests {
16061621
DataType::Duration(arrow::datatypes::TimeUnit::Millisecond),
16071622
DataType::Duration(arrow::datatypes::TimeUnit::Microsecond),
16081623
DataType::Duration(arrow::datatypes::TimeUnit::Nanosecond),
1624+
DataType::Interval(arrow::datatypes::IntervalUnit::YearMonth),
1625+
DataType::Interval(arrow::datatypes::IntervalUnit::DayTime),
1626+
DataType::Interval(arrow::datatypes::IntervalUnit::MonthDayNano),
16091627
];
16101628

16111629
for dt in &supported_cases {
@@ -1758,6 +1776,65 @@ mod tests {
17581776
assert_eq!(ids.values().to_vec(), vec![3, 3, 4, 3, 3]);
17591777
}
17601778

1779+
// `(Interval, Int32)` keys for each of the three interval units: null keys
1780+
// dedup, the Int32 key splits equal intervals, and emit gives back Interval.
1781+
#[test]
1782+
fn test_group_values_column_interval() {
1783+
use arrow::datatypes::{
1784+
ArrowPrimitiveType, IntervalDayTime, IntervalDayTimeType,
1785+
IntervalMonthDayNano, IntervalMonthDayNanoType, IntervalUnit,
1786+
IntervalYearMonthType,
1787+
};
1788+
1789+
fn check<T: ArrowPrimitiveType>(unit: IntervalUnit, value: T::Native) {
1790+
let schema = Arc::new(Schema::new(vec![
1791+
Field::new("i", DataType::Interval(unit), true),
1792+
Field::new("n", DataType::Int32, true),
1793+
]));
1794+
assert!(supported_schema(&schema), "{unit:?} schema not supported");
1795+
let mut group_values =
1796+
GroupValuesColumn::<false>::try_new(Arc::clone(&schema)).unwrap();
1797+
1798+
let i: ArrayRef = Arc::new(PrimitiveArray::<T>::from_iter([
1799+
Some(value),
1800+
None,
1801+
Some(value),
1802+
None,
1803+
Some(value),
1804+
]));
1805+
let n: ArrayRef = Arc::new(Int32Array::from(vec![3, 3, 3, 3, 4]));
1806+
let mut groups = Vec::new();
1807+
group_values.intern(&[i, n], &mut groups).unwrap();
1808+
assert_eq!(groups, vec![0, 1, 0, 1, 2], "{unit:?}");
1809+
1810+
let emitted = group_values.emit(EmitTo::All).unwrap();
1811+
assert_eq!(emitted.len(), 2);
1812+
// The emitted key keeps its Interval type, not the bare native.
1813+
assert_eq!(emitted[0].data_type(), &DataType::Interval(unit));
1814+
let actual = emitted[0]
1815+
.as_any()
1816+
.downcast_ref::<PrimitiveArray<T>>()
1817+
.unwrap_or_else(|| panic!("emitted column should be a {unit:?} array"));
1818+
// Three groups in first-seen order: value, null, value (n=4).
1819+
assert_eq!(actual.len(), 3, "{unit:?}");
1820+
assert_eq!(actual.value(0), value, "{unit:?}");
1821+
assert!(actual.is_null(1), "{unit:?}");
1822+
assert_eq!(actual.value(2), value, "{unit:?}");
1823+
let ids = emitted[1]
1824+
.as_any()
1825+
.downcast_ref::<Int32Array>()
1826+
.expect("emitted column should be an Int32Array");
1827+
assert_eq!(ids.values().to_vec(), vec![3, 3, 4], "{unit:?}");
1828+
}
1829+
1830+
check::<IntervalYearMonthType>(IntervalUnit::YearMonth, 13);
1831+
check::<IntervalDayTimeType>(IntervalUnit::DayTime, IntervalDayTime::new(1, 500));
1832+
check::<IntervalMonthDayNanoType>(
1833+
IntervalUnit::MonthDayNano,
1834+
IntervalMonthDayNano::new(1, 0, 0),
1835+
);
1836+
}
1837+
17611838
#[test]
17621839
fn supported_schema_rejects_mix_of_supported_and_unsupported() {
17631840
// One unsupported column flips the whole schema to the GroupValuesRows

‎datafusion/sqllogictest/test_files/group_by.slt‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5745,3 +5745,30 @@ SELECT count(*) FROM (SELECT column1, column2 FROM float16_group_test GROUP BY c
57455745

57465746
statement ok
57475747
DROP TABLE float16_group_test;
5748+
5749+
# Test multi group by int + Interval
5750+
statement ok
5751+
CREATE TABLE interval_group_test AS VALUES
5752+
(1, INTERVAL '1' MONTH),
5753+
(1, INTERVAL '1' MONTH),
5754+
(1, INTERVAL '30' DAY),
5755+
(2, INTERVAL '1' MONTH);
5756+
5757+
# Single Interval group key ({1 month, 30 days}) via the GroupValuesPrimitive path.
5758+
query I
5759+
SELECT count(*) FROM interval_group_test GROUP BY column2 ORDER BY count(*);
5760+
----
5761+
1
5762+
3
5763+
5764+
# Multi-column GROUP BY: a primitive key and an Interval key on the same path.
5765+
query II
5766+
SELECT column1, count(*)
5767+
FROM interval_group_test GROUP BY column1, column2 ORDER BY column1, count(*);
5768+
----
5769+
1 1
5770+
1 2
5771+
2 1
5772+
5773+
statement ok
5774+
DROP TABLE interval_group_test;

0 commit comments

Comments
 (0)