Skip to content

Commit fc1cea6

Browse files
tohuya6alamb
authored andcommitted
feat: add GroupColumn support for Float16 in multi-column GROUP BY (apache#23785)
## Which issue does this PR close? - Part of apache#22715. ## Rationale for this change `multi_group_by::group_column_supported_type` gates which GROUP BY columns may use the column-wise `GroupValuesColumn` fast path, and the gate is all-or-nothing: a single unsupported column forces the **entire** grouping onto the byte-encoded `GroupValuesRows` fallback, even when every other key column would have qualified. A `Float16` key triggers exactly that today. `Float16` reuses the existing `PrimitiveGroupValueBuilder` with no new builder type: its native `half::f16` already implements the `HashValue` canonicalization (`hash_float!(f16, f32, f64)`), so `-0.0`/`+0.0` folding and `NaN` grouping match `Float32`/`Float64` with no extra handling. ## What changes are included in this PR? - Accept `Float16` in `group_column_supported_type` and dispatch it in `make_group_column`. - Move `Float16` from the rejected to the accepted set in the `group_column_supported_type` ↔ `make_group_column` consistency fuzz, and repoint the two tests that used `Float16` as their stock "unsupported" example to a permanently-invalid unit combo (`Time64(Second)`), so they stay stable as sibling primitive builders (Decimal256, Interval, …) land independently. - Add a `(Float16, Int32)` group-count benchmark to `benches/multi_group_by.rs` (capped below f16's ~63.5k distinct finite values). ## Are these changes tested? Yes. - New unit test `test_group_values_column_float16`: a `(Float16, Int32)` key stays on the `GroupValuesColumn` path, dedups including nulls, folds `-0.0`/`+0.0` into one group (stored as `+0.0`), groups equal `NaN`s, keeps `(0.0, 4)` distinct from `(±0.0, 3)` via the `Int32` key, and round-trips with the `Float16` output type preserved. - The consistency fuzz now asserts `Float16` routes through the dispatcher. - New single- and multi-column `Float16` `GROUP BY` coverage in `group_by.slt`. ## Are there any user-facing changes? No API changes. `GROUP BY` queries with a `Float16` key now use the column-wise fast path instead of the row-encoded fallback; results are unchanged. --------- Co-authored-by: tohuya6 <[email protected]> Co-authored-by: Andrew Lamb <[email protected]>
1 parent 83a1d66 commit fc1cea6

3 files changed

Lines changed: 203 additions & 15 deletions

File tree

‎datafusion/physical-plan/benches/multi_group_by.rs‎

Lines changed: 94 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,14 +27,17 @@
2727
//! covers a `(FixedSizeBinary, Int32)` key to exercise the
2828
//! `FixedSizeBinaryGroupValueBuilder`.
2929
30-
use arrow::array::{ArrayRef, DurationMicrosecondArray, Int32Array, UInt32Array};
30+
use arrow::array::{
31+
ArrayRef, DurationMicrosecondArray, Float16Array, Int32Array, UInt32Array,
32+
};
3133
use arrow::compute::take;
3234
use arrow::datatypes::{DataType, Field, Schema, SchemaRef, TimeUnit};
3335
use arrow::util::bench_util::create_fsb_array;
3436
use criterion::{BenchmarkId, Criterion, criterion_group, criterion_main};
3537
use datafusion_physical_plan::aggregates::group_values::GroupValues;
3638
use datafusion_physical_plan::aggregates::group_values::GroupValuesRows;
3739
use datafusion_physical_plan::aggregates::group_values::multi_group_by::GroupValuesColumn;
40+
use half::f16;
3841
use std::hint::black_box;
3942
use std::sync::Arc;
4043

@@ -444,6 +447,95 @@ fn bench_fixed_size_binary(c: &mut Criterion) {
444447
group.finish();
445448
}
446449

450+
fn make_f16_schema() -> SchemaRef {
451+
Arc::new(Schema::new(vec![
452+
Field::new("f16", DataType::Float16, false),
453+
Field::new("id", DataType::Int32, false),
454+
]))
455+
}
456+
457+
/// Generate `(Float16, Int32)` batches with `num_distinct_groups` distinct keys.
458+
///
459+
/// `f16` has only ~63.5k finite values, so `num_distinct_groups` must stay well
460+
/// under that (see `bench_float16`). Distinct keys are the low finite `f16` bit
461+
/// patterns, skipping NaN and inf. The `Int32` column is keyed identically so
462+
/// the combined cardinality equals `num_distinct_groups`.
463+
fn generate_f16_batches(
464+
num_distinct_groups: usize,
465+
num_rows: usize,
466+
batch_size: usize,
467+
) -> Vec<Vec<ArrayRef>> {
468+
let pool: Vec<f16> = (0u16..)
469+
.map(f16::from_bits)
470+
.filter(|v| v.is_finite())
471+
.take(num_distinct_groups)
472+
.collect();
473+
474+
let num_full_batches = num_rows / batch_size;
475+
let remainder = num_rows % batch_size;
476+
let num_batches = num_full_batches + if remainder > 0 { 1 } else { 0 };
477+
478+
(0..num_batches)
479+
.map(|batch_idx| {
480+
let batch_start = batch_idx * batch_size;
481+
let current_batch_size = if batch_idx == num_batches - 1 && remainder > 0 {
482+
remainder
483+
} else {
484+
batch_size
485+
};
486+
487+
let group_ids = (0..current_batch_size)
488+
.map(|row| (batch_start + row) % num_distinct_groups);
489+
490+
let keys = Float16Array::from_iter_values(group_ids.clone().map(|g| pool[g]));
491+
let id: Int32Array = group_ids.map(|g| g as i32).collect();
492+
493+
vec![Arc::new(keys) as ArrayRef, Arc::new(id) as ArrayRef]
494+
})
495+
.collect()
496+
}
497+
498+
/// Experiment 8: Group count sweep for a `(Float16, Int32)` key.
499+
///
500+
/// Exercises the primitive `GroupColumn` builder for `Float16` on the
501+
/// multi-column path (previously such a schema fell back to `GroupValuesRows`).
502+
/// Group counts are capped below `f16`'s ~63.5k distinct finite values.
503+
fn bench_float16(c: &mut Criterion) {
504+
let mut group = c.benchmark_group("float16");
505+
group.sample_size(15);
506+
507+
let schema = make_f16_schema();
508+
509+
for num_groups in [1_000, 60_000] {
510+
let batches = generate_f16_batches(num_groups, 1_000_000, DEFAULT_BATCH_SIZE);
511+
512+
for vectorized in [true, false] {
513+
let label = if vectorized {
514+
"vectorized"
515+
} else {
516+
"row_based"
517+
};
518+
group.bench_with_input(
519+
BenchmarkId::new(label, format!("grp_{num_groups}")),
520+
&batches,
521+
|b, batches| {
522+
b.iter_batched_ref(
523+
|| {
524+
(
525+
create_group_values(&schema, vectorized),
526+
Vec::<usize>::with_capacity(DEFAULT_BATCH_SIZE),
527+
)
528+
},
529+
|(gv, groups)| bench_intern(gv, batches, groups),
530+
criterion::BatchSize::LargeInput,
531+
);
532+
},
533+
);
534+
}
535+
}
536+
group.finish();
537+
}
538+
447539
fn make_duration_schema() -> SchemaRef {
448540
Arc::new(Schema::new(vec![
449541
Field::new("dur", DataType::Duration(TimeUnit::Microsecond), false),
@@ -537,6 +629,7 @@ criterion_group!(
537629
bench_high_cardinality_scaling,
538630
bench_group_count_sweep,
539631
bench_fixed_size_binary,
632+
bench_float16,
540633
bench_duration,
541634
);
542635
criterion_main!(benches);

‎datafusion/physical-plan/src/aggregates/group_values/multi_group_by/mod.rs‎

Lines changed: 85 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -36,11 +36,12 @@ use arrow::compute::cast;
3636
use arrow::datatypes::{
3737
BinaryViewType, DataType, Date32Type, Date64Type, Decimal128Type,
3838
DurationMicrosecondType, DurationMillisecondType, DurationNanosecondType,
39-
DurationSecondType, Field, Float32Type, Float64Type, Int8Type, Int16Type, Int32Type,
40-
Int64Type, Schema, SchemaRef, StringViewType, Time32MillisecondType,
41-
Time32SecondType, Time64MicrosecondType, Time64NanosecondType, TimeUnit,
42-
TimestampMicrosecondType, TimestampMillisecondType, TimestampNanosecondType,
43-
TimestampSecondType, UInt8Type, UInt16Type, UInt32Type, UInt64Type,
39+
DurationSecondType, Field, Float16Type, Float32Type, Float64Type, Int8Type,
40+
Int16Type, Int32Type, Int64Type, Schema, SchemaRef, StringViewType,
41+
Time32MillisecondType, Time32SecondType, Time64MicrosecondType, Time64NanosecondType,
42+
TimeUnit, TimestampMicrosecondType, TimestampMillisecondType,
43+
TimestampNanosecondType, TimestampSecondType, UInt8Type, UInt16Type, UInt32Type,
44+
UInt64Type,
4445
};
4546
use datafusion_common::hash_utils::RandomState;
4647
use datafusion_common::hash_utils::create_hashes;
@@ -945,6 +946,7 @@ fn group_column_supported_type(data_type: &DataType) -> bool {
945946
| DataType::UInt16
946947
| DataType::UInt32
947948
| DataType::UInt64
949+
| DataType::Float16
948950
| DataType::Float32
949951
| DataType::Float64
950952
| DataType::Decimal128(_, _)
@@ -999,6 +1001,9 @@ fn make_group_column(field: &Field) -> Result<Box<dyn GroupColumn>> {
9991001
DataType::UInt16 => instantiate_primitive!(v, nullable, UInt16Type, data_type),
10001002
DataType::UInt32 => instantiate_primitive!(v, nullable, UInt32Type, data_type),
10011003
DataType::UInt64 => instantiate_primitive!(v, nullable, UInt64Type, data_type),
1004+
DataType::Float16 => {
1005+
instantiate_primitive!(v, nullable, Float16Type, data_type)
1006+
}
10021007
DataType::Float32 => {
10031008
instantiate_primitive!(v, nullable, Float32Type, data_type)
10041009
}
@@ -1295,8 +1300,8 @@ mod tests {
12951300
use std::{collections::HashMap, sync::Arc};
12961301

12971302
use arrow::array::{
1298-
Array, ArrayRef, DurationMicrosecondArray, Int64Array, RecordBatch, StringArray,
1299-
StringViewArray,
1303+
Array, ArrayRef, DurationMicrosecondArray, Float16Array, Int32Array, Int64Array,
1304+
RecordBatch, StringArray, StringViewArray,
13001305
};
13011306
use arrow::datatypes::{DataType, Field, Schema, SchemaRef};
13021307
use arrow::{compute::concat_batches, util::pretty::pretty_format_batches};
@@ -1581,6 +1586,7 @@ mod tests {
15811586
DataType::UInt64,
15821587
DataType::Float32,
15831588
DataType::Float64,
1589+
DataType::Float16,
15841590
DataType::Decimal128(38, 10),
15851591
DataType::Utf8,
15861592
DataType::LargeUtf8,
@@ -1616,7 +1622,6 @@ mod tests {
16161622
}
16171623

16181624
let unsupported_cases: Vec<DataType> = vec![
1619-
DataType::Float16,
16201625
DataType::Decimal256(76, 10),
16211626
// Invalid Time-unit combinations: Time32 is defined only for
16221627
// Second / Millisecond and Time64 only for Microsecond /
@@ -1694,14 +1699,77 @@ mod tests {
16941699
assert_eq!(actual.value(2), 20);
16951700
}
16961701

1702+
// `(Float16, Int32)` keys: ±0.0 collapse (stored as +0.0), NaNs collapse, and
1703+
// the Int32 key keeps `(0.0, 4)` distinct from `(±0.0, 3)`.
1704+
#[test]
1705+
fn test_group_values_column_float16() {
1706+
use half::f16;
1707+
1708+
let schema = Arc::new(Schema::new(vec![
1709+
Field::new("f", DataType::Float16, true),
1710+
Field::new("i", DataType::Int32, true),
1711+
]));
1712+
assert!(supported_schema(&schema));
1713+
let mut group_values =
1714+
GroupValuesColumn::<false>::try_new(Arc::clone(&schema)).unwrap();
1715+
1716+
let f: ArrayRef = Arc::new(Float16Array::from(vec![
1717+
Some(f16::from_f32(1.0)),
1718+
Some(f16::from_f32(-0.0)),
1719+
Some(f16::from_f32(0.0)),
1720+
Some(f16::from_f32(0.0)),
1721+
Some(f16::NAN),
1722+
Some(f16::NAN),
1723+
None,
1724+
None,
1725+
]));
1726+
let i: ArrayRef = Arc::new(Int32Array::from(vec![
1727+
Some(3),
1728+
Some(3),
1729+
Some(3),
1730+
Some(4),
1731+
Some(3),
1732+
Some(3),
1733+
Some(3),
1734+
Some(3),
1735+
]));
1736+
let mut groups = Vec::new();
1737+
group_values.intern(&[f, i], &mut groups).unwrap();
1738+
assert_eq!(groups, vec![0, 1, 1, 2, 3, 3, 4, 4]);
1739+
1740+
let emitted = group_values.emit(EmitTo::All).unwrap();
1741+
assert_eq!(emitted.len(), 2);
1742+
assert_eq!(emitted[0].data_type(), &DataType::Float16);
1743+
let keys = emitted[0]
1744+
.as_any()
1745+
.downcast_ref::<Float16Array>()
1746+
.expect("emitted column should be a Float16Array");
1747+
assert_eq!(keys.len(), 5);
1748+
assert_eq!(keys.value(0), f16::from_f32(1.0));
1749+
// The ±0.0 group is stored canonically as +0.0 (not -0.0).
1750+
assert_eq!(keys.value(1).to_bits(), f16::from_f32(0.0).to_bits());
1751+
assert_eq!(keys.value(2).to_bits(), f16::from_f32(0.0).to_bits());
1752+
assert!(keys.value(3).is_nan());
1753+
assert!(keys.is_null(4));
1754+
let ids = emitted[1]
1755+
.as_any()
1756+
.downcast_ref::<Int32Array>()
1757+
.expect("emitted column should be an Int32Array");
1758+
assert_eq!(ids.values().to_vec(), vec![3, 3, 4, 3, 3]);
1759+
}
1760+
16971761
#[test]
16981762
fn supported_schema_rejects_mix_of_supported_and_unsupported() {
1699-
// One Float16 column among supported columns flips the whole
1700-
// schema to GroupValuesRows fallback.
1763+
// One unsupported column flips the whole schema to the GroupValuesRows
1764+
// fallback. Time64(Second) stays invalid as new primitive builders land.
17011765
let schema = Schema::new(vec![
17021766
Field::new("a", DataType::Int32, true),
17031767
Field::new("b", DataType::Utf8, true),
1704-
Field::new("c", DataType::Float16, true),
1768+
Field::new(
1769+
"c",
1770+
DataType::Time64(arrow::datatypes::TimeUnit::Second),
1771+
true,
1772+
),
17051773
]);
17061774
assert!(!supported_schema(&schema));
17071775

@@ -1720,8 +1788,11 @@ mod tests {
17201788
// rejected at construction time rather than at first `intern`.
17211789
// `GroupValuesColumn` doesn't implement `Debug`, so explicit match
17221790
// instead of `unwrap_err`.
1723-
let schema =
1724-
Arc::new(Schema::new(vec![Field::new("x", DataType::Float16, true)]));
1791+
let schema = Arc::new(Schema::new(vec![Field::new(
1792+
"x",
1793+
DataType::Time64(arrow::datatypes::TimeUnit::Second),
1794+
true,
1795+
)]));
17251796
match GroupValuesColumn::<false>::try_new(schema) {
17261797
Ok(_) => panic!("expected NotImpl error, but try_new succeeded"),
17271798
Err(e) => {
@@ -1808,7 +1879,7 @@ mod tests {
18081879
// `emit(EmitTo::First(4))` calls can `take_n` without panicking.
18091880
// The hashmap entries below reference group indices 0..=11, so the
18101881
// single column builder needs at least 12 rows to back them.
1811-
let seed: ArrayRef = Arc::new(arrow::array::Int32Array::from(vec![0_i32; 12]));
1882+
let seed: ArrayRef = Arc::new(Int32Array::from(vec![0_i32; 12]));
18121883
for row in 0..12 {
18131884
group_values.group_values[0]
18141885
.append_val(&seed, row)

‎datafusion/sqllogictest/test_files/group_by.slt‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5721,3 +5721,27 @@ FROM duration_group_test GROUP BY column1, column2 ORDER BY column1, column2;
57215721

57225722
statement ok
57235723
DROP TABLE duration_group_test;
5724+
5725+
# Test multi group by int + Float16
5726+
statement ok
5727+
CREATE TABLE float16_group_test AS VALUES
5728+
(arrow_cast(1.5, 'Float16'), 1),
5729+
(arrow_cast(1.5, 'Float16'), 1),
5730+
(arrow_cast(2.5, 'Float16'), 2),
5731+
(arrow_cast(-0.0, 'Float16'), 3),
5732+
(arrow_cast(0.0, 'Float16'), 3);
5733+
5734+
# Single Float16 group key ({1.5, 2.5, ±0.0}) via the GroupValuesPrimitive path.
5735+
query I
5736+
SELECT count(*) FROM (SELECT column1 FROM float16_group_test GROUP BY column1);
5737+
----
5738+
3
5739+
5740+
# Multi-column GROUP BY: a primitive key and a Float16 key on the same path.
5741+
query I
5742+
SELECT count(*) FROM (SELECT column1, column2 FROM float16_group_test GROUP BY column1, column2);
5743+
----
5744+
3
5745+
5746+
statement ok
5747+
DROP TABLE float16_group_test;

0 commit comments

Comments
 (0)