Skip to content

Commit 3e058f0

Browse files
authored
feat: Implement state conversion for remaining group accumulators (#23275)
## Which issue does this PR close? <!-- We generally require a GitHub issue to be filed for all bug fixes and enhancements and this helps us generate change logs for our releases. You can link an issue to this PR using the GitHub syntax. For example `Closes #123` indicates that this PR will close issue #123. --> - Part of #23081. ## Rationale for this change <!-- Why are you proposing this change? If this is already explained clearly in the issue then this section is not needed. Explaining clearly why changes are proposed helps reviewers understand your changes and offer better suggestions for fixes. --> #23081 proposes making `GroupsAccumulator::convert_to_state` mandatory and removing `supports_convert_to_state`. Before that API cleanup can happen, all existing `GroupsAccumulator` implementations need a correct `convert_to_state` implementation. This PR is the first step toward that cleanup: it fills in missing `convert_to_state` implementations for the remaining group accumulators that currently participate in grouped aggregation. ## What changes are included in this PR? <!-- There is no need to duplicate the description in the issue here but it is sometimes worth providing a summary of the individual changes in this PR. --> This PR adds `convert_to_state` support for: - `PrimitiveDistinctCountGroupsAccumulator` - `HllGroupsAccumulator` for `approx_distinct` - `VarianceGroupsAccumulator` - `StddevGroupsAccumulator` - `CorrelationGroupsAccumulator` - `GeometricMeanGroupsAccumulator` in the advanced UDAF example ## Are these changes tested? <!-- We typically require tests for all PRs in order to: 1. Prevent the code from being accidentally broken by subsequent changes 2. Serve as another way to document the expected behavior of the code If tests are not included in your PR, please explain why (for example, are they covered by existing tests)? --> Yes. This PR adds unit tests for the new `convert_to_state` implementations. ## Are there any user-facing changes? <!-- If there are user-facing changes then we may require documentation to be updated before approving the PR. --> <!-- If there are any breaking changes to public APIs, please add the `api change` label. --> No. This PR does not remove `supports_convert_to_state` yet and does not change the public `GroupsAccumulator` API.
1 parent f269b94 commit 3e058f0

8 files changed

Lines changed: 622 additions & 5 deletions

File tree

‎datafusion-examples/examples/udf/advanced_udaf.rs‎

Lines changed: 41 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,8 +23,10 @@ use datafusion::{arrow::datatypes::DataType, logical_expr::Volatility};
2323
use std::sync::Arc;
2424

2525
use arrow::array::{
26-
ArrayRef, AsArray, Float32Array, PrimitiveArray, PrimitiveBuilder, UInt32Array,
26+
Array, ArrayRef, AsArray, BooleanArray, Float32Array, PrimitiveArray,
27+
PrimitiveBuilder, UInt32Array,
2728
};
29+
use arrow::buffer::NullBuffer;
2830
use arrow::datatypes::{ArrowNativeTypeOp, ArrowPrimitiveType, Float64Type, UInt32Type};
2931
use arrow::record_batch::RecordBatch;
3032
use arrow_schema::FieldRef;
@@ -237,7 +239,7 @@ impl GroupsAccumulator for GeometricMeanGroupsAccumulator {
237239
&mut self,
238240
values: &[ArrayRef],
239241
group_indices: &[usize],
240-
opt_filter: Option<&arrow::array::BooleanArray>,
242+
opt_filter: Option<&BooleanArray>,
241243
total_num_groups: usize,
242244
) -> Result<()> {
243245
assert_eq!(values.len(), 1, "single argument to update_batch");
@@ -359,6 +361,43 @@ impl GroupsAccumulator for GeometricMeanGroupsAccumulator {
359361
])
360362
}
361363

364+
fn convert_to_state(
365+
&self,
366+
values: &[ArrayRef],
367+
opt_filter: Option<&BooleanArray>,
368+
) -> Result<Vec<ArrayRef>> {
369+
assert_eq!(values.len(), 1, "single argument to convert_to_state");
370+
371+
let prods = values[0]
372+
.as_primitive::<Float64Type>()
373+
.clone()
374+
.with_data_type(self.prod_data_type.clone());
375+
let counts = UInt32Array::from_value(1, prods.len());
376+
377+
let filter_nulls = opt_filter.map(|filter| {
378+
let validity = match filter.nulls() {
379+
Some(nulls) => filter.values() & nulls.inner(),
380+
None => filter.values().clone(),
381+
};
382+
NullBuffer::new(validity)
383+
});
384+
let nulls = NullBuffer::union(filter_nulls.as_ref(), prods.nulls());
385+
386+
let prods =
387+
PrimitiveArray::<Float64Type>::new(prods.values().clone(), nulls.clone())
388+
.with_data_type(self.prod_data_type.clone());
389+
let counts = UInt32Array::new(counts.values().clone(), nulls);
390+
391+
Ok(vec![
392+
Arc::new(prods) as ArrayRef,
393+
Arc::new(counts) as ArrayRef,
394+
])
395+
}
396+
397+
fn supports_convert_to_state(&self) -> bool {
398+
true
399+
}
400+
362401
fn size(&self) -> usize {
363402
self.counts.capacity() * size_of::<u32>()
364403
+ self.prods.capacity() * size_of::<Float64Type>()

‎datafusion/core/tests/user_defined/user_defined_aggregates.rs‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -877,6 +877,22 @@ impl GroupsAccumulator for TestGroupsAccumulator {
877877
Ok(())
878878
}
879879

880+
fn convert_to_state(
881+
&self,
882+
values: &[ArrayRef],
883+
_opt_filter: Option<&arrow::array::BooleanArray>,
884+
) -> Result<Vec<ArrayRef>> {
885+
let len = values.first().map_or(0, |value| value.len());
886+
Ok(vec![
887+
Arc::new(PrimitiveArray::<UInt64Type>::from_value(self.result, len))
888+
as ArrayRef,
889+
])
890+
}
891+
892+
fn supports_convert_to_state(&self) -> bool {
893+
true
894+
}
895+
880896
fn size(&self) -> usize {
881897
size_of::<u64>()
882898
}

‎datafusion/functions-aggregate-common/src/aggregate/count_distinct/groups.rs‎

Lines changed: 120 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,8 @@
1616
// under the License.
1717

1818
use arrow::array::{
19-
ArrayRef, AsArray, BooleanArray, Int64Array, ListArray, PrimitiveArray,
19+
Array, ArrayRef, AsArray, BooleanArray, Int64Array, ListArray, ListBuilder,
20+
PrimitiveArray, PrimitiveBuilder,
2021
};
2122
use arrow::buffer::{OffsetBuffer, ScalarBuffer};
2223
use arrow::datatypes::{ArrowPrimitiveType, Field};
@@ -182,9 +183,127 @@ where
182183
Ok(())
183184
}
184185

186+
fn convert_to_state(
187+
&self,
188+
values: &[ArrayRef],
189+
opt_filter: Option<&BooleanArray>,
190+
) -> datafusion_common::Result<Vec<ArrayRef>> {
191+
debug_assert_eq!(values.len(), 1);
192+
let arr = values[0].as_primitive::<T>();
193+
194+
let values_builder = PrimitiveBuilder::<T>::with_capacity(arr.len());
195+
let mut builder = ListBuilder::new(values_builder)
196+
.with_field(Arc::new(Field::new_list_field(T::DATA_TYPE, true)));
197+
198+
for row in 0..arr.len() {
199+
let included = arr.is_valid(row)
200+
&& opt_filter
201+
.is_none_or(|filter| filter.is_valid(row) && filter.value(row));
202+
if included {
203+
builder.values().append_value(arr.value(row));
204+
}
205+
builder.append(true);
206+
}
207+
208+
Ok(vec![Arc::new(builder.finish())])
209+
}
210+
211+
fn supports_convert_to_state(&self) -> bool {
212+
true
213+
}
214+
185215
fn size(&self) -> usize {
186216
size_of::<Self>()
187217
+ self.seen.capacity() * (size_of::<(usize, T::Native)>() + size_of::<u64>())
188218
+ self.counts.capacity() * size_of::<i64>()
189219
}
190220
}
221+
222+
#[cfg(test)]
223+
mod tests {
224+
use super::*;
225+
use arrow::array::Int32Array;
226+
use arrow::datatypes::Int32Type;
227+
use datafusion_common::Result;
228+
229+
#[test]
230+
fn convert_to_state_roundtrips_through_merge() -> Result<()> {
231+
let values = Arc::new(Int32Array::from(vec![
232+
Some(1),
233+
Some(2),
234+
Some(2),
235+
None,
236+
Some(3),
237+
Some(4),
238+
Some(5),
239+
Some(5),
240+
])) as ArrayRef;
241+
let filter = BooleanArray::from(vec![
242+
Some(true),
243+
Some(true),
244+
Some(true),
245+
Some(true),
246+
None,
247+
Some(true),
248+
Some(true),
249+
Some(true),
250+
]);
251+
let group_indices = vec![0usize, 1, 0, 1, 0, 0, 0, 0];
252+
253+
let mut direct = PrimitiveDistinctCountGroupsAccumulator::<Int32Type>::new();
254+
direct.update_batch(
255+
std::slice::from_ref(&values),
256+
&group_indices,
257+
Some(&filter),
258+
2,
259+
)?;
260+
let direct = direct.evaluate(EmitTo::All)?;
261+
262+
let converter = PrimitiveDistinctCountGroupsAccumulator::<Int32Type>::new();
263+
let state =
264+
converter.convert_to_state(std::slice::from_ref(&values), Some(&filter))?;
265+
assert_eq!(state[0].null_count(), 0);
266+
let mut merged = PrimitiveDistinctCountGroupsAccumulator::<Int32Type>::new();
267+
merged.merge_batch(&state, &group_indices, 2)?;
268+
let merged = merged.evaluate(EmitTo::All)?;
269+
270+
assert_eq!(
271+
direct.as_any().downcast_ref::<Int64Array>().unwrap(),
272+
merged.as_any().downcast_ref::<Int64Array>().unwrap()
273+
);
274+
Ok(())
275+
}
276+
277+
#[test]
278+
fn convert_to_state_preserves_empty_and_filtered_rows() -> Result<()> {
279+
let converter = PrimitiveDistinctCountGroupsAccumulator::<Int32Type>::new();
280+
let empty_values =
281+
Arc::new(Int32Array::from(Vec::<Option<i32>>::new())) as ArrayRef;
282+
let state =
283+
converter.convert_to_state(std::slice::from_ref(&empty_values), None)?;
284+
assert_eq!(state[0].len(), 0);
285+
assert_eq!(state[0].null_count(), 0);
286+
287+
let values = Arc::new(Int32Array::from(vec![Some(1), Some(2), None])) as ArrayRef;
288+
let filter = BooleanArray::from(vec![Some(false), None, Some(false)]);
289+
let group_indices = vec![0usize, 1, 0];
290+
291+
let state =
292+
converter.convert_to_state(std::slice::from_ref(&values), Some(&filter))?;
293+
assert_eq!(state[0].len(), values.len());
294+
assert_eq!(state[0].null_count(), 0);
295+
let list_state = state[0].as_list::<i32>();
296+
for row in 0..list_state.len() {
297+
assert_eq!(list_state.value_length(row), 0);
298+
}
299+
300+
let mut merged = PrimitiveDistinctCountGroupsAccumulator::<Int32Type>::new();
301+
merged.merge_batch(&state, &group_indices, 2)?;
302+
let result = merged.evaluate(EmitTo::All)?;
303+
assert_eq!(
304+
result.as_any().downcast_ref::<Int64Array>().unwrap(),
305+
&Int64Array::from(vec![0, 0])
306+
);
307+
Ok(())
308+
}
309+
}

‎datafusion/functions-aggregate/src/approx_distinct.rs‎

Lines changed: 129 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -583,6 +583,43 @@ impl GroupsAccumulator for HllGroupsAccumulator {
583583
Ok(vec![Arc::new(builder.finish())])
584584
}
585585

586+
fn convert_to_state(
587+
&self,
588+
values: &[ArrayRef],
589+
opt_filter: Option<&BooleanArray>,
590+
) -> Result<Vec<ArrayRef>> {
591+
assert_eq!(values.len(), 1, "single argument to convert_to_state");
592+
let array = values[0].as_ref();
593+
let mut hashes = vec![0; array.len()];
594+
create_hashes([array], &HLL_HASH_STATE, &mut hashes)?;
595+
596+
let filter_nulls = opt_filter.map(filter_to_nulls);
597+
let value_nulls = array.logical_nulls();
598+
let combined_nulls =
599+
NullBuffer::union(filter_nulls.as_ref(), value_nulls.as_ref());
600+
601+
let mut builder = BinaryBuilder::new();
602+
let mut scratch = Vec::new();
603+
for (row, hash) in hashes.into_iter().enumerate() {
604+
if combined_nulls
605+
.as_ref()
606+
.is_none_or(|nulls| nulls.is_valid(row))
607+
{
608+
scratch.clear();
609+
scratch.extend_from_slice(&hash.to_le_bytes());
610+
builder.append_value(&scratch);
611+
} else {
612+
builder.append_value([]);
613+
}
614+
}
615+
616+
Ok(vec![Arc::new(builder.finish())])
617+
}
618+
619+
fn supports_convert_to_state(&self) -> bool {
620+
true
621+
}
622+
586623
fn size(&self) -> usize {
587624
self.groups.capacity() * size_of::<GroupHll>()
588625
+ self.allocated_bytes
@@ -1081,6 +1118,98 @@ mod tests {
10811118
assert_eq!(counts.value(0), expected);
10821119
}
10831120

1121+
#[test]
1122+
fn groups_convert_to_state_roundtrips_through_merge() {
1123+
let values: ArrayRef = Arc::new(Int64Array::from(vec![
1124+
Some(1),
1125+
Some(2),
1126+
Some(2),
1127+
None,
1128+
Some(3),
1129+
]));
1130+
let filter = BooleanArray::from(vec![
1131+
Some(true),
1132+
Some(true),
1133+
Some(true),
1134+
Some(true),
1135+
None,
1136+
]);
1137+
let group_indices = vec![0usize, 1, 0, 1, 0];
1138+
1139+
let mut direct = HllGroupsAccumulator::new();
1140+
direct
1141+
.update_batch(
1142+
std::slice::from_ref(&values),
1143+
&group_indices,
1144+
Some(&filter),
1145+
2,
1146+
)
1147+
.unwrap();
1148+
let direct = direct
1149+
.evaluate(EmitTo::All)
1150+
.unwrap()
1151+
.as_any()
1152+
.downcast_ref::<UInt64Array>()
1153+
.unwrap()
1154+
.clone();
1155+
1156+
let converter = HllGroupsAccumulator::new();
1157+
let state = converter
1158+
.convert_to_state(std::slice::from_ref(&values), Some(&filter))
1159+
.unwrap();
1160+
assert_eq!(state[0].null_count(), 0);
1161+
let mut merged = HllGroupsAccumulator::new();
1162+
merged.merge_batch(&state, &group_indices, 2).unwrap();
1163+
let merged = merged
1164+
.evaluate(EmitTo::All)
1165+
.unwrap()
1166+
.as_any()
1167+
.downcast_ref::<UInt64Array>()
1168+
.unwrap()
1169+
.clone();
1170+
1171+
assert_eq!(direct, merged);
1172+
}
1173+
1174+
#[test]
1175+
fn groups_convert_to_state_preserves_empty_and_filtered_rows() {
1176+
let converter = HllGroupsAccumulator::new();
1177+
let empty_values: ArrayRef =
1178+
Arc::new(Int64Array::from(Vec::<Option<i64>>::new()));
1179+
let state = converter
1180+
.convert_to_state(std::slice::from_ref(&empty_values), None)
1181+
.unwrap();
1182+
assert_eq!(state[0].len(), 0);
1183+
assert_eq!(state[0].null_count(), 0);
1184+
1185+
let values: ArrayRef =
1186+
Arc::new(Int64Array::from(vec![Some(1), Some(2), None]));
1187+
let filter = BooleanArray::from(vec![Some(false), None, Some(false)]);
1188+
let group_indices = vec![0usize, 1, 0];
1189+
let state = converter
1190+
.convert_to_state(std::slice::from_ref(&values), Some(&filter))
1191+
.unwrap();
1192+
assert_eq!(state[0].len(), values.len());
1193+
assert_eq!(state[0].null_count(), 0);
1194+
let state = state[0].as_any().downcast_ref::<BinaryArray>().unwrap();
1195+
for row in 0..state.len() {
1196+
assert_eq!(state.value(row), b"");
1197+
}
1198+
1199+
let mut merged = HllGroupsAccumulator::new();
1200+
merged
1201+
.merge_batch(&[Arc::new(state.clone())], &group_indices, 2)
1202+
.unwrap();
1203+
let result = merged
1204+
.evaluate(EmitTo::All)
1205+
.unwrap()
1206+
.as_any()
1207+
.downcast_ref::<UInt64Array>()
1208+
.unwrap()
1209+
.clone();
1210+
assert_eq!(result, UInt64Array::from(vec![0, 0]));
1211+
}
1212+
10841213
/// Regression: a short (≤ 12-byte) Utf8View string must hash identically
10851214
/// in an all-inline batch and in a mixed batch that also contains a long
10861215
/// string (which forces a data buffer).

0 commit comments

Comments
 (0)