Skip to content

Commit 132905e

Browse files
committed
Address review feedback on RowsGroupColumn PR
Fixes from @Rich-T-kid and @copilot-pull-request-reviewer: * row.rs: correct the doc on `encode_array_if_necessary` — arrow's `RowConverter` flattens dictionary / REE values at `append()` (during row encoding), not "on the way out". * row.rs: extend the recursion in `encode_array_if_necessary` to cover `FixedSizeList`, `LargeList`, and `Map` in addition to `Struct` and `List`. Without this, a nested key like `List<Dict<Int32, Utf8>>` emits as `List<Utf8>` because the fallback arm just clones the RowConverter-flattened array, giving a data-type mismatch against the declared output type. * row.rs: rename `dictionary_encode_if_necessary` → `encode_array_if_necessary` since the helper also handles run-end encoding (and now the nested container walks around them). * row_backed.rs: add `build_preserves_list_of_dictionary_schema` — a round-trip regression test that appends a `List<Dict<Int32, Utf8>>` row via `RowsGroupColumn` and asserts the built array's data type matches the declared schema (would fail with the pre-fix helper). * multi_group_by/mod.rs: expand the doc on `canonical_grouping` to spell out that the `GroupValues` trait deliberately does not fix the order in which fresh group-ids are assigned — canonicalizing before comparison is what lets the vectorized column path and the sequential rows fallback be asserted equivalent. * row_backed.rs (module doc): reword "instead of dragging the entire key onto GroupValuesRows" → "instead of dragging both columns onto GroupValuesRows" per Rich-T-kid's suggestion diff.
1 parent cef4ed4 commit 132905e

3 files changed

Lines changed: 115 additions & 17 deletions

File tree

‎datafusion/physical-plan/src/aggregates/group_values/multi_group_by/mod.rs‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1398,6 +1398,13 @@ mod tests {
13981398
/// forms are equal — this ignores the (opaque, non-semantic) difference in
13991399
/// group-index numbering between the vectorized column path and the
14001400
/// sequential rows fallback.
1401+
///
1402+
/// The [`GroupValues`] trait only guarantees that equal keys receive the
1403+
/// same group-id and that new keys receive a fresh id; the order in which
1404+
/// new ids are handed out is deliberately not part of the contract, and
1405+
/// can differ between correct implementations (e.g. because of internal
1406+
/// hash-map ordering). Canonicalizing before comparison is what lets us
1407+
/// assert equivalence across implementations.
14011408
fn canonical_grouping(groups: &[usize]) -> Vec<usize> {
14021409
let mut map = HashMap::new();
14031410
let mut next = 0usize;

‎datafusion/physical-plan/src/aggregates/group_values/multi_group_by/row_backed.rs‎

Lines changed: 51 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -32,8 +32,8 @@
3232
//! and heavier for the columns that *would* have qualified for the column-wise
3333
//! fast path. By providing a generic fallback `GroupColumn`, a schema like
3434
//! `GROUP BY int_col, struct_col` keeps `int_col` on its fast native builder
35-
//! and only pays the row-encoding cost on `struct_col`, instead of dragging the
36-
//! entire key onto `GroupValuesRows`.
35+
//! and only pays the row-encoding cost on `struct_col`, instead of dragging both
36+
//! columns onto `GroupValuesRows`.
3737
//!
3838
//! # Relationship to hashing
3939
//!
@@ -47,7 +47,7 @@
4747
//! [`GroupValuesRows`]: crate::aggregates::group_values::GroupValuesRows
4848
4949
use crate::aggregates::group_values::multi_group_by::GroupColumn;
50-
use crate::aggregates::group_values::row::dictionary_encode_if_necessary;
50+
use crate::aggregates::group_values::row::encode_array_if_necessary;
5151

5252
use arrow::array::{Array, ArrayRef, BooleanBufferBuilder};
5353
use arrow::datatypes::DataType;
@@ -118,7 +118,7 @@ impl RowsGroupColumn {
118118
.expect("row conversion during emit");
119119
debug_assert_eq!(arrays.len(), 1, "single-field row converter");
120120
let array = arrays.swap_remove(0);
121-
dictionary_encode_if_necessary(&array, &self.output_type)
121+
encode_array_if_necessary(&array, &self.output_type)
122122
.expect("dictionary re-encode during emit")
123123
}
124124

@@ -370,4 +370,51 @@ mod tests {
370370
RowConverter::supports_fields(&[SortField::new(map_dt.clone())]);
371371
assert_eq!(RowsGroupColumn::supports_type(&map_dt), arrow_supports);
372372
}
373+
374+
/// Regression test for the nested-container recursion in
375+
/// [`crate::aggregates::group_values::row::encode_array_if_necessary`].
376+
/// `RowConverter` flattens dictionary values on the way in, so a
377+
/// `List<Dict<Int32, Utf8>>` schema round-trips with `Utf8` values
378+
/// unless the helper re-encodes the leaf. Without that recursion,
379+
/// `build()` would emit an array whose data type does not match the
380+
/// group column's declared type.
381+
#[test]
382+
fn build_preserves_list_of_dictionary_schema() {
383+
use arrow::array::{DictionaryArray, ListArray, StringArray};
384+
use arrow::buffer::OffsetBuffer;
385+
use arrow::datatypes::Int32Type;
386+
387+
let dict_dt =
388+
DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Utf8));
389+
let item_field = Arc::new(Field::new("item", dict_dt.clone(), true));
390+
let outer_dt = DataType::List(Arc::clone(&item_field));
391+
392+
// Skip if this arrow-rs version rejects the nesting — the invariant we
393+
// care about is `output().data_type() == declared type` conditional on
394+
// supports_type saying yes.
395+
if !RowsGroupColumn::supports_type(&outer_dt) {
396+
return;
397+
}
398+
399+
let mut col = Box::new(RowsGroupColumn::try_new(outer_dt.clone()).unwrap());
400+
401+
// Build List<Dict<Int32,Utf8>> of one row = ["a", "b"].
402+
let values = Arc::new(StringArray::from(vec!["a", "b"]));
403+
let keys = Int32Array::from(vec![0, 1]);
404+
let dict = DictionaryArray::<Int32Type>::try_new(keys, values).unwrap();
405+
let offsets = OffsetBuffer::from_lengths([2]);
406+
let list =
407+
ListArray::try_new(Arc::clone(&item_field), offsets, Arc::new(dict), None)
408+
.unwrap();
409+
let input: ArrayRef = Arc::new(list);
410+
411+
col.vectorized_append(&input, &[0]).unwrap();
412+
let built = col.build();
413+
assert_eq!(
414+
built.data_type(),
415+
&outer_dt,
416+
"build() must return the declared List<Dict> data type, \
417+
not the RowConverter-flattened List<Utf8>",
418+
);
419+
}
373420
}

‎datafusion/physical-plan/src/aggregates/group_values/row.rs‎

Lines changed: 57 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -17,8 +17,8 @@
1717

1818
use crate::aggregates::group_values::GroupValues;
1919
use arrow::array::{
20-
Array, ArrayRef, ListArray, PrimitiveArray, RunArray, StructArray,
21-
downcast_run_end_index,
20+
Array, ArrayRef, FixedSizeListArray, LargeListArray, ListArray, MapArray,
21+
PrimitiveArray, RunArray, StructArray, downcast_run_end_index,
2222
};
2323
use arrow::compute::cast;
2424
use arrow::datatypes::{DataType, SchemaRef};
@@ -247,7 +247,7 @@ impl GroupValues for GroupValuesRows {
247247
// https://github.com/apache/datafusion/issues/7647
248248
for (field, array) in self.schema.fields.iter().zip(&mut output) {
249249
let expected = field.data_type();
250-
*array = dictionary_encode_if_necessary(array, expected)?;
250+
*array = encode_array_if_necessary(array, expected)?;
251251
}
252252

253253
self.group_values = Some(group_values);
@@ -269,13 +269,15 @@ impl GroupValues for GroupValuesRows {
269269

270270
/// Re-apply dictionary / run-end encoding to `array` so it matches `expected`.
271271
///
272-
/// Arrow's [`RowConverter`] decodes dictionary and run-end-encoded values to
273-
/// their plain value type on the way out, so any group-value array produced
274-
/// from the row format must be re-encoded to the schema's expected type before
272+
/// Arrow's [`RowConverter`] flattens dictionary and run-end-encoded values to
273+
/// their plain value type during row encoding (at [`RowConverter::append`]),
274+
/// so any group-value array produced from the row format is in that plain
275+
/// type and must be re-encoded to match the schema's expected type before
275276
/// being returned. Shared with the generic row-backed `GroupColumn`.
276277
///
277278
/// [`RowConverter`]: arrow::row::RowConverter
278-
pub(crate) fn dictionary_encode_if_necessary(
279+
/// [`RowConverter::append`]: arrow::row::RowConverter::append
280+
pub(crate) fn encode_array_if_necessary(
279281
array: &ArrayRef,
280282
expected: &DataType,
281283
) -> Result<ArrayRef> {
@@ -286,7 +288,7 @@ pub(crate) fn dictionary_encode_if_necessary(
286288
.iter()
287289
.zip(struct_array.columns())
288290
.map(|(expected_field, column)| {
289-
dictionary_encode_if_necessary(column, expected_field.data_type())
291+
encode_array_if_necessary(column, expected_field.data_type())
290292
})
291293
.collect::<Result<Vec<_>>>()?;
292294

@@ -302,13 +304,55 @@ pub(crate) fn dictionary_encode_if_necessary(
302304
Ok(Arc::new(ListArray::try_new(
303305
Arc::<arrow::datatypes::Field>::clone(expected_field),
304306
list.offsets().clone(),
305-
dictionary_encode_if_necessary(
306-
list.values(),
307-
expected_field.data_type(),
308-
)?,
307+
encode_array_if_necessary(list.values(), expected_field.data_type())?,
309308
list.nulls().cloned(),
310309
)?))
311310
}
311+
(DataType::LargeList(expected_field), &DataType::LargeList(_)) => {
312+
let list = array.as_any().downcast_ref::<LargeListArray>().unwrap();
313+
314+
Ok(Arc::new(LargeListArray::try_new(
315+
Arc::<arrow::datatypes::Field>::clone(expected_field),
316+
list.offsets().clone(),
317+
encode_array_if_necessary(list.values(), expected_field.data_type())?,
318+
list.nulls().cloned(),
319+
)?))
320+
}
321+
(
322+
DataType::FixedSizeList(expected_field, expected_size),
323+
&DataType::FixedSizeList(_, _),
324+
) => {
325+
let list = array.as_any().downcast_ref::<FixedSizeListArray>().unwrap();
326+
327+
Ok(Arc::new(FixedSizeListArray::try_new(
328+
Arc::<arrow::datatypes::Field>::clone(expected_field),
329+
*expected_size,
330+
encode_array_if_necessary(list.values(), expected_field.data_type())?,
331+
list.nulls().cloned(),
332+
)?))
333+
}
334+
(DataType::Map(expected_entries_field, ordered), &DataType::Map(_, _)) => {
335+
let map = array.as_any().downcast_ref::<MapArray>().unwrap();
336+
// Re-encode the entries `StructArray` (which holds key/value
337+
// columns) against the expected entries field's struct type.
338+
let entries_as_ref: ArrayRef = Arc::new(map.entries().clone());
339+
let entries = encode_array_if_necessary(
340+
&entries_as_ref,
341+
expected_entries_field.data_type(),
342+
)?;
343+
let entries = entries
344+
.as_any()
345+
.downcast_ref::<StructArray>()
346+
.expect("Map entries recurse must yield a StructArray")
347+
.clone();
348+
Ok(Arc::new(MapArray::try_new(
349+
Arc::<arrow::datatypes::Field>::clone(expected_entries_field),
350+
map.offsets().clone(),
351+
entries,
352+
map.nulls().cloned(),
353+
*ordered,
354+
)?))
355+
}
312356
(DataType::Dictionary(_, _), _) => Ok(cast(array.as_ref(), expected)?),
313357
(
314358
DataType::RunEndEncoded(run_ends_field, expected_values_field),
@@ -320,7 +364,7 @@ pub(crate) fn dictionary_encode_if_necessary(
320364
.as_any()
321365
.downcast_ref::<RunArray<$run_end_type>>()
322366
.unwrap();
323-
let values = dictionary_encode_if_necessary(
367+
let values = encode_array_if_necessary(
324368
&(Arc::clone(run_array.values()) as ArrayRef),
325369
expected_values_field.data_type(),
326370
)?;

0 commit comments

Comments
 (0)