Skip to content

Commit 8588db1

Browse files
committed
test(functions-aggregate): cover ordered ARRAY_AGG coalescing payloads
Add tests that verify evaluated order after coalescing Utf8View and Dictionary payloads across the 64-row boundary. Add a test confirming that large Utf8 payloads stay in separate batches. Rename the existing Int64 ordering test to make its payload type explicit.
1 parent 970091b commit 8588db1

1 file changed

Lines changed: 132 additions & 1 deletion

File tree

‎datafusion/functions-aggregate/src/array_agg.rs‎

Lines changed: 132 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2999,7 +2999,7 @@ mod tests {
29992999
}
30003000

30013001
#[test]
3002-
fn ordered_array_agg_sorts_across_coalesced_batches() -> Result<()> {
3002+
fn ordered_array_agg_sorts_across_coalesced_int64_batches() -> Result<()> {
30033003
use arrow::array::Int64Array;
30043004

30053005
let mut acc = ordered_accumulator(
@@ -3031,6 +3031,137 @@ mod tests {
30313031
Ok(())
30323032
}
30333033

3034+
#[test]
3035+
fn ordered_array_agg_does_not_coalesce_large_payloads() -> Result<()> {
3036+
use arrow::array::{Int64Array, StringArray};
3037+
3038+
let mut acc = ordered_accumulator(
3039+
DataType::Utf8,
3040+
DataType::Int64,
3041+
SortOptions::new(false, false),
3042+
false,
3043+
false,
3044+
)?;
3045+
3046+
let value = "x".repeat(ORDERED_ARRAY_AGG_COALESCE_BYTES);
3047+
3048+
for i in 0_i64..2 {
3049+
let payload = Arc::new(StringArray::from(vec![value.as_str()])) as ArrayRef;
3050+
let ordering = Arc::new(Int64Array::from(vec![i])) as ArrayRef;
3051+
acc.update_batch(&[payload, ordering])?;
3052+
}
3053+
3054+
// Each payload exceeds the byte cap, so the tail batch must not be coalesced.
3055+
assert_eq!(acc.batches.len(), 2);
3056+
assert_eq!(acc.entries[1].batch_idx, 1);
3057+
assert_eq!(acc.entries[1].row_idx, 0);
3058+
Ok(())
3059+
}
3060+
3061+
#[test]
3062+
fn ordered_array_agg_sorts_across_coalesced_utf8_view_batches() -> Result<()> {
3063+
use arrow::array::{Int64Array, StringViewArray};
3064+
3065+
let mut acc = ordered_accumulator(
3066+
DataType::Utf8View,
3067+
DataType::Int64,
3068+
SortOptions::new(false, false),
3069+
false,
3070+
false,
3071+
)?;
3072+
3073+
// Longer than 12 bytes so view payloads are stored in data buffers.
3074+
let value = |i: usize| format!("payload-value-{i:03}");
3075+
let total_payloads = 100;
3076+
3077+
for i in (0..total_payloads).rev() {
3078+
let payload =
3079+
Arc::new(StringViewArray::from_iter_values([value(i)])) as ArrayRef;
3080+
let ordering =
3081+
Arc::new(Int64Array::from(vec![i64::try_from(i).unwrap()])) as ArrayRef;
3082+
acc.update_batch(&[payload, ordering])?;
3083+
}
3084+
3085+
assert_eq!(
3086+
acc.batches
3087+
.iter()
3088+
.map(|batch| batch.len())
3089+
.collect::<Vec<_>>(),
3090+
vec![ORDERED_ARRAY_AGG_COALESCE_ROWS, 36],
3091+
);
3092+
3093+
let ScalarValue::List(result) = acc.evaluate()? else {
3094+
return internal_err!("expect list");
3095+
};
3096+
3097+
let values = result
3098+
.values()
3099+
.as_any()
3100+
.downcast_ref::<StringViewArray>()
3101+
.expect("expected StringViewArray");
3102+
3103+
for i in 0..total_payloads {
3104+
assert_eq!(values.value(i), value(i));
3105+
}
3106+
3107+
Ok(())
3108+
}
3109+
3110+
#[test]
3111+
fn ordered_array_agg_sorts_across_coalesced_dictionary_batches() -> Result<()> {
3112+
use arrow::array::{ArrayAccessor, DictionaryArray, Int64Array, StringArray};
3113+
use arrow::datatypes::Int8Type;
3114+
3115+
let dictionary_type =
3116+
DataType::Dictionary(Box::new(DataType::Int8), Box::new(DataType::Utf8));
3117+
let mut acc = ordered_accumulator(
3118+
dictionary_type,
3119+
DataType::Int64,
3120+
SortOptions::new(false, false),
3121+
false,
3122+
false,
3123+
)?;
3124+
3125+
// Each input has its own dictionary, so coalescing must remap its keys.
3126+
let value = |i: usize| format!("payload-value-{i:03}");
3127+
let total_payloads = 100;
3128+
3129+
for i in (0..total_payloads).rev() {
3130+
let payload = Arc::new(
3131+
std::iter::once(value(i).as_str()).collect::<DictionaryArray<Int8Type>>(),
3132+
) as ArrayRef;
3133+
let ordering =
3134+
Arc::new(Int64Array::from(vec![i64::try_from(i).unwrap()])) as ArrayRef;
3135+
acc.update_batch(&[payload, ordering])?;
3136+
}
3137+
3138+
assert_eq!(
3139+
acc.batches
3140+
.iter()
3141+
.map(|batch| batch.len())
3142+
.collect::<Vec<_>>(),
3143+
vec![ORDERED_ARRAY_AGG_COALESCE_ROWS, 36],
3144+
);
3145+
3146+
let ScalarValue::List(result) = acc.evaluate()? else {
3147+
return internal_err!("expect list");
3148+
};
3149+
3150+
let values = result
3151+
.values()
3152+
.as_any()
3153+
.downcast_ref::<DictionaryArray<Int8Type>>()
3154+
.expect("expected DictionaryArray<Int8Type>")
3155+
.downcast_dict::<StringArray>()
3156+
.expect("expected Utf8 dictionary values");
3157+
3158+
for i in 0..total_payloads {
3159+
assert_eq!(values.value(i), value(i));
3160+
}
3161+
3162+
Ok(())
3163+
}
3164+
30343165
struct ArrayAggAccumulatorBuilder {
30353166
return_field: FieldRef,
30363167
distinct: bool,

0 commit comments

Comments
 (0)