Skip to content

Commit 6f1436f

Browse files
authored
fix: compact FIRST_VALUE scalars during merge (apache#26743)
## Which issue does this PR close? Closes apache#26744. ## Rationale for this change `DISTINCT ON` queries carrying nested payloads can fail with a memory reservation error during aggregate spill/replay because merged `FIRST_VALUE` states retain backing buffers from larger arrays. ## What changes are included in this PR? Call `self.first.compact()` after extracting the selected scalar in `TrivialFirstValueAccumulator::merge_batch`, matching the existing behavior in `update_batch`. ## What is the testing strategy for this PR? Locally validated against unpatched and patched DataFusion 55.2.0. An accumulator regression check confirms that merging a list state preserves its value and reports the same memory size as updating it directly; it fails before the fix and passes afterward. The spill workload described in apache#26744 fails before the fix and returns all 8,192 rows with correct nested payloads afterward, while still exercising spilling. ## Are there any user-facing changes? No.
1 parent 97ff724 commit 6f1436f

2 files changed

Lines changed: 47 additions & 0 deletions

File tree

‎datafusion/functions-aggregate/src/first_last.rs‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -977,6 +977,7 @@ impl Accumulator for TrivialFirstValueAccumulator {
977977
&& !first.is_empty()
978978
{
979979
self.first = ScalarValue::try_from_array(first, 0)?;
980+
self.first.compact();
980981
self.is_set = true;
981982
}
982983
}
@@ -2972,4 +2973,26 @@ mod tests {
29722973
assert_eq!(int64_values(&out), vec![Some(500)]);
29732974
Ok(())
29742975
}
2976+
2977+
#[test]
2978+
fn test_first_value_merge_compacts_list_state() -> Result<()> {
2979+
let values: ArrayRef =
2980+
Arc::new(ListArray::from_iter_primitive::<Int32Type, _, _>(
2981+
repeat_with(|| Some(vec![Some(1)])).take(10_000),
2982+
));
2983+
2984+
let mut updated =
2985+
TrivialFirstValueAccumulator::try_new(values.data_type(), false)?;
2986+
let mut merged =
2987+
TrivialFirstValueAccumulator::try_new(values.data_type(), false)?;
2988+
2989+
updated.update_batch(&[Arc::clone(&values)])?;
2990+
2991+
let flags: ArrayRef = Arc::new(BooleanArray::from(vec![true; values.len()]));
2992+
merged.merge_batch(&[values, flags])?;
2993+
2994+
assert_eq!(merged.evaluate()?, updated.evaluate()?);
2995+
assert_eq!(merged.size(), updated.size());
2996+
Ok(())
2997+
}
29752998
}

‎datafusion/sqllogictest/test_files/aggregate_memory_spill.slt‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -239,6 +239,30 @@ FROM (
239239
08)--------------AggregateExec: mode=Partial, gby=[v@0 * 7 % 100000 as t.v * Int64(7) % Int64(100000)], aggr=[sum(t.v)], metrics=[<slt:ignore>early_emit_count=<slt:ignore>]
240240
<slt:ignore>
241241

242+
# --- Case H: DISTINCT ON with nested FIRST_VALUE state after spilling ---
243+
# Merged list states must not retain the entire replay batch for every group.
244+
# Check the complete nested payload as well as row count.
245+
statement ok
246+
SET datafusion.execution.target_partitions = 1
247+
248+
query II
249+
SELECT count(*), count(*) FILTER (
250+
WHERE contents = [named_struct('key', k, 'value', repeat('v', 224))]
251+
)
252+
FROM (
253+
SELECT DISTINCT ON (k) k,
254+
[named_struct('key', k, 'value', repeat('v', 224))] AS contents
255+
FROM (
256+
SELECT (v * 7) % 8192 AS k
257+
FROM generate_series(0, 8191) AS t(v)
258+
)
259+
)
260+
----
261+
8192 8192
262+
263+
statement ok
264+
SET datafusion.execution.target_partitions = 4
265+
242266
# Restore settings to slt runner defaults
243267
statement ok
244268
RESET datafusion.runtime.memory_limit

0 commit comments

Comments
 (0)