Skip to content

Commit e19fc94

Browse files
viiryakosiew
authored andcommitted
fix: correct percentile_cont(DISTINCT) accumulation and sliding-window retract (apache#23913)
## Which issue does this PR close? - Closes apache#23912. ## Rationale for this change `DistinctPercentileContAccumulator` reused the shared set-based `GenericDistinctBuffer`, which doesn't fit it: (1) the buffer asserts a single input column but `percentile_cont` passes two (value + percentile), so every `percentile_cont(DISTINCT ...)` panicked; (2) the buffer is a plain `HashSet` with no multiplicity, so sliding-window `retract_batch` dropped a value while duplicates were still in the frame. ## What changes are included in this PR? - Replace the shared buffer in this accumulator with a per-accumulator `HashMap<Hashable, usize>` count map: `update_batch` reads only the value column and increments; `retract_batch` decrements and removes a key only at zero; `state`/`merge_batch` keep the same List state shape. Other `GenericDistinctBuffer` users are untouched. - Regression tests in `aggregate.slt` for the plain distinct query and the sliding-window duplicate-retract case. ## Are these changes tested? Yes — new regression tests; the full `aggregate.slt` suite passes. ## Are there any user-facing changes? `percentile_cont(DISTINCT ...)` now works instead of panicking, and returns correct results in sliding windows.
1 parent 6504756 commit e19fc94

2 files changed

Lines changed: 75 additions & 11 deletions

File tree

‎datafusion/functions-aggregate/src/percentile_cont.rs‎

Lines changed: 48 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,7 @@ use datafusion_functions_aggregate_common::noop_accumulator::NoopAccumulator;
3939
use crate::min_max::{max_udaf, min_udaf};
4040
use datafusion_common::{
4141
Result, ScalarValue, exec_datafusion_err, internal_datafusion_err,
42-
utils::take_function_args,
42+
utils::{SingleRowListArrayBuilder, take_function_args},
4343
};
4444
use datafusion_expr::utils::format_state_name;
4545
use datafusion_expr::{
@@ -54,7 +54,7 @@ use datafusion_expr::{
5454
};
5555
use datafusion_functions_aggregate_common::aggregate::groups_accumulator::accumulate::accumulate;
5656
use datafusion_functions_aggregate_common::aggregate::groups_accumulator::nulls::filtered_null_mask;
57-
use datafusion_functions_aggregate_common::utils::{GenericDistinctBuffer, Hashable};
57+
use datafusion_functions_aggregate_common::utils::Hashable;
5858
use datafusion_macros::user_doc;
5959

6060
use crate::utils::validate_percentile_expr;
@@ -662,16 +662,24 @@ where
662662
}
663663
}
664664

665+
/// Sliding-window–capable accumulator for `percentile_cont(DISTINCT ...)`.
666+
///
667+
/// Distinct values are tracked with a per-value multiplicity count (how many
668+
/// rows currently in the window carry that value) rather than a plain set, so
669+
/// that `retract_batch` only drops a value once *all* of its occurrences have
670+
/// left the window frame. The percentile is then computed over the set of keys
671+
/// with a positive count.
665672
#[derive(Debug)]
666673
struct DistinctPercentileContAccumulator<T: ArrowNumericType> {
667-
distinct_values: GenericDistinctBuffer<T>,
674+
/// Distinct value -> number of in-window rows carrying it.
675+
counts: HashMap<Hashable<T::Native>, usize>,
668676
percentile: f64,
669677
}
670678

671679
impl<T: ArrowNumericType + Debug> DistinctPercentileContAccumulator<T> {
672680
fn new(percentile: f64) -> Self {
673681
Self {
674-
distinct_values: GenericDistinctBuffer::new(T::DATA_TYPE),
682+
counts: HashMap::default(),
675683
percentile,
676684
}
677685
}
@@ -684,26 +692,50 @@ where
684692
f64: AsPrimitive<T::Native>,
685693
{
686694
fn state(&mut self) -> Result<Vec<ScalarValue>> {
687-
self.distinct_values.state()
695+
// Emit the distinct keys as a single List scalar, matching the state
696+
// shape declared in `state_fields` (a List of the input type). Counts
697+
// are window-local bookkeeping and are intentionally not serialized:
698+
// cross-partition merges only need the distinct key set.
699+
let arr = Arc::new(
700+
PrimitiveArray::<T>::from_iter_values(self.counts.keys().map(|v| v.0))
701+
.with_data_type(T::DATA_TYPE),
702+
);
703+
Ok(vec![
704+
SingleRowListArrayBuilder::new(arr).build_list_scalar(),
705+
])
688706
}
689707

690708
fn update_batch(&mut self, values: &[ArrayRef]) -> Result<()> {
691-
self.distinct_values.update_batch(values)
709+
// `values` may carry extra argument columns (e.g. the percentile
710+
// literal); only the first column holds the aggregated values.
711+
let arr = values[0].as_primitive::<T>();
712+
for value in arr.iter().flatten() {
713+
*self.counts.entry(Hashable(value)).or_default() += 1;
714+
}
715+
Ok(())
692716
}
693717

694718
fn merge_batch(&mut self, states: &[ArrayRef]) -> Result<()> {
695-
self.distinct_values.merge_batch(states)
719+
let list = states[0].as_list::<i32>();
720+
for values in list.iter().flatten() {
721+
let arr = values.as_primitive::<T>();
722+
for value in arr.iter().flatten() {
723+
*self.counts.entry(Hashable(value)).or_default() += 1;
724+
}
725+
}
726+
Ok(())
696727
}
697728

698729
fn evaluate(&mut self) -> Result<ScalarValue> {
699-
let mut values: Vec<T::Native> =
700-
self.distinct_values.values.iter().map(|v| v.0).collect();
730+
let mut values: Vec<T::Native> = self.counts.keys().map(|v| v.0).collect();
701731
let value = calculate_percentile::<T>(&mut values, self.percentile);
702732
ScalarValue::new_primitive::<T>(value, &T::DATA_TYPE)
703733
}
704734

705735
fn size(&self) -> usize {
706-
size_of_val(self) + self.distinct_values.size()
736+
size_of_val(self)
737+
+ self.counts.capacity()
738+
* (size_of::<Hashable<T::Native>>() + size_of::<usize>())
707739
}
708740

709741
fn retract_batch(&mut self, values: &[ArrayRef]) -> Result<()> {
@@ -713,7 +745,12 @@ where
713745

714746
let arr = values[0].as_primitive::<T>();
715747
for value in arr.iter().flatten() {
716-
self.distinct_values.values.remove(&Hashable(value));
748+
if let Some(count) = self.counts.get_mut(&Hashable(value)) {
749+
*count -= 1;
750+
if *count == 0 {
751+
self.counts.remove(&Hashable(value));
752+
}
753+
}
717754
}
718755
Ok(())
719756
}

‎datafusion/sqllogictest/test_files/aggregate.slt‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1104,6 +1104,33 @@ ORDER BY tags, timestamp;
11041104
statement ok
11051105
DROP TABLE median_window_test;
11061106

1107+
# Regression: percentile_cont(DISTINCT ...) used to forward the extra
1108+
# percentile-argument column into the distinct-values buffer (which asserts a
1109+
# single input array), panicking on every distinct query. Plain aggregate:
1110+
statement ok
1111+
CREATE TABLE distinct_pct(id INT, x DOUBLE) AS VALUES
1112+
(1, 5), (2, 5), (3, 9);
1113+
1114+
query R
1115+
SELECT percentile_cont(DISTINCT x, 0.5) FROM distinct_pct;
1116+
----
1117+
7
1118+
1119+
# Regression: distinct sliding-window percentile must count value multiplicity
1120+
# on retract. Row 3's frame is {5, 9}; the row-1 `5` leaves the frame but the
1121+
# row-2 `5` remains, so the distinct set is still {5, 9} (median 7), not {9}.
1122+
query IR
1123+
SELECT id, percentile_cont(DISTINCT x, 0.5)
1124+
OVER (ORDER BY id ROWS BETWEEN 1 PRECEDING AND CURRENT ROW)
1125+
FROM distinct_pct;
1126+
----
1127+
1 5
1128+
2 5
1129+
3 7
1130+
1131+
statement ok
1132+
DROP TABLE distinct_pct;
1133+
11071134
query RT
11081135
select approx_median(arrow_cast(col_f32, 'Float16')), arrow_typeof(approx_median(arrow_cast(col_f32, 'Float16'))) from median_table;
11091136
----

0 commit comments

Comments
 (0)