Skip to content

Commit 5941552

Browse files
authored
perf: null-free fast path for COUNT(DISTINCT) primitive accumulator (apache#23956)
## Which issue does this PR close? - Closes apache#23955. ## Rationale for this change `PrimitiveDistinctCountAccumulator::update_batch` does a per-element validity check on every row even when the array has no nulls. Its own `merge_batch` already inserts wholesale from the values buffer, so the same fast path applies to `update_batch`. Inspired by the null-free fast paths added to `percentile_cont`/`median` in apache#23954. ## What changes are included in this PR? - When `null_count() == 0`, insert directly from the values buffer (`self.values.extend(arr.values().iter().copied())`) instead of the per-element `Option` check. - Unit test asserting the fast and general paths agree (a dense array and a null-containing array with the same non-null values yield the same distinct count). ## Benchmarks `count_distinct` benchmark, null-free 8192-row batches: ``` count_distinct i64 80% distinct -67.0% (54.9 -> 18.1 us) count_distinct i64 99% distinct -66.0% (54.9 -> 18.7 us) count_distinct u32 80% distinct -66.9% (53.9 -> 17.8 us) count_distinct u32 99% distinct -65.3% (53.3 -> 18.5 us) count_distinct i32 80% distinct -66.9% (54.1 -> 17.9 us) count_distinct i32 99% distinct -66.2% (53.2 -> 18.0 us) ``` The bitmap-backed cases (u8/i8/u16/i16) and the grouped-accumulator cases don't use this path and are unchanged, as expected. ## Are these changes tested? Yes — new unit test; existing count-distinct tests pass. ## Are there any user-facing changes? No.
1 parent 043d97f commit 5941552

1 file changed

Lines changed: 48 additions & 4 deletions

File tree

  • datafusion/functions-aggregate-common/src/aggregate/count_distinct

‎datafusion/functions-aggregate-common/src/aggregate/count_distinct/native.rs‎

Lines changed: 48 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ use std::hash::Hash;
2626
use std::mem::size_of_val;
2727
use std::sync::Arc;
2828

29+
use arrow::array::Array;
2930
use arrow::array::ArrayRef;
3031
use arrow::array::BooleanArray;
3132
use arrow::array::PrimitiveArray;
@@ -86,11 +87,15 @@ where
8687
}
8788

8889
let arr = as_primitive_array::<T>(&values[0])?;
89-
arr.iter().for_each(|value| {
90-
if let Some(value) = value {
90+
if arr.null_count() == 0 {
91+
// Fast path: no nulls, so skip the per-element validity check and
92+
// insert directly from the values buffer (mirrors `merge_batch`).
93+
self.values.extend(arr.values().iter().copied());
94+
} else {
95+
arr.iter().flatten().for_each(|value| {
9196
self.values.insert(value);
92-
}
93-
});
97+
});
98+
}
9499

95100
Ok(())
96101
}
@@ -617,3 +622,42 @@ impl Accumulator for BooleanDistinctCountAccumulator {
617622
size_of_val(self)
618623
}
619624
}
625+
626+
#[cfg(test)]
627+
mod tests {
628+
use super::*;
629+
use arrow::array::Int64Array;
630+
use arrow::datatypes::Int64Type;
631+
632+
#[test]
633+
fn update_batch_null_free_fast_path_agrees_with_general_path() {
634+
// The null-free fast path must produce the same distinct set as the
635+
// general (validity-checking) path.
636+
let dense: ArrayRef = Arc::new(Int64Array::from(vec![1, 2, 3, 2, 1]));
637+
let sparse: ArrayRef = Arc::new(Int64Array::from(vec![
638+
Some(1),
639+
None,
640+
Some(2),
641+
None,
642+
Some(3),
643+
Some(2),
644+
Some(1),
645+
]));
646+
647+
let mut dense_acc =
648+
PrimitiveDistinctCountAccumulator::<Int64Type>::new(&DataType::Int64);
649+
dense_acc
650+
.update_batch(std::slice::from_ref(&dense))
651+
.unwrap();
652+
653+
let mut sparse_acc =
654+
PrimitiveDistinctCountAccumulator::<Int64Type>::new(&DataType::Int64);
655+
sparse_acc
656+
.update_batch(std::slice::from_ref(&sparse))
657+
.unwrap();
658+
659+
// Both should count the 3 distinct non-null values {1, 2, 3}.
660+
assert_eq!(dense_acc.evaluate().unwrap(), ScalarValue::Int64(Some(3)));
661+
assert_eq!(sparse_acc.evaluate().unwrap(), ScalarValue::Int64(Some(3)));
662+
}
663+
}

0 commit comments

Comments
 (0)