Skip to content

Commit 68a1b3e

Browse files
cool9850311claude
andcommitted
fix: return NULL from bit_xor when a sliding frame has no non-NULL values
`BitXorAccumulator` kept a single `Option<T::Native>` whose `None` doubled as "has not seen a non-NULL value yet". `retract_batch` could undo the value, since XOR is its own inverse, but never moved the state back to `None`, so once every non-NULL value had left a sliding window frame the accumulator reported `Some(0)` and the query returned 0 where SQL semantics — and `bit_xor`'s own documented "all non-null input values" contract — require NULL. The same aggregate over a whole partition returned NULL correctly, because that path never retracts. Track the number of non-NULL values currently accumulated and return NULL when it is zero, the same way `avg` does. `retract_batch` no longer delegates to `update_batch`, since the count has to move the other way. A genuine zero, such as `bit_xor` of [5, 5], is still returned as 0. `state_fields` is unchanged: the count never has to be serialized, because `state()` already yields NULL when there is no non-NULL input and `merge_batch` adds nothing for a NULL state. `bit_and`, `bit_or` and `bit_xor(DISTINCT ..)` do not implement `retract_batch` and are unaffected. Closes apache#26470 Co-Authored-By: Claude Opus 5 <[email protected]>
1 parent 862843f commit 68a1b3e

2 files changed

Lines changed: 119 additions & 5 deletions

File tree

‎datafusion/functions-aggregate/src/bit_and_or_xor.rs‎

Lines changed: 70 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -434,6 +434,8 @@ where
434434

435435
struct BitXorAccumulator<T: ArrowNumericType> {
436436
value: Option<T::Native>,
437+
/// Number of non-NULL values currently accumulated (inserted minus retracted)
438+
non_null_count: usize,
437439
}
438440

439441
impl<T: ArrowNumericType> std::fmt::Debug for BitXorAccumulator<T> {
@@ -444,7 +446,10 @@ impl<T: ArrowNumericType> std::fmt::Debug for BitXorAccumulator<T> {
444446

445447
impl<T: ArrowNumericType> Default for BitXorAccumulator<T> {
446448
fn default() -> Self {
447-
Self { value: None }
449+
Self {
450+
value: None,
451+
non_null_count: 0,
452+
}
448453
}
449454
}
450455

@@ -457,20 +462,33 @@ where
457462
let v = self.value.get_or_insert_with(|| T::Native::usize_as(0));
458463
*v = *v ^ x;
459464
}
465+
self.non_null_count += values[0].len() - values[0].null_count();
460466
Ok(())
461467
}
462468

463469
fn retract_batch(&mut self, values: &[ArrayRef]) -> Result<()> {
464-
// XOR is it's own inverse
465-
self.update_batch(values)
470+
// XOR is its own inverse, so the value is updated as in `update_batch`;
471+
// only the count of non-NULL values moves the other way
472+
if let Some(x) = arrow::compute::bit_xor(values[0].as_primitive::<T>()) {
473+
let v = self.value.get_or_insert_with(|| T::Native::usize_as(0));
474+
*v = *v ^ x;
475+
}
476+
self.non_null_count -= values[0].len() - values[0].null_count();
477+
Ok(())
466478
}
467479

468480
fn supports_retract_batch(&self) -> bool {
469481
true
470482
}
471483

472484
fn evaluate(&mut self) -> Result<ScalarValue> {
473-
ScalarValue::new_primitive::<T>(self.value, &T::DATA_TYPE)
485+
// All non-NULL values may have been retracted, in which case the result is NULL
486+
let value = if self.non_null_count == 0 {
487+
None
488+
} else {
489+
self.value
490+
};
491+
ScalarValue::new_primitive::<T>(value, &T::DATA_TYPE)
474492
}
475493

476494
fn size(&self) -> usize {
@@ -580,7 +598,7 @@ mod tests {
580598

581599
#[test]
582600
fn test_bit_xor_accumulator() {
583-
let mut accumulator = BitXorAccumulator::<UInt64Type> { value: None };
601+
let mut accumulator = BitXorAccumulator::<UInt64Type>::default();
584602
let batches: Vec<_> = vec![vec![1, 2], vec![1]]
585603
.into_iter()
586604
.map(|b| Arc::new(b.into_iter().collect::<UInt64Array>()) as ArrayRef)
@@ -603,4 +621,51 @@ mod tests {
603621
ScalarValue::UInt64(Some(2))
604622
);
605623
}
624+
625+
#[test]
626+
fn test_bit_xor_accumulator_retract_to_all_null() {
627+
let mut accumulator = BitXorAccumulator::<UInt64Type>::default();
628+
let five = [Arc::new(UInt64Array::from(vec![Some(5)])) as ArrayRef];
629+
let nulls = [Arc::new(UInt64Array::from(vec![None])) as ArrayRef];
630+
let seven = [Arc::new(UInt64Array::from(vec![Some(7)])) as ArrayRef];
631+
632+
accumulator.update_batch(&five).unwrap();
633+
assert_eq!(
634+
accumulator.evaluate().unwrap(),
635+
ScalarValue::UInt64(Some(5))
636+
);
637+
638+
accumulator.update_batch(&nulls).unwrap();
639+
assert_eq!(
640+
accumulator.evaluate().unwrap(),
641+
ScalarValue::UInt64(Some(5))
642+
);
643+
644+
// All non-NULL values left the frame, so the result is NULL
645+
accumulator.retract_batch(&five).unwrap();
646+
assert_eq!(accumulator.evaluate().unwrap(), ScalarValue::UInt64(None));
647+
648+
// Retracting NULLs must not move the count
649+
accumulator.retract_batch(&nulls).unwrap();
650+
assert_eq!(accumulator.evaluate().unwrap(), ScalarValue::UInt64(None));
651+
652+
accumulator.update_batch(&seven).unwrap();
653+
assert_eq!(
654+
accumulator.evaluate().unwrap(),
655+
ScalarValue::UInt64(Some(7))
656+
);
657+
}
658+
659+
#[test]
660+
fn test_bit_xor_accumulator_legitimate_zero() {
661+
let mut accumulator = BitXorAccumulator::<UInt64Type>::default();
662+
let values = [Arc::new(UInt64Array::from(vec![Some(5), Some(5)])) as ArrayRef];
663+
664+
// 5 ^ 5 is a real zero, not NULL
665+
accumulator.update_batch(&values).unwrap();
666+
assert_eq!(
667+
accumulator.evaluate().unwrap(),
668+
ScalarValue::UInt64(Some(0))
669+
);
670+
}
606671
}

‎datafusion/sqllogictest/test_files/window.slt‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6736,6 +6736,55 @@ ORDER BY i;
67366736
3 1
67376737
4 NULL
67386738

6739+
# bit_xor over a sliding window must yield NULL when the frame has no non-NULL
6740+
# values, matching the result over the whole partition (apache/datafusion#26470).
6741+
# Before the fix the sliding result was 0 on the NULL row.
6742+
query III
6743+
SELECT x,
6744+
bit_xor(x) OVER (ORDER BY i ROWS BETWEEN CURRENT ROW AND CURRENT ROW) AS sliding,
6745+
bit_xor(x) OVER (PARTITION BY i) AS whole
6746+
FROM (VALUES (1, 5), (2, CAST(NULL AS INT)), (3, 6)) t(i, x)
6747+
ORDER BY i;
6748+
----
6749+
5 5 5
6750+
NULL NULL NULL
6751+
6 6 6
6752+
6753+
# Narrow sliding frame that drains to empty each row.
6754+
query II
6755+
SELECT i, bit_xor(v) OVER (ORDER BY i ROWS BETWEEN 1 PRECEDING AND 1 PRECEDING)
6756+
FROM (VALUES(1,CAST(NULL AS INT)),(2,1),(3,CAST(NULL AS INT)),(4,CAST(NULL AS INT))) t(i,v)
6757+
ORDER BY i;
6758+
----
6759+
1 NULL
6760+
2 NULL
6761+
3 1
6762+
4 NULL
6763+
6764+
# All-NULL input — every frame is empty.
6765+
query II
6766+
SELECT i, bit_xor(v) OVER (ORDER BY i ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING)
6767+
FROM (VALUES(1,CAST(NULL AS INT)),(2,CAST(NULL AS INT))) t(i,v)
6768+
ORDER BY i;
6769+
----
6770+
1 NULL
6771+
2 NULL
6772+
6773+
# A legitimate 0 must not be turned into NULL, and NULL must follow it.
6774+
# Row 2's frame is [5, 5], whose XOR is genuinely 0. Row 4's frame is
6775+
# [NULL, NULL] and must be NULL even though the accumulated value was already 0.
6776+
# Row 5 shows the accumulator recovering once a non-NULL value re-enters.
6777+
query II
6778+
SELECT i, bit_xor(v) OVER (ORDER BY i ROWS BETWEEN 1 PRECEDING AND CURRENT ROW)
6779+
FROM (VALUES(1,5),(2,5),(3,CAST(NULL AS INT)),(4,CAST(NULL AS INT)),(5,3)) t(i,v)
6780+
ORDER BY i;
6781+
----
6782+
1 5
6783+
2 0
6784+
3 5
6785+
4 NULL
6786+
5 3
6787+
67396788
# Covariance/correlation sliding-window regression test. Verifies correct
67406789
# results across row removals and a NULL-gap empty-frame transition.
67416790
query IRRR

0 commit comments

Comments
 (0)