Skip to content

Commit d66bcee

Browse files
committed
test: cover maps and nested floats in the hash NaN tests
1 parent d329999 commit d66bcee

3 files changed

Lines changed: 64 additions & 3 deletions

File tree

‎native/spark-expr/src/agg_funcs/hll_plus_plus.rs‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -528,6 +528,42 @@ mod tests {
528528
assert_eq!(eval(&mut a), 2);
529529
}
530530

531+
/// The hash kernel canonicalizes floats inside nested types, so a struct or list holding a
532+
/// negative zero or a non-canonical NaN counts the same as one holding the plain value.
533+
#[test]
534+
fn nested_floats_fold_negative_zero_and_nan() {
535+
use arrow::array::{ListArray, StructArray};
536+
use arrow::buffer::OffsetBuffer;
537+
use arrow::datatypes::Field;
538+
539+
let floats: ArrayRef = Arc::new(Float64Array::from(vec![
540+
0.0,
541+
-0.0,
542+
f64::NAN,
543+
f64::from_bits(0xfff8_0000_0000_0000),
544+
f64::from_bits(0x7ff0_0000_0000_0001),
545+
]));
546+
let field = Arc::new(Field::new("item", DataType::Float64, true));
547+
let nested: Vec<ArrayRef> = vec![
548+
Arc::new(StructArray::new(
549+
vec![Arc::clone(&field)].into(),
550+
vec![Arc::clone(&floats)],
551+
None,
552+
)),
553+
Arc::new(ListArray::new(
554+
field,
555+
OffsetBuffer::from_lengths([1; 5]),
556+
floats,
557+
None,
558+
)),
559+
];
560+
for values in nested {
561+
let mut a = acc(9);
562+
a.update_batch(&[Arc::clone(&values)]).unwrap();
563+
assert_eq!(eval(&mut a), 2, "{}", values.data_type());
564+
}
565+
}
566+
531567
#[test]
532568
fn strings_small_cardinality() {
533569
let mut a = acc(9);

‎native/spark-expr/src/hash_funcs/murmur3.rs‎

Lines changed: 21 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -820,18 +820,29 @@ mod tests {
820820

821821
/// Spark hashes a float through `doubleToLongBits` or `floatToIntBits`, which canonicalize
822822
/// NaN, so a NaN with the sign bit set or with a payload hashes like the canonical NaN inside
823-
/// lists, structs and dictionaries too. Checks both hashes, which expand the same macros.
823+
/// lists, structs, maps and dictionaries too. Checks both hashes, which expand the same macros.
824824
#[test]
825825
fn test_non_canonical_nan_hashes_like_canonical_nan() {
826826
use crate::hash_funcs::create_xxhash64_hashes;
827827
use arrow::array::{
828-
DictionaryArray, FixedSizeListArray, Int32Array, LargeListArray, ListArray, StructArray,
828+
DictionaryArray, FixedSizeListArray, Int32Array, LargeListArray, ListArray, MapArray,
829+
StructArray,
829830
};
830831
use arrow::buffer::OffsetBuffer;
831-
use arrow::datatypes::Field;
832+
use arrow::datatypes::{DataType, Field, Fields};
832833

833834
let shapes = |values: ArrayRef| -> Vec<ArrayRef> {
834835
let field = Arc::new(Field::new("item", values.data_type().clone(), true));
836+
// One entry per row, with the float as both the key and the value.
837+
let entry_fields = Fields::from(vec![
838+
Field::new("key", values.data_type().clone(), false),
839+
Field::new("value", values.data_type().clone(), true),
840+
]);
841+
let entries = StructArray::new(
842+
entry_fields.clone(),
843+
vec![Arc::clone(&values), Arc::clone(&values)],
844+
None,
845+
);
835846
vec![
836847
Arc::clone(&values),
837848
Arc::new(ListArray::new(
@@ -861,6 +872,13 @@ mod tests {
861872
Int32Array::from(vec![0, 1, 2]),
862873
values,
863874
)),
875+
Arc::new(MapArray::new(
876+
Arc::new(Field::new("entries", DataType::Struct(entry_fields), false)),
877+
OffsetBuffer::from_lengths([1; 3]),
878+
entries,
879+
None,
880+
false,
881+
)),
864882
]
865883
};
866884
let canonical64: ArrayRef = Arc::new(Float64Array::from(vec![f64::NAN; 3]));

‎spark/src/test/resources/sql-tests/expressions/hash/hash.sql‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,8 @@
1515
-- specific language governing permissions and limitations
1616
-- under the License.
1717

18+
-- Config: spark.sql.legacy.allowHashOnMapType=true
19+
1820
-- hash functions
1921
statement
2022
CREATE TABLE test(col string, a int, b float) USING parquet
@@ -46,3 +48,8 @@ SELECT hash(d), hash(-d), xxhash64(d), xxhash64(-d), hash(f), hash(-f), xxhash64
4648

4749
query
4850
SELECT hash(array(-d)), xxhash64(array(-d)), hash(named_struct('a', -d, 'b', -f)), xxhash64(named_struct('a', -d, 'b', -f)) FROM test_nan
51+
52+
-- Spark rejects hashing a map unless the legacy config above is set. Map keys cannot be null,
53+
-- so the null row is filtered out.
54+
query
55+
SELECT hash(map(-d, -f)), xxhash64(map(-d, -f)) FROM test_nan WHERE d IS NOT NULL

0 commit comments

Comments
 (0)