Skip to content

Commit 036502f

Browse files
committed
normalize -0.0 in the buffered extreme
1 parent b2703c8 commit 036502f

2 files changed

Lines changed: 76 additions & 3 deletions

File tree

‎datafusion/physical-plan/src/joins/piecewise_merge_join/classic_join.rs‎

Lines changed: 33 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ use arrow::{
2626
};
2727
use arrow_schema::{Schema, SchemaRef, SortOptions};
2828
use datafusion_common::NullEquality;
29+
use datafusion_common::utils::normalize_float_zero;
2930
use datafusion_common::{Result, ScalarValue, internal_err};
3031
use datafusion_execution::{RecordBatchStream, SendableRecordBatchStream};
3132
use datafusion_expr::{ColumnarValue, JoinType, Operator};
@@ -591,9 +592,14 @@ fn build_matched_indices_and_mark_buffered(
591592
// NULLs sort first, so the last key is NULL only when every buffered key is.
592593
fn buffered_extreme(values: &ArrayRef) -> Result<Option<ColumnarValue>> {
593594
Ok(match values.len().checked_sub(1) {
594-
Some(last) if values.is_valid(last) => Some(ColumnarValue::Scalar(
595-
ScalarValue::try_from_array(values, last)?,
596-
)),
595+
// `apply_cmp` normalizes `-0.0` only in flat float scalars, not inside a
596+
// `ScalarValue::Dictionary`, so normalize the key before taking it.
597+
Some(last) if values.is_valid(last) => {
598+
let extreme = normalize_float_zero(&values.slice(last, 1));
599+
Some(ColumnarValue::Scalar(ScalarValue::try_from_array(
600+
&extreme, 0,
601+
)?))
602+
}
597603
_ => None,
598604
})
599605
}
@@ -1347,6 +1353,30 @@ mod tests {
13471353
.collect::<DictionaryArray<Int32Type>>(),
13481354
),
13491355
),
1356+
(
1357+
// `-0.0` as the buffered extreme inside a dictionary: the smallest
1358+
// key for `<`/`<=`, the largest for `>`/`>=`.
1359+
"dictionary_float64_neg_zero_min",
1360+
Arc::new(DictionaryArray::<Int32Type>::new(
1361+
Int32Array::from(vec![0, 1]),
1362+
Arc::new(Float64Array::from(vec![-0.0, 0.5])),
1363+
)),
1364+
Arc::new(DictionaryArray::<Int32Type>::new(
1365+
Int32Array::from(vec![0, 1, 2]),
1366+
Arc::new(Float64Array::from(vec![0.0, -0.0, 0.25])),
1367+
)),
1368+
),
1369+
(
1370+
"dictionary_float64_neg_zero_max",
1371+
Arc::new(DictionaryArray::<Int32Type>::new(
1372+
Int32Array::from(vec![0, 1]),
1373+
Arc::new(Float64Array::from(vec![-1.0, -0.0])),
1374+
)),
1375+
Arc::new(DictionaryArray::<Int32Type>::new(
1376+
Int32Array::from(vec![0, 1, 2]),
1377+
Arc::new(Float64Array::from(vec![0.0, -0.0, -0.5])),
1378+
)),
1379+
),
13501380
(
13511381
"decimal128",
13521382
Arc::new(

‎datafusion/sqllogictest/test_files/piecewise_merge_join_matrix.slt‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -199,6 +199,49 @@ SELECT l.lid, l.lv, r.rid, r.rv FROM cj_fl l FULL JOIN cj_fr r ON l.lv < r.rv;
199199
NULL NULL 1 0
200200
NULL NULL 2 -1
201201

202+
# The same signed-zero agreement with dictionary-encoded float keys, whose
203+
# values are normalized one level down. -0.0 is the smallest buffered key for
204+
# `<` and the largest for `>=`.
205+
statement ok
206+
CREATE TABLE cj_dfl(lid INT, lv DOUBLE);
207+
208+
statement ok
209+
INSERT INTO cj_dfl VALUES (1, -0.0), (2, 0.5);
210+
211+
statement ok
212+
CREATE TABLE cj_dfr(rid INT, rv DOUBLE);
213+
214+
statement ok
215+
INSERT INTO cj_dfr VALUES (1, 0.0), (2, 0.25);
216+
217+
query I?I? rowsort
218+
SELECT l.lid, l.lv, r.rid, r.rv
219+
FROM (SELECT lid, arrow_cast(lv, 'Dictionary(Int32, Float64)') AS lv FROM cj_dfl) l
220+
JOIN (SELECT rid, arrow_cast(rv, 'Dictionary(Int32, Float64)') AS rv FROM cj_dfr) r
221+
ON l.lv < r.rv;
222+
----
223+
1 0 2 0.25
224+
225+
statement ok
226+
CREATE TABLE cj_dfl2(lid INT, lv DOUBLE);
227+
228+
statement ok
229+
INSERT INTO cj_dfl2 VALUES (1, -1.0), (2, -0.0);
230+
231+
statement ok
232+
CREATE TABLE cj_dfr2(rid INT, rv DOUBLE);
233+
234+
statement ok
235+
INSERT INTO cj_dfr2 VALUES (1, 0.0);
236+
237+
query I?I? rowsort
238+
SELECT l.lid, l.lv, r.rid, r.rv
239+
FROM (SELECT lid, arrow_cast(lv, 'Dictionary(Int32, Float64)') AS lv FROM cj_dfl2) l
240+
RIGHT JOIN (SELECT rid, arrow_cast(rv, 'Dictionary(Int32, Float64)') AS rv FROM cj_dfr2) r
241+
ON l.lv >= r.rv;
242+
----
243+
2 0 1 0
244+
202245
# One unsorted streamed batch mixing matching, non-matching and NULL keys. The
203246
# non-matching and NULL rows are set aside before the sort, and the rest are
204247
# sorted and mapped back to their rows in the batch. Against cj_l.lv = {1,3,5,

0 commit comments

Comments
 (0)