Skip to content

Commit 4e7b69e

Browse files
fix: keep -0.0 rows in TopK dynamic filter when the threshold is a float zero
The TopK heap orders -0.0 before 0.0, but comparisons treat them as equal since apache#22835, so a filter like `x < 0.0 OR (x = 0.0 AND y < 1)` dropped -0.0 rows that belong in the result. For a ±0.0 threshold use a non-strict bound and stop filtering on later sort keys. Closes apache#26618 Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent cd571e5 commit 4e7b69e

2 files changed

Lines changed: 74 additions & 4 deletions

File tree

‎datafusion/physical-plan/src/topk/mod.rs‎

Lines changed: 27 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -362,6 +362,17 @@ pub(crate) fn build_sort_fields(
362362
.collect::<Result<_>>()
363363
}
364364

365+
/// Whether `value` is a floating-point `0.0` or `-0.0`.
366+
fn is_float_zero(value: &ScalarValue) -> bool {
367+
match value {
368+
ScalarValue::Float16(Some(v)) => v.to_f32() == 0.0,
369+
ScalarValue::Float32(Some(v)) => *v == 0.0,
370+
ScalarValue::Float64(Some(v)) => *v == 0.0,
371+
ScalarValue::Dictionary(_, inner) => is_float_zero(inner),
372+
_ => false,
373+
}
374+
}
375+
365376
impl TopK {
366377
/// Create a new [`TopK`] that stores the top `k` values, as
367378
/// defined by the sort expressions in `expr`.
@@ -643,13 +654,21 @@ impl TopK {
643654
break;
644655
}
645656

657+
// The heap orders `-0.0` before `0.0`, but comparisons treat them as
658+
// equal (#22835), so `col < 0.0 OR (col = 0.0 AND ...)` would drop
659+
// `-0.0` rows that belong in the TopK. For a zero threshold, use a
660+
// non-strict bound that keeps both zeros and do not filter on any
661+
// later key.
662+
let float_zero = is_float_zero(value);
663+
646664
// Create the appropriate operator based on sort order
647-
let op = if sort_expr.options.descending {
665+
let op = match (sort_expr.options.descending, float_zero) {
648666
// For descending sort, we want col > threshold (exclude smaller values)
649-
Operator::Gt
650-
} else {
667+
(true, false) => Operator::Gt,
668+
(true, true) => Operator::GtEq,
651669
// For ascending sort, we want col < threshold (exclude larger values)
652-
Operator::Lt
670+
(false, false) => Operator::Lt,
671+
(false, true) => Operator::LtEq,
653672
};
654673

655674
let value_null = value.is_null();
@@ -708,6 +727,10 @@ impl TopK {
708727
Some(Arc::new(BinaryExpr::new(p, Operator::And, eq_expr)));
709728
}
710729
}
730+
731+
if float_zero {
732+
break;
733+
}
711734
}
712735

713736
let dynamic_predicate = filters

‎datafusion/sqllogictest/test_files/topk.slt‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -824,3 +824,50 @@ drop table nested_progress;
824824

825825
statement ok
826826
set datafusion.execution.target_partitions = 4;
827+
828+
# The TopK dynamic filter must keep `-0.0` rows when the threshold is `0.0`
829+
# (and vice versa): the heap orders `-0.0` before `0.0`, but comparisons treat
830+
# them as equal. https://github.com/apache/datafusion/issues/26618
831+
statement ok
832+
set datafusion.execution.target_partitions = 1;
833+
834+
statement ok
835+
create table float_zero (x double, y int) as values (0.0, 1), (-0.0, 7);
836+
837+
statement ok
838+
insert into float_zero values (-0.0, 5);
839+
840+
# `-0.0` is displayed as `0`; `y` tells the rows apart.
841+
query RI
842+
select x, y from float_zero order by x, y;
843+
----
844+
0 5
845+
0 7
846+
0 1
847+
848+
query RI
849+
select x, y from float_zero order by x, y limit 2;
850+
----
851+
0 5
852+
0 7
853+
854+
statement ok
855+
create table float_zero_desc (x double, y int) as values (-0.0, 1), (0.0, 7);
856+
857+
statement ok
858+
insert into float_zero_desc values (0.0, 5);
859+
860+
query RI
861+
select x, y from float_zero_desc order by x desc, y limit 2;
862+
----
863+
0 5
864+
0 7
865+
866+
statement ok
867+
drop table float_zero;
868+
869+
statement ok
870+
drop table float_zero_desc;
871+
872+
statement ok
873+
set datafusion.execution.target_partitions = 4;

0 commit comments

Comments
 (0)