Skip to content

Commit 51cf54b

Browse files
Handle run-end encoded zeros and add parquet coverage
Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent da40aea commit 51cf54b

3 files changed

Lines changed: 60 additions & 3 deletions

File tree

‎datafusion/common/src/scalar/mod.rs‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2783,7 +2783,7 @@ impl ScalarValue {
27832783
}
27842784

27852785
/// Whether this is a float `0.0` or `-0.0`, including one inside a
2786-
/// dictionary. The two zeros are equal but sort apart (see
2786+
/// dictionary or run-end encoded value. The two zeros are equal but sort apart (see
27872787
/// [`compare_floats`]), so a sort boundary at a zero has to account for
27882788
/// both.
27892789
///
@@ -2793,7 +2793,8 @@ impl ScalarValue {
27932793
ScalarValue::Float16(Some(v)) => f32::from(*v) == 0.0,
27942794
ScalarValue::Float32(Some(v)) => *v == 0.0,
27952795
ScalarValue::Float64(Some(v)) => *v == 0.0,
2796-
ScalarValue::Dictionary(_, inner) => inner.is_float_zero(),
2796+
ScalarValue::Dictionary(_, inner)
2797+
| ScalarValue::RunEndEncoded(_, _, inner) => inner.is_float_zero(),
27972798
_ => false,
27982799
}
27992800
}
@@ -12104,6 +12105,11 @@ mod tests {
1210412105
Box::new(DataType::Int32),
1210512106
Box::new(ScalarValue::Float64(Some(-0.0))),
1210612107
),
12108+
ScalarValue::RunEndEncoded(
12109+
Field::new("run_ends", DataType::Int32, false).into(),
12110+
Field::new("values", DataType::Float64, true).into(),
12111+
Box::new(ScalarValue::Float64(Some(-0.0))),
12112+
),
1210712113
] {
1210812114
assert!(v.is_float_zero(), "{v:?}");
1210912115
}

‎datafusion/physical-plan/src/topk/mod.rs‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -634,7 +634,8 @@ impl TopK {
634634
// equal (#22835), so `col < 0.0 OR (col = 0.0 AND ...)` would drop
635635
// `-0.0` rows that belong in the TopK. For a zero threshold, use a
636636
// non-strict bound that keeps both zeros and do not filter on any
637-
// later key.
637+
// later key. Temporary: remove once -0.0 and 0.0 sort as equal,
638+
// see https://github.com/apache/datafusion/issues/26138
638639
let float_zero = value.is_float_zero();
639640

640641
// Create the appropriate operator based on sort order

‎datafusion/sqllogictest/test_files/topk.slt‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -854,11 +854,61 @@ select x, y from float_zero_desc order by x desc, y limit 2;
854854
0 5
855855
0 7
856856

857+
statement ok
858+
create table float_zero_ree as select arrow_cast(column1, 'RunEndEncoded("run_ends": non-null Int32, "values": Float64)') x, column2 y from (values (0.0, 1), (-0.0, 7));
859+
860+
statement ok
861+
insert into float_zero_ree select arrow_cast(-0.0, 'RunEndEncoded("run_ends": non-null Int32, "values": Float64)'), 5;
862+
863+
query I
864+
select y from float_zero_ree order by x, y limit 2;
865+
----
866+
5
867+
7
868+
869+
# The filter also prunes files and row groups in the scan.
870+
query I
871+
copy (select * from (values (0.0, 1), (-0.0, 7)) t(x, y)) to 'test_files/scratch/topk/float_zero/a.parquet';
872+
----
873+
2
874+
875+
query I
876+
copy (select * from (values (-0.0, 5)) t(x, y)) to 'test_files/scratch/topk/float_zero/b.parquet';
877+
----
878+
1
879+
880+
statement ok
881+
create external table float_zero_pq stored as parquet location 'test_files/scratch/topk/float_zero/';
882+
883+
query I
884+
select y from float_zero_pq order by x, y limit 2;
885+
----
886+
5
887+
7
888+
889+
statement ok
890+
set datafusion.execution.parquet.pushdown_filters = true;
891+
892+
query I
893+
select y from float_zero_pq order by x, y limit 2;
894+
----
895+
5
896+
7
897+
898+
statement ok
899+
set datafusion.execution.parquet.pushdown_filters = false;
900+
857901
statement ok
858902
drop table float_zero;
859903

860904
statement ok
861905
drop table float_zero_desc;
862906

907+
statement ok
908+
drop table float_zero_ree;
909+
910+
statement ok
911+
drop table float_zero_pq;
912+
863913
statement ok
864914
set datafusion.execution.target_partitions = 4;

0 commit comments

Comments
 (0)