Skip to content

Commit e87fe71

Browse files
getChankosiew
authored andcommitted
fix: exclude precision-losing integer-to-float conversions from CastExpr::check_bigger_cast (apache#23808) (apache#23809)
## Which issue does this PR close? - Closes apache#23808. ## Rationale for this change ref. apache#23807 `CastExpr::check_bigger_cast` is used to determine whether a cast is a widening (order-preserving) conversion. Currently, it classifies `Int32 -> Float32`, `UInt32 -> Float32`, `Int64 -> Float64`, and `UInt64 -> Float64` as widening casts. However, integer-to-float conversions for 32-bit and 64-bit integers lose precision when values exceed the mantissa bit limit (24 bits for `Float32`, 53 bits for `Float64`). For example: `16_777_216_i32 as f32 == 16_777_217_i32 as f32` (both yield 16777216.0f32). Because distinct integer inputs can collapse to the same float output, these casts are not strictly 1-to-1 (injective) and can break suffix sort key ordering in multi-column ordering analysis (e.g. `[CAST(a AS Float32), b]`). ## What changes are included in this PR? - Updated `CastExpr::check_bigger_cast` to exclude precision-losing integer-to-float conversions (`Int32/UInt32 -> Float32` and `Int64/UInt64 -> Float64`). - Added unit test `test_check_bigger_cast_precision_loss` to verify precision-losing casts return `false` while exact conversions (`Int16 -> Float32`, `Int32 -> Float64`, etc.) continue to return `true`. ## Are these changes tested? Yes, new unit test `test_check_bigger_cast_precision_loss` in `cast.rs`. ## Are there any user-facing changes? No breaking API changes. Internal optimizer behavior fix.
1 parent 20e79e2 commit e87fe71

3 files changed

Lines changed: 93 additions & 5 deletions

File tree

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
k,v
2+
1,16777217
3+
2,16777216

‎datafusion/physical-expr/src/expressions/cast.rs‎

Lines changed: 27 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -179,11 +179,8 @@ impl CastExpr {
179179
| (UInt8, UInt16 | UInt32 | UInt64)
180180
| (UInt16, UInt32 | UInt64)
181181
| (UInt32, UInt64)
182-
| (
183-
Int8 | Int16 | Int32 | UInt8 | UInt16 | UInt32,
184-
Float32 | Float64
185-
)
186-
| (Int64 | UInt64, Float64)
182+
| (Int8 | Int16 | UInt8 | UInt16, Float32)
183+
| (Int8 | Int16 | Int32 | UInt8 | UInt16 | UInt32, Float64)
187184
| (Utf8, LargeUtf8)
188185
)
189186
}
@@ -1208,6 +1205,31 @@ mod tests {
12081205

12091206
Ok(())
12101207
}
1208+
1209+
#[test]
1210+
fn test_check_bigger_cast_precision_loss() {
1211+
use DataType::*;
1212+
1213+
// Exact conversions without precision loss
1214+
assert!(CastExpr::check_bigger_cast(&Int16, &Int8));
1215+
assert!(CastExpr::check_bigger_cast(&Int64, &Int32));
1216+
assert!(CastExpr::check_bigger_cast(&Float32, &Int16));
1217+
assert!(CastExpr::check_bigger_cast(&Float32, &UInt16));
1218+
assert!(CastExpr::check_bigger_cast(&Float64, &Int32));
1219+
assert!(CastExpr::check_bigger_cast(&Float64, &UInt32));
1220+
assert!(CastExpr::check_bigger_cast(&LargeUtf8, &Utf8));
1221+
1222+
// Precision-losing int-to-float conversions should return false
1223+
assert!(!CastExpr::check_bigger_cast(&Float32, &Int32));
1224+
assert!(!CastExpr::check_bigger_cast(&Float32, &UInt32));
1225+
assert!(!CastExpr::check_bigger_cast(&Float64, &Int64));
1226+
assert!(!CastExpr::check_bigger_cast(&Float64, &UInt64));
1227+
1228+
// Signed <-> Unsigned conversions should return false (not order-preserving due to negative values)
1229+
assert!(!CastExpr::check_bigger_cast(&UInt16, &Int8));
1230+
assert!(!CastExpr::check_bigger_cast(&UInt32, &Int16));
1231+
assert!(!CastExpr::check_bigger_cast(&Int16, &UInt8));
1232+
}
12111233
}
12121234

12131235
/// Tests for the `try_to_proto` / `try_from_proto` hooks.

‎datafusion/sqllogictest/test_files/monotonic_projection_test.slt‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -252,3 +252,66 @@ ORDER BY a, b;
252252
----
253253
a 1
254254
a0 1
255+
256+
# Test that precision-losing int-to-float casts do not invalidate suffix sort keys.
257+
#
258+
# When CAST(Int32 AS Float32) collapses distinct integer values (e.g., 16777216 and
259+
# 16777217 both become 16777216.0), the suffix sort key (k) must still be sorted
260+
# correctly. Before the fix, the optimizer incorrectly reused the pre-existing sort
261+
# order and dropped the SortExec, producing wrong results.
262+
#
263+
# t1 is declared with a sort order, t2 is not — their results should be identical
264+
# since CAST(v AS FLOAT) is not injective for 32-bit integers.
265+
statement ok
266+
CREATE EXTERNAL TABLE t1_int_float (k int, v int)
267+
STORED AS CSV
268+
WITH ORDER (v DESC, k DESC)
269+
LOCATION '../core/tests/data/int_to_float_cast_precision.csv'
270+
OPTIONS ('format.has_header' 'true');
271+
272+
statement ok
273+
CREATE EXTERNAL TABLE t2_int_float (k int, v int)
274+
STORED AS CSV
275+
LOCATION '../core/tests/data/int_to_float_cast_precision.csv'
276+
OPTIONS ('format.has_header' 'true');
277+
278+
# Both queries must return the same result: k=2 before k=1.
279+
# (v_=16777216.0 for both rows; when tied on v_, DESC on k means k=2 comes first)
280+
query IR
281+
SELECT k, cast(v as float) v_ FROM t1_int_float ORDER BY v_ DESC, k DESC;
282+
----
283+
2 16777216
284+
1 16777216
285+
286+
query IR
287+
SELECT k, cast(v as float) v_ FROM t2_int_float ORDER BY v_ DESC, k DESC;
288+
----
289+
2 16777216
290+
1 16777216
291+
292+
# Widening cast (Int32 -> Int64) is strictly 1-to-1, so the optimizer CAN
293+
# legally reuse the pre-existing sort order and omit a SortExec.
294+
statement ok
295+
CREATE EXTERNAL TABLE t3_int_bigint (k int, v int)
296+
STORED AS CSV
297+
WITH ORDER (v DESC, k DESC)
298+
LOCATION '../core/tests/data/int_to_float_cast_precision.csv'
299+
OPTIONS ('format.has_header' 'true');
300+
301+
# CAST(Int32 AS BIGINT) is injective, so suffix key ordering is preserved.
302+
query II
303+
SELECT k, cast(v as bigint) v_ FROM t3_int_bigint ORDER BY v_ DESC, k DESC;
304+
----
305+
1 16777217
306+
2 16777216
307+
308+
# Cleanup
309+
statement ok
310+
DROP TABLE t1_int_float;
311+
312+
statement ok
313+
DROP TABLE t2_int_float;
314+
315+
statement ok
316+
DROP TABLE t3_int_bigint;
317+

0 commit comments

Comments
 (0)