Skip to content

Commit 951b821

Browse files
authored
fix: Preserve integer values in round() for large Int64 and UInt64 inputs (apache#22697)
## Which issue does this PR close? - Closes apache#22696 . ## Rationale for this change `round()` should not change integer values when the scale is non-negative, since no fractional digits need to be rounded. Currently, core `round()` coerces large `Int64` values through `Float64`, causing precision loss: ```sql SELECT round(arrow_cast(9007199254740993, 'Int64')); ```` Before/Current Buggy Output: ```text 9007199254740992.0 ``` Expected: ```text 9007199254740993 ``` The Spark-compatible `round()` also fails for `UInt64` values above `i64::MAX` even when the scale is non-negative: ```sql SELECT round(arrow_cast(18446744073709551615, 'UInt64')); ``` Before/Current Buggy Output: ```text round: UInt64 value 18446744073709551615 exceeds i64::MAX and cannot be rounded ``` ## What changes are included in this PR? * Preserved integer inputs in core `round()` for non-negative scales instead of routing them through `Float64`. * Preserved `UInt64` values in Spark-compatible `round()` when the scale is non-negative, avoiding unnecessary `UInt64 -> i64` conversion. * Added SQLLogicTest coverage for: * `Int64` values above `2^53` in core `round()`. * `UInt64::MAX` in Spark-compatible `round()`. * Both one-argument and two-argument forms. ## Are these changes tested? Yes. ```bash cargo fmt --all git diff --check cargo test -p datafusion-sqllogictest --test sqllogictests -- spark/math/round.slt cargo test -p datafusion-functions round cargo test -p datafusion-spark round ``` I also verified the core regression queries manually: ```sql SELECT arrow_typeof(round(arrow_cast(9007199254740993, 'Int64'))), round(arrow_cast(9007199254740993, 'Int64')); ``` Result: ```text Int64 9007199254740993 ``` ```sql SELECT arrow_typeof(round(arrow_cast(9007199254740993, 'Int64'), 2)), round(arrow_cast(9007199254740993, 'Int64'), 2); ``` Result: ```text Int64 9007199254740993 ``` ## Are there any user-facing changes? No.
1 parent 93e26f4 commit 951b821

5 files changed

Lines changed: 331 additions & 35 deletions

File tree

‎datafusion/functions/src/math/round.rs‎

Lines changed: 232 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -17,13 +17,15 @@
1717

1818
use crate::utils::{calculate_binary_decimal_math_cast, calculate_binary_math};
1919

20-
use arrow::array::ArrayRef;
20+
use arrow::array::{Array, ArrayRef, AsArray};
2121
use arrow::datatypes::DataType::{
22-
Decimal32, Decimal64, Decimal128, Decimal256, Float32, Float64,
22+
Decimal32, Decimal64, Decimal128, Decimal256, Float32, Float64, Int8, Int16, Int32,
23+
Int64, UInt8, UInt16, UInt32, UInt64,
2324
};
2425
use arrow::datatypes::{
2526
ArrowNativeTypeOp, DataType, Decimal32Type, Decimal64Type, Decimal128Type,
26-
Decimal256Type, DecimalType, Float32Type, Float64Type, Int32Type,
27+
Decimal256Type, DecimalType, Float32Type, Float64Type, Int8Type, Int16Type,
28+
Int32Type, Int64Type, UInt8Type, UInt16Type, UInt32Type, UInt64Type,
2729
};
2830
use arrow::datatypes::{Field, FieldRef};
2931
use arrow::error::ArrowError;
@@ -37,6 +39,7 @@ use datafusion_expr::{
3739
ScalarUDFImpl, Signature, TypeSignature, TypeSignatureClass, Volatility,
3840
};
3941
use datafusion_macros::user_doc;
42+
use num_traits::{PrimInt, Signed, cast, checked_pow};
4043
use std::sync::Arc;
4144

4245
fn output_scale_for_decimal(precision: u8, input_scale: i8, decimal_places: i32) -> i8 {
@@ -185,6 +188,7 @@ impl RoundFunc {
185188
vec![TypeSignatureClass::Integer],
186189
NativeType::Int32,
187190
);
191+
let integer = Coercion::new_exact(TypeSignatureClass::Integer);
188192
let float32 = Coercion::new_exact(TypeSignatureClass::Native(logical_float32()));
189193
let float64 = Coercion::new_implicit(
190194
TypeSignatureClass::Native(logical_float64()),
@@ -199,6 +203,11 @@ impl RoundFunc {
199203
decimal_places.clone(),
200204
]),
201205
TypeSignature::Coercible(vec![decimal]),
206+
TypeSignature::Coercible(vec![
207+
integer.clone(),
208+
decimal_places.clone(),
209+
]),
210+
TypeSignature::Coercible(vec![integer]),
202211
TypeSignature::Coercible(vec![
203212
float32.clone(),
204213
decimal_places.clone(),
@@ -245,6 +254,7 @@ impl ScalarUDFImpl for RoundFunc {
245254
// extra precision to accommodate potential carry-over.
246255
let return_type =
247256
match input_type {
257+
input_type if input_type.is_integer() => input_type.clone(),
248258
Float32 => Float32,
249259
Decimal32(precision, scale) => calculate_new_precision_scale::<
250260
Decimal32Type,
@@ -308,6 +318,9 @@ impl ScalarUDFImpl for RoundFunc {
308318
};
309319

310320
match (value_scalar, args.return_type()) {
321+
(value_scalar, return_type) if return_type.is_integer() => {
322+
round_integer_scalar(value_scalar, return_type, dp)
323+
}
311324
(ScalarValue::Float32(Some(v)), _) => {
312325
let rounded = round_float(*v, dp)?;
313326
Ok(ColumnarValue::Scalar(ScalarValue::from(rounded)))
@@ -468,6 +481,20 @@ fn round_columnar(
468481
let decimal_places_is_array = matches!(decimal_places, ColumnarValue::Array(_));
469482

470483
let arr: ArrayRef = match (value_array.data_type(), return_type) {
484+
(input_type, return_type)
485+
if input_type == return_type && return_type.is_integer() =>
486+
{
487+
match decimal_places {
488+
ColumnarValue::Scalar(ScalarValue::Int32(Some(dp))) if *dp >= 0 => {
489+
value_array
490+
}
491+
_ => round_integer_array(
492+
value_array.as_ref(),
493+
decimal_places,
494+
return_type,
495+
)?,
496+
}
497+
}
471498
(Float64, _) => {
472499
let result = calculate_binary_math::<Float64Type, Int32Type, Float64Type, _>(
473500
value_array.as_ref(),
@@ -518,7 +545,7 @@ fn round_columnar(
518545
},
519546
*precision,
520547
*new_scale,
521-
&DataType::Int32,
548+
&Int32,
522549
)?;
523550
result as _
524551
}
@@ -552,7 +579,7 @@ fn round_columnar(
552579
},
553580
*precision,
554581
*new_scale,
555-
&DataType::Int32,
582+
&Int32,
556583
)?;
557584
result as _
558585
}
@@ -586,7 +613,7 @@ fn round_columnar(
586613
},
587614
*precision,
588615
*new_scale,
589-
&DataType::Int32,
616+
&Int32,
590617
)?;
591618
result as _
592619
}
@@ -620,7 +647,7 @@ fn round_columnar(
620647
},
621648
*precision,
622649
*new_scale,
623-
&DataType::Int32,
650+
&Int32,
624651
)?;
625652
result as _
626653
}
@@ -634,6 +661,204 @@ fn round_columnar(
634661
}
635662
}
636663

664+
fn round_signed_integer<T>(
665+
value: T,
666+
decimal_places: i32,
667+
type_name: &str,
668+
) -> Result<T, ArrowError>
669+
where
670+
T: PrimInt + Signed,
671+
{
672+
if decimal_places >= 0 || value == T::zero() {
673+
return Ok(value);
674+
}
675+
676+
let ten = cast::<_, T>(10).expect("10 fits in all integer types");
677+
let Some(factor) = checked_pow(ten, decimal_places.unsigned_abs() as usize) else {
678+
return Ok(T::zero());
679+
};
680+
681+
let two = cast::<_, T>(2).expect("2 fits in all integer types");
682+
let one = T::one();
683+
let threshold = factor / two;
684+
let mut quotient = value / factor;
685+
let remainder = value % factor;
686+
687+
if remainder >= threshold {
688+
quotient = quotient.checked_add(&one).ok_or_else(|| {
689+
ArrowError::ComputeError(format!("Overflow while rounding {type_name}"))
690+
})?;
691+
} else if remainder <= -threshold {
692+
quotient = quotient.checked_sub(&one).ok_or_else(|| {
693+
ArrowError::ComputeError(format!("Overflow while rounding {type_name}"))
694+
})?;
695+
}
696+
697+
quotient.checked_mul(&factor).ok_or_else(|| {
698+
ArrowError::ComputeError(format!("Overflow while rounding {type_name}"))
699+
})
700+
}
701+
702+
fn round_unsigned_integer<T>(
703+
value: T,
704+
decimal_places: i32,
705+
type_name: &str,
706+
) -> Result<T, ArrowError>
707+
where
708+
T: PrimInt,
709+
{
710+
if decimal_places >= 0 || value == T::zero() {
711+
return Ok(value);
712+
}
713+
714+
let ten = cast::<_, T>(10).expect("10 fits in all integer types");
715+
let Some(factor) = checked_pow(ten, decimal_places.unsigned_abs() as usize) else {
716+
return Ok(T::zero());
717+
};
718+
719+
let two = cast::<_, T>(2).expect("2 fits in all integer types");
720+
let one = T::one();
721+
let threshold = factor / two;
722+
let mut quotient = value / factor;
723+
let remainder = value % factor;
724+
725+
if remainder >= threshold {
726+
quotient = quotient.checked_add(&one).ok_or_else(|| {
727+
ArrowError::ComputeError(format!("Overflow while rounding {type_name}"))
728+
})?;
729+
}
730+
731+
quotient.checked_mul(&factor).ok_or_else(|| {
732+
ArrowError::ComputeError(format!("Overflow while rounding {type_name}"))
733+
})
734+
}
735+
736+
fn round_integer_scalar(
737+
value: &ScalarValue,
738+
return_type: &DataType,
739+
decimal_places: i32,
740+
) -> Result<ColumnarValue> {
741+
match (value, return_type) {
742+
(ScalarValue::Int8(Some(v)), Int8) => Ok(ColumnarValue::Scalar(
743+
ScalarValue::Int8(Some(round_signed_integer(*v, decimal_places, "Int8")?)),
744+
)),
745+
(ScalarValue::Int16(Some(v)), Int16) => Ok(ColumnarValue::Scalar(
746+
ScalarValue::Int16(Some(round_signed_integer(*v, decimal_places, "Int16")?)),
747+
)),
748+
(ScalarValue::Int32(Some(v)), Int32) => Ok(ColumnarValue::Scalar(
749+
ScalarValue::Int32(Some(round_signed_integer(*v, decimal_places, "Int32")?)),
750+
)),
751+
(ScalarValue::Int64(Some(v)), Int64) => Ok(ColumnarValue::Scalar(
752+
ScalarValue::Int64(Some(round_signed_integer(*v, decimal_places, "Int64")?)),
753+
)),
754+
(ScalarValue::UInt8(Some(v)), UInt8) => {
755+
Ok(ColumnarValue::Scalar(ScalarValue::UInt8(Some(
756+
round_unsigned_integer(*v, decimal_places, "UInt8")?,
757+
))))
758+
}
759+
(ScalarValue::UInt16(Some(v)), UInt16) => {
760+
Ok(ColumnarValue::Scalar(ScalarValue::UInt16(Some(
761+
round_unsigned_integer(*v, decimal_places, "UInt16")?,
762+
))))
763+
}
764+
(ScalarValue::UInt32(Some(v)), UInt32) => {
765+
Ok(ColumnarValue::Scalar(ScalarValue::UInt32(Some(
766+
round_unsigned_integer(*v, decimal_places, "UInt32")?,
767+
))))
768+
}
769+
(ScalarValue::UInt64(Some(v)), UInt64) => {
770+
Ok(ColumnarValue::Scalar(ScalarValue::UInt64(Some(
771+
round_unsigned_integer(*v, decimal_places, "UInt64")?,
772+
))))
773+
}
774+
_ => internal_err!(
775+
"Unexpected integer round input/output types: {} -> {}",
776+
value.data_type(),
777+
return_type
778+
),
779+
}
780+
}
781+
782+
macro_rules! round_integer_array {
783+
($ARRAY:expr, $DP:expr, $ARRAY_TYPE:ty, $ROUND_FN:ident, $TYPE_NAME:expr) => {{
784+
let array = $ARRAY.as_primitive::<$ARRAY_TYPE>();
785+
786+
let result = calculate_binary_math::<$ARRAY_TYPE, Int32Type, $ARRAY_TYPE, _>(
787+
array,
788+
$DP,
789+
|v, dp| $ROUND_FN(v, dp, $TYPE_NAME),
790+
)?;
791+
792+
Ok(result as ArrayRef)
793+
}};
794+
}
795+
796+
fn round_integer_array(
797+
value_array: &dyn Array,
798+
decimal_places: &ColumnarValue,
799+
return_type: &DataType,
800+
) -> Result<ArrayRef> {
801+
match return_type {
802+
Int8 => round_integer_array!(
803+
value_array,
804+
decimal_places,
805+
Int8Type,
806+
round_signed_integer,
807+
"Int8"
808+
),
809+
Int16 => round_integer_array!(
810+
value_array,
811+
decimal_places,
812+
Int16Type,
813+
round_signed_integer,
814+
"Int16"
815+
),
816+
Int32 => round_integer_array!(
817+
value_array,
818+
decimal_places,
819+
Int32Type,
820+
round_signed_integer,
821+
"Int32"
822+
),
823+
Int64 => round_integer_array!(
824+
value_array,
825+
decimal_places,
826+
Int64Type,
827+
round_signed_integer,
828+
"Int64"
829+
),
830+
UInt8 => round_integer_array!(
831+
value_array,
832+
decimal_places,
833+
UInt8Type,
834+
round_unsigned_integer,
835+
"UInt8"
836+
),
837+
UInt16 => round_integer_array!(
838+
value_array,
839+
decimal_places,
840+
UInt16Type,
841+
round_unsigned_integer,
842+
"UInt16"
843+
),
844+
UInt32 => round_integer_array!(
845+
value_array,
846+
decimal_places,
847+
UInt32Type,
848+
round_unsigned_integer,
849+
"UInt32"
850+
),
851+
UInt64 => round_integer_array!(
852+
value_array,
853+
decimal_places,
854+
UInt64Type,
855+
round_unsigned_integer,
856+
"UInt64"
857+
),
858+
_ => internal_err!("Unexpected return type for integer round: {return_type}"),
859+
}
860+
}
861+
637862
fn round_float<T>(value: T, decimal_places: i32) -> Result<T, ArrowError>
638863
where
639864
T: num_traits::Float,

‎datafusion/spark/src/function/math/round.rs‎

Lines changed: 15 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -462,18 +462,7 @@ fn spark_round(args: &[ColumnarValue], enable_ansi_mode: bool) -> Result<Columna
462462
impl_integer_array_round!(array, UInt32Type, scale, enable_ansi_mode)
463463
}
464464
DataType::UInt64 => {
465-
let array = array.as_primitive::<UInt64Type>();
466-
let result: PrimitiveArray<UInt64Type> = array.try_unary(|x| {
467-
let v_i64 = i64::try_from(x).map_err(|_| {
468-
(exec_err!(
469-
"round: UInt64 value {x} exceeds i64::MAX and cannot be rounded"
470-
) as Result<(), _>)
471-
.unwrap_err()
472-
})?;
473-
round_integer(v_i64, scale, enable_ansi_mode)
474-
.map(|v| v as u64)
475-
})?;
476-
Ok(ColumnarValue::Array(Arc::new(result)))
465+
impl_integer_array_round!(array, UInt64Type, scale, enable_ansi_mode)
477466
}
478467

479468
// Float types
@@ -588,16 +577,20 @@ fn spark_round(args: &[ColumnarValue], enable_ansi_mode: bool) -> Result<Columna
588577
Ok(ColumnarValue::Scalar(ScalarValue::UInt32(Some(result))))
589578
}
590579
ScalarValue::UInt64(Some(v)) => {
591-
let v_i64 = i64::try_from(*v).map_err(|_| {
592-
(exec_err!(
593-
"round: UInt64 value {v} exceeds i64::MAX and cannot be rounded"
594-
) as Result<(), _>)
595-
.unwrap_err()
596-
})?;
597-
let result = round_integer(v_i64, scale, enable_ansi_mode)?;
598-
Ok(ColumnarValue::Scalar(ScalarValue::UInt64(Some(
599-
result as u64,
600-
))))
580+
if scale >= 0 {
581+
Ok(ColumnarValue::Scalar(ScalarValue::UInt64(Some(*v))))
582+
} else {
583+
let v_i64 = i64::try_from(*v).map_err(|_| {
584+
(exec_err!(
585+
"round: UInt64 value {v} exceeds i64::MAX and cannot be rounded"
586+
) as Result<(), _>)
587+
.unwrap_err()
588+
})?;
589+
let result = round_integer(v_i64, scale, enable_ansi_mode)?;
590+
Ok(ColumnarValue::Scalar(ScalarValue::UInt64(Some(
591+
result as u64,
592+
))))
593+
}
601594
}
602595

603596
// Float scalars

0 commit comments

Comments
 (0)