Skip to content

Commit 9442d88

Browse files
authored
fix: keep decimal precision for SLT tests (apache#24934)
## Which issue does this PR close? - Closes apache#23835. ## Rationale for this change Provide full precision for decimals in SLT tests. Separate float formatting (round to 12/15 digits) from decimal formatting (avoid rounding) ## What changes are included in this PR? - For floats, round to 12 digits by default and 15 for Spark (historically), via a generic function - For decimals, avoid rounding entirely. Emit any trailing zeroes to SLT output, as they are part of decimals. Rely on `format_decimal` directly - For Postgres mode with `decimal_to_str` - emit BigDecimal representation of numerics directly - Remove `big_decimal_to_str` helper with rounding behaviour - A few amends to pg_compat tests. I introduced casts in pg_compat tests to fit pg_compat mode with explicit types, either double precision or numeric(p,s). Postgres treats literals as numeric and so prints without rounding. For some UDF (avg, exp), it was required to cast the output to float explicitly, because Postgres widens inputs and outputs to decimal. In DataFusion, it produces floats for float inputs. - Refines apache#23160 - CC @AdamGS ## What is the testing strategy for this PR? - An SLT test case for the issue - now it produces the expected decimal with full precision - Reworked unit tests to test float and decimal conversion, but not an intermediate `big_decimal_to_str` - Regenerated SLTs with `--complete` to introduce trailing zeroes - a huge inevitable change ## Are there any user-facing changes?
1 parent be5f265 commit 9442d88

37 files changed

Lines changed: 629 additions & 651 deletions

‎Cargo.lock‎

Lines changed: 1 addition & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎datafusion/sqllogictest/Cargo.toml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,7 @@ half = { workspace = true, default-features = true }
5454
indicatif = "0.18"
5555
itertools = { workspace = true }
5656
log = { workspace = true }
57+
num-traits = { workspace = true }
5758
object_store = { workspace = true }
5859
postgres-types = { version = "0.2.13", features = ["derive", "with-chrono-0_4"], optional = true }
5960
# When updating the following dependency verify that sqlite test file regeneration works correctly

‎datafusion/sqllogictest/README.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -449,7 +449,7 @@ query <type_string> <sort_mode>
449449
- 'T' - **T**ext,
450450
- "?" - any other types
451451
- `expected_result`: In the results section, some values are converted according to some rules:
452-
- floating point values are rounded to the scale of "12",
452+
- floating point values are rounded to the scale of "12" and "15" for Spark,
453453
- NULL values are rendered as `NULL`,
454454
- empty strings are rendered as `(empty)`,
455455
- boolean values are rendered as `true`/`false`,

‎datafusion/sqllogictest/src/engines/conversion.rs‎

Lines changed: 74 additions & 116 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
use arrow::datatypes::DecimalType;
1919
use bigdecimal::BigDecimal;
2020
use half::f16;
21+
use num_traits::Float;
2122
use std::str::FromStr;
2223

2324
/// Represents a constant for NULL string in your database.
@@ -40,158 +41,89 @@ pub(crate) fn varchar_to_str(value: &str) -> String {
4041
}
4142
}
4243

43-
pub(crate) fn f16_to_str(value: f16) -> String {
44+
/// Decimal places to keep in floats for string representation.
45+
const FLOAT_ROUND_DIGITS: i64 = 12;
46+
47+
/// Decimal places to keep in floats for Spark (derived from expm1 behaviour)
48+
const SPARK_FLOAT_ROUND_DIGITS: i64 = 15;
49+
50+
pub(crate) fn float_to_str<T: Float + ToString>(value: T, round_digits: i64) -> String {
4451
if value.is_nan() {
4552
// The sign of NaN can be different depending on platform.
4653
// So the string representation of NaN ignores the sign.
4754
"NaN".to_string()
48-
} else if value == f16::INFINITY {
55+
} else if value == T::infinity() {
4956
"Infinity".to_string()
50-
} else if value == f16::NEG_INFINITY {
57+
} else if value == T::neg_infinity() {
5158
"-Infinity".to_string()
5259
} else {
53-
big_decimal_to_str(BigDecimal::from_str(&value.to_string()).unwrap(), None)
60+
float_decimal_to_str(value, round_digits)
5461
}
5562
}
5663

64+
pub(crate) fn f16_to_str(value: f16) -> String {
65+
float_to_str(value, FLOAT_ROUND_DIGITS)
66+
}
67+
5768
pub(crate) fn f32_to_str(value: f32) -> String {
58-
if value.is_nan() {
59-
// The sign of NaN can be different depending on platform.
60-
// So the string representation of NaN ignores the sign.
61-
"NaN".to_string()
62-
} else if value == f32::INFINITY {
63-
"Infinity".to_string()
64-
} else if value == f32::NEG_INFINITY {
65-
"-Infinity".to_string()
66-
} else {
67-
big_decimal_to_str(BigDecimal::from_str(&value.to_string()).unwrap(), None)
68-
}
69+
float_to_str(value, FLOAT_ROUND_DIGITS)
6970
}
7071

7172
pub(crate) fn f64_to_str(value: f64) -> String {
72-
if value.is_nan() {
73-
// The sign of NaN can be different depending on platform.
74-
// So the string representation of NaN ignores the sign.
75-
"NaN".to_string()
76-
} else if value == f64::INFINITY {
77-
"Infinity".to_string()
78-
} else if value == f64::NEG_INFINITY {
79-
"-Infinity".to_string()
80-
} else {
81-
big_decimal_to_str(BigDecimal::from_str(&value.to_string()).unwrap(), None)
82-
}
73+
float_to_str(value, FLOAT_ROUND_DIGITS)
8374
}
8475

8576
pub(crate) fn spark_f64_to_str(value: f64) -> String {
86-
if value.is_nan() {
87-
// The sign of NaN can be different depending on platform.
88-
// So the string representation of NaN ignores the sign.
89-
"NaN".to_string()
90-
} else if value == f64::INFINITY {
91-
"Infinity".to_string()
92-
} else if value == f64::NEG_INFINITY {
93-
"-Infinity".to_string()
94-
} else {
95-
big_decimal_to_str(BigDecimal::from_str(&value.to_string()).unwrap(), Some(15))
96-
}
77+
// Spark uses 15 decimal places for doubles
78+
float_to_str(value, SPARK_FLOAT_ROUND_DIGITS)
9779
}
9880

81+
/// Converts a float to its plain string representation, rounding to a specified number of decimal places.
82+
fn float_decimal_to_str<T: Float + ToString>(value: T, round_digits: i64) -> String {
83+
let bd = BigDecimal::from_str(&value.to_string()).unwrap();
84+
// Round the value to limit the number of decimal places
85+
let value = bd.round(round_digits).normalized();
86+
// Format the value to a string
87+
value.to_plain_string()
88+
}
89+
90+
/// Converts a decimal to its plain string representation, using the given scale
9991
pub(crate) fn arrow_decimal_to_str<T: DecimalType>(
10092
value: T::Native,
10193
scale: i8,
10294
) -> String {
103-
let precision = u8::MAX; // does not matter
104-
let value = T::format_decimal(value, precision, scale);
105-
big_decimal_to_str(
106-
BigDecimal::from_str(&value).unwrap(),
107-
Some(i64::from(scale)),
108-
)
95+
T::format_decimal(value, u8::MAX, scale)
10996
}
11097

11198
#[cfg(feature = "postgres")]
11299
pub(crate) fn decimal_to_str(value: BigDecimal) -> String {
113-
big_decimal_to_str(value, None)
114-
}
115-
116-
/// Converts a `BigDecimal` to its plain string representation, optionally rounding to a specified number of decimal places.
117-
///
118-
/// If `round_digits` is `None`, the value is rounded to 12 decimal places by default.
119-
#[expect(clippy::needless_pass_by_value)]
120-
pub(crate) fn big_decimal_to_str(value: BigDecimal, round_digits: Option<i64>) -> String {
121-
// Round the value to limit the number of decimal places
122-
let value = value.round(round_digits.unwrap_or(12)).normalized();
123-
// Format the value to a string
124100
value.to_plain_string()
125101
}
126102

127103
#[cfg(test)]
128104
mod tests {
129-
use super::{arrow_decimal_to_str, big_decimal_to_str};
105+
use super::*;
130106
use arrow::datatypes::{
131-
Decimal32Type, Decimal64Type, Decimal128Type, Decimal256Type, i256,
107+
DECIMAL128_MAX_SCALE, Decimal32Type, Decimal64Type, Decimal128Type,
108+
Decimal256Type, i256,
132109
};
133-
use bigdecimal::{BigDecimal, num_bigint::BigInt};
134-
135-
macro_rules! assert_decimal_str_eq {
136-
($integer:expr, $scale:expr, $round_digits:expr, $expected:expr) => {
137-
assert_eq!(
138-
big_decimal_to_str(
139-
BigDecimal::from_bigint(BigInt::from($integer), $scale),
140-
$round_digits
141-
),
142-
$expected
143-
);
144-
};
145-
}
146110

147111
#[test]
148-
fn test_big_decimal_to_str() {
149-
assert_decimal_str_eq!(110, 3, None, "0.11");
150-
assert_decimal_str_eq!(11, 3, None, "0.011");
151-
assert_decimal_str_eq!(11, 2, None, "0.11");
152-
assert_decimal_str_eq!(11, 1, None, "1.1");
153-
assert_decimal_str_eq!(11, 0, None, "11");
154-
assert_decimal_str_eq!(11, -1, None, "110");
155-
assert_decimal_str_eq!(0, 0, None, "0");
156-
assert_decimal_str_eq!(
157-
12345678901234567890123456789012345678_i128,
158-
0,
159-
None,
160-
"12345678901234567890123456789012345678"
161-
);
162-
assert_decimal_str_eq!(
163-
12345678901234567890123456789012345678_i128,
164-
38,
165-
None,
166-
"0.123456789012"
167-
);
168-
169-
// Negative cases
170-
assert_decimal_str_eq!(-110, 3, None, "-0.11");
171-
assert_decimal_str_eq!(-11, 3, None, "-0.011");
172-
assert_decimal_str_eq!(-11, 2, None, "-0.11");
173-
assert_decimal_str_eq!(-11, 1, None, "-1.1");
174-
assert_decimal_str_eq!(-11, 0, None, "-11");
175-
assert_decimal_str_eq!(-11, -1, None, "-110");
176-
assert_decimal_str_eq!(
177-
-12345678901234567890123456789012345678_i128,
178-
0,
179-
None,
180-
"-12345678901234567890123456789012345678"
181-
);
182-
assert_decimal_str_eq!(
183-
-12345678901234567890123456789012345678_i128,
184-
38,
185-
None,
186-
"-0.123456789012"
187-
);
188-
189-
// Round to 12 decimal places
190-
// 1.0000000000011 -> 1.000000000001
191-
assert_decimal_str_eq!(10_i128.pow(13) + 11, 13, None, "1.000000000001");
192-
assert_decimal_str_eq!(10_i128.pow(13) + 11, 13, Some(12), "1.000000000001");
193-
194-
assert_decimal_str_eq!(10_i128.pow(13) + 11, 13, Some(13), "1.0000000000011");
112+
fn test_float_decimal_to_str() {
113+
assert_eq!(f64_to_str(0.11), "0.11");
114+
assert_eq!(f64_to_str(0.011), "0.011");
115+
assert_eq!(f64_to_str(1.1), "1.1");
116+
assert_eq!(f64_to_str(11.0), "11");
117+
assert_eq!(f64_to_str(-0.11), "-0.11");
118+
assert_eq!(f64_to_str(-0.011), "-0.011");
119+
assert_eq!(f64_to_str(-1.1,), "-1.1");
120+
assert_eq!(f64_to_str(-11.0), "-11");
121+
// Keep 12 decimal places
122+
assert_eq!(f64_to_str(0.12345678901234567), "0.123456789012");
123+
124+
assert_eq!(spark_f64_to_str(-0.011), "-0.011");
125+
// Keep 15 decimal places for Spark
126+
assert_eq!(spark_f64_to_str(0.12345678901234567), "0.123456789012346");
195127
}
196128

197129
#[test]
@@ -203,5 +135,31 @@ mod tests {
203135
arrow_decimal_to_str::<Decimal256Type>(i256::from(12345), 2),
204136
"123.45"
205137
);
138+
// Ensure it doesn't trim trailing zeros
139+
assert_eq!(arrow_decimal_to_str::<Decimal128Type>(12300, 2), "123.00");
140+
assert_eq!(arrow_decimal_to_str::<Decimal128Type>(12300, 4), "1.2300");
141+
142+
// Keep full precision besides 12 decimal places
143+
assert_eq!(
144+
arrow_decimal_to_str::<Decimal128Type>(10_i128.pow(13) + 11, 13),
145+
"1.0000000000011"
146+
);
147+
148+
// Zero-scale integers
149+
assert_eq!(
150+
arrow_decimal_to_str::<Decimal128Type>(
151+
12345678901234567890123456789012345678_i128,
152+
0
153+
),
154+
"12345678901234567890123456789012345678"
155+
);
156+
// Keep full precision
157+
assert_eq!(
158+
arrow_decimal_to_str::<Decimal128Type>(
159+
12345678901234567890123456789012345678_i128,
160+
DECIMAL128_MAX_SCALE
161+
),
162+
"0.12345678901234567890123456789012345678"
163+
);
206164
}
207165
}

0 commit comments

Comments
 (0)