Skip to content

Commit ad7d6ea

Browse files
authored
fix: Handle decimal columns consistently in SLT tests (#23161)
## Which issue does this PR close? - Closes #23160. ## Rationale for this change As described in the issue, different width decimals are currently handled differently, going through inconsistent display path and not presenting the full precision. ## What changes are included in this PR? 1. Unifies all handling and formatting for the four decimal variants, which are now all supported. Also includes full display of decimal values up to the specified precision, which should be deterministic. 2. I've also reran the all tests with `--complete`, which makes a few random changes around empty whitespaces which I think is good to run periodically (I keep doing that locally in unrelated changes and having to `git restore` a bunch of files). Would it make sense to maybe add this check to CI and verify it leaves the worktree clean? ## Are these changes tested? Ran all tests locally ## Are there any user-facing changes? Users of `datafusion-sqllogictest` will get different results if they depend on the existing behavior. --------- Signed-off-by: Adam Gutglick <[email protected]>
1 parent e1c9012 commit ad7d6ea

19 files changed

Lines changed: 115 additions & 97 deletions

‎datafusion/sqllogictest/src/engines/conversion.rs‎

Lines changed: 23 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@
1515
// specific language governing permissions and limitations
1616
// under the License.
1717

18-
use arrow::datatypes::{Decimal128Type, Decimal256Type, DecimalType, i256};
18+
use arrow::datatypes::DecimalType;
1919
use bigdecimal::BigDecimal;
2020
use half::f16;
2121
use std::str::FromStr;
@@ -96,21 +96,15 @@ pub(crate) fn spark_f64_to_str(value: f64) -> String {
9696
}
9797
}
9898

99-
pub(crate) fn decimal_128_to_str(value: i128, scale: i8) -> String {
99+
pub(crate) fn arrow_decimal_to_str<T: DecimalType>(
100+
value: T::Native,
101+
scale: i8,
102+
) -> String {
100103
let precision = u8::MAX; // does not matter
104+
let value = T::format_decimal(value, precision, scale);
101105
big_decimal_to_str(
102-
BigDecimal::from_str(&Decimal128Type::format_decimal(value, precision, scale))
103-
.unwrap(),
104-
None,
105-
)
106-
}
107-
108-
pub(crate) fn decimal_256_to_str(value: i256, scale: i8) -> String {
109-
let precision = u8::MAX; // does not matter
110-
big_decimal_to_str(
111-
BigDecimal::from_str(&Decimal256Type::format_decimal(value, precision, scale))
112-
.unwrap(),
113-
None,
106+
BigDecimal::from_str(&value).unwrap(),
107+
Some(i64::from(scale)),
114108
)
115109
}
116110

@@ -132,7 +126,10 @@ pub(crate) fn big_decimal_to_str(value: BigDecimal, round_digits: Option<i64>) -
132126

133127
#[cfg(test)]
134128
mod tests {
135-
use super::big_decimal_to_str;
129+
use super::{arrow_decimal_to_str, big_decimal_to_str};
130+
use arrow::datatypes::{
131+
Decimal32Type, Decimal64Type, Decimal128Type, Decimal256Type, i256,
132+
};
136133
use bigdecimal::{BigDecimal, num_bigint::BigInt};
137134

138135
macro_rules! assert_decimal_str_eq {
@@ -196,4 +193,15 @@ mod tests {
196193

197194
assert_decimal_str_eq!(10_i128.pow(13) + 11, 13, Some(13), "1.0000000000011");
198195
}
196+
197+
#[test]
198+
fn test_arrow_decimal_to_str() {
199+
assert_eq!(arrow_decimal_to_str::<Decimal32Type>(12345, 2), "123.45");
200+
assert_eq!(arrow_decimal_to_str::<Decimal64Type>(12345, 2), "123.45");
201+
assert_eq!(arrow_decimal_to_str::<Decimal128Type>(12345, 2), "123.45");
202+
assert_eq!(
203+
arrow_decimal_to_str::<Decimal256Type>(i256::from(12345), 2),
204+
"123.45"
205+
);
206+
}
199207
}

‎datafusion/sqllogictest/src/engines/datafusion_engine/normalize.rs‎

Lines changed: 18 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,9 @@ use super::super::conversion::*;
1919
use super::error::{DFSqlLogicTestError, Result};
2020
use crate::engines::output::DFColumnType;
2121
use arrow::array::{Array, AsArray};
22-
use arrow::datatypes::{Fields, Schema};
22+
use arrow::datatypes::{
23+
Decimal32Type, Decimal64Type, Decimal128Type, Decimal256Type, Fields, Schema,
24+
};
2325
use arrow::util::display::ArrayFormatter;
2426
use arrow::{array, array::ArrayRef, datatypes::DataType, record_batch::RecordBatch};
2527
use datafusion::common::internal_datafusion_err;
@@ -209,13 +211,21 @@ pub fn cell_to_string(col: &ArrayRef, row: usize, is_spark_path: bool) -> Result
209211
Ok(f64_to_str(result))
210212
}
211213
}
214+
DataType::Decimal32(_, scale) => {
215+
let value = get_row_value!(array::Decimal32Array, col, row);
216+
Ok(arrow_decimal_to_str::<Decimal32Type>(value, *scale))
217+
}
218+
DataType::Decimal64(_, scale) => {
219+
let value = get_row_value!(array::Decimal64Array, col, row);
220+
Ok(arrow_decimal_to_str::<Decimal64Type>(value, *scale))
221+
}
212222
DataType::Decimal128(_, scale) => {
213223
let value = get_row_value!(array::Decimal128Array, col, row);
214-
Ok(decimal_128_to_str(value, *scale))
224+
Ok(arrow_decimal_to_str::<Decimal128Type>(value, *scale))
215225
}
216226
DataType::Decimal256(_, scale) => {
217227
let value = get_row_value!(array::Decimal256Array, col, row);
218-
Ok(decimal_256_to_str(value, *scale))
228+
Ok(arrow_decimal_to_str::<Decimal256Type>(value, *scale))
219229
}
220230
DataType::LargeUtf8 => Ok(varchar_to_str(get_row_value!(
221231
array::LargeStringArray,
@@ -268,9 +278,11 @@ pub fn convert_schema_to_types(columns: &Fields) -> Vec<DFColumnType> {
268278
| DataType::UInt16
269279
| DataType::UInt32
270280
| DataType::UInt64 => DFColumnType::Integer,
271-
DataType::Float16
272-
| DataType::Float32
273-
| DataType::Float64
281+
DataType::Float16 | DataType::Float32 | DataType::Float64 => {
282+
DFColumnType::Float
283+
}
284+
DataType::Decimal32(_, _)
285+
| DataType::Decimal64(_, _)
274286
| DataType::Decimal128(_, _)
275287
| DataType::Decimal256(_, _) => DFColumnType::Float,
276288
DataType::Utf8 | DataType::LargeUtf8 | DataType::Utf8View => {

‎datafusion/sqllogictest/test_files/aggregate.slt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9179,7 +9179,7 @@ ORDER BY g;
91799179

91809180
# Config reset
91819181

9182-
# The SLT runner sets `target_partitions` to 4 instead of using the default, so
9182+
# The SLT runner sets `target_partitions` to 4 instead of using the default, so
91839183
# reset it explicitly.
91849184
statement ok
91859185
set datafusion.execution.target_partitions = 4;

‎datafusion/sqllogictest/test_files/aggregate_repartition.slt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -131,7 +131,7 @@ physical_plan
131131

132132
# Config reset
133133

134-
# The SLT runner sets `target_partitions` to 4 instead of using the default, so
134+
# The SLT runner sets `target_partitions` to 4 instead of using the default, so
135135
# reset it explicitly.
136136
statement ok
137137
SET datafusion.execution.target_partitions = 4;

‎datafusion/sqllogictest/test_files/aggregate_skip_partial.slt‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -220,7 +220,7 @@ e true false NULL
220220
statement ok
221221
reset datafusion.execution.batch_size;
222222

223-
# The SLT runner sets `target_partitions` to 4 instead of using the default, so
223+
# The SLT runner sets `target_partitions` to 4 instead of using the default, so
224224
# reset it explicitly.
225225
statement ok
226226
set datafusion.execution.target_partitions = 4;
@@ -712,7 +712,7 @@ ORDER BY i;
712712
statement ok
713713
reset datafusion.execution.batch_size;
714714

715-
# The SLT runner sets `target_partitions` to 4 instead of using the default, so
715+
# The SLT runner sets `target_partitions` to 4 instead of using the default, so
716716
# reset it explicitly.
717717
statement ok
718718
set datafusion.execution.target_partitions = 4;
@@ -772,7 +772,7 @@ true false false false false true false NULL
772772
statement ok
773773
reset datafusion.execution.batch_size;
774774

775-
# The SLT runner sets `target_partitions` to 4 instead of using the default, so
775+
# The SLT runner sets `target_partitions` to 4 instead of using the default, so
776776
# reset it explicitly.
777777
statement ok
778778
set datafusion.execution.target_partitions = 4;

‎datafusion/sqllogictest/test_files/array/cleanup.slt.part‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -167,4 +167,3 @@ drop table large_arrays_values_without_nulls;
167167

168168
statement ok
169169
drop table fixed_size_arrays_values_without_nulls;
170-

‎datafusion/sqllogictest/test_files/array/init_data.slt.part‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -689,4 +689,3 @@ AS
689689
FROM arrays_distance_table
690690
;
691691

692-

‎datafusion/sqllogictest/test_files/array_agg_sliding_window.slt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -430,4 +430,4 @@ statement ok
430430
DROP TABLE t_dist_parts;
431431

432432
statement ok
433-
DROP TABLE t_dist_int;
433+
DROP TABLE t_dist_int;

‎datafusion/sqllogictest/test_files/group_by.slt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5634,7 +5634,7 @@ physical_plan
56345634
statement ok
56355635
reset datafusion.execution.batch_size;
56365636

5637-
# The SLT runner sets `target_partitions` to 4 instead of using the default, so
5637+
# The SLT runner sets `target_partitions` to 4 instead of using the default, so
56385638
# reset it explicitly.
56395639
statement ok
56405640
set datafusion.execution.target_partitions = 4;

‎datafusion/sqllogictest/test_files/options.slt‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -209,20 +209,20 @@ query RT
209209
select 123456789.0123456789012345678901234567890,
210210
arrow_typeof(123456789.0123456789012345678901234567890)
211211
----
212-
123456789.012345678901 Decimal256(40, 31)
212+
123456789.012345678901234567890123456789 Decimal256(40, 31)
213213

214214
query RT
215215
select -123456789.0123456789012345678901234567890,
216216
arrow_typeof(-123456789.0123456789012345678901234567890)
217217
----
218-
-123456789.012345678901 Decimal256(40, 31)
218+
-123456789.012345678901234567890123456789 Decimal256(40, 31)
219219

220220
# max precision and scale of Decimal256
221221
query RTRT
222222
select -1e-76, arrow_typeof(-1e-76),
223223
-1.234567e-70, arrow_typeof(-1.234567e-70)
224224
----
225-
0 Decimal256(76, 76) 0 Decimal256(76, 76)
225+
-0.0000000000000000000000000000000000000000000000000000000000000000000000000001 Decimal256(76, 76) -0.0000000000000000000000000000000000000000000000000000000000000000000001234567 Decimal256(76, 76)
226226

227227
# Decimal256::MAX for nonnegative scale
228228
query RT
@@ -243,13 +243,13 @@ query RTRT
243243
select 1e-38, arrow_typeof(1e-38),
244244
1e-39, arrow_typeof(1e-39);
245245
----
246-
0 Decimal128(38, 38) 0 Decimal256(39, 39)
246+
0.00000000000000000000000000000000000001 Decimal128(38, 38) 0.000000000000000000000000000000000000001 Decimal256(39, 39)
247247

248248
query RTRT
249249
select -1e-38, arrow_typeof(-1e-38),
250250
-1e-39, arrow_typeof(-1e-39);
251251
----
252-
0 Decimal128(38, 38) 0 Decimal256(39, 39)
252+
-0.00000000000000000000000000000000000001 Decimal128(38, 38) -0.000000000000000000000000000000000000001 Decimal256(39, 39)
253253

254254
# unsupported precision
255255
query error Decimal precision 77 exceeds the maximum supported precision: 76

0 commit comments

Comments
 (0)