Skip to content

Commit 6603fcf

Browse files
joseph-isaacsjoseph-isaacs
committed
Emit Decimal32/Decimal64 from the Arrow exporter (on DataFusion 54)
Rebased onto the DataFusion 54 upgrade branch (adamg/df-54) to evaluate narrow decimal export against DataFusion 54. Maps decimals to the smallest Arrow width that fits the precision: Decimal32 for <=9, Decimal64 for <=18, Decimal128 for <=38, else Decimal256. The Arrow executor already handles all four widths. Note: DataFusion 54's decimal SUM/AVG accumulators are unchanged from 53 (branch-54 keeps SUM(Decimal32) -> Decimal32(min(9, p+10), s) in i32), so aggregating inferred narrow decimals is still expected to overflow (e.g. TPC-DS Q1). This branch is for evaluating that behavior on 54. Signed-off-by: Joe Isaacs <[email protected]>
1 parent 6faea70 commit 6603fcf

1 file changed

Lines changed: 24 additions & 6 deletions

File tree

‎vortex-array/src/dtype/arrow.rs‎

Lines changed: 24 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -298,13 +298,12 @@ pub(crate) fn to_data_type_naive(dtype: &DType) -> VortexResult<DataType> {
298298
let scale = dt.scale();
299299

300300
match precision {
301-
// This code is commented out until DataFusion improves its support for smaller decimals.
302-
// // DECIMAL32_MAX_PRECISION
303-
// 0..=9 => DataType::Decimal32(precision, scale),
304-
// // DECIMAL64_MAX_PRECISION
305-
// 10..=18 => DataType::Decimal64(precision, scale),
301+
// DECIMAL32_MAX_PRECISION
302+
0..=9 => DataType::Decimal32(precision, scale),
303+
// DECIMAL64_MAX_PRECISION
304+
10..=18 => DataType::Decimal64(precision, scale),
306305
// DECIMAL128_MAX_PRECISION
307-
0..=38 => DataType::Decimal128(precision, scale),
306+
19..=38 => DataType::Decimal128(precision, scale),
308307
// DECIMAL256_MAX_PRECISION
309308
39.. => DataType::Decimal256(precision, scale),
310309
}
@@ -450,6 +449,25 @@ mod test {
450449
);
451450
}
452451

452+
#[rstest]
453+
#[case(1, DataType::Decimal32(1, 0))]
454+
#[case(9, DataType::Decimal32(9, 0))]
455+
#[case(10, DataType::Decimal64(10, 0))]
456+
#[case(18, DataType::Decimal64(18, 0))]
457+
#[case(19, DataType::Decimal128(19, 0))]
458+
#[case(38, DataType::Decimal128(38, 0))]
459+
#[case(39, DataType::Decimal256(39, 0))]
460+
#[case(76, DataType::Decimal256(76, 0))]
461+
fn test_decimal_dtype_to_arrow_smallest_width(
462+
#[case] precision: u8,
463+
#[case] expected: DataType,
464+
) {
465+
use crate::dtype::DecimalDType;
466+
467+
let dtype = DType::Decimal(DecimalDType::new(precision, 0), Nullability::NonNullable);
468+
assert_eq!(dtype.to_arrow_dtype().unwrap(), expected);
469+
}
470+
453471
#[test]
454472
fn test_variant_dtype_to_arrow_dtype_errors() {
455473
let err = DType::Variant(Nullability::NonNullable)

0 commit comments

Comments
 (0)