Skip to content

Commit 38ea24d

Browse files
haohuaijindiscord9
authored andcommitted
feat: recognize more lossless casts for statistics and ordering (apache#25500)
## Which issue does this PR close? Follow-up to apache#25227 and [its discussion](apache#25227 (comment)). ## Rationale for this change Some lossless casts unnecessarily discard column statistics or fail to preserve strict ordering information. For example, UInt32 → Int64 currently requires both bounds to be exact to propagate statistics, even though every UInt32 value fits in Int64. Recognizing this conversion as globally safe preserves available statistics even when one bound is absent or bounds are inexact. ## What changes are included in this PR? - Recognize unsigned-to-signed widening casts: UInt8 → Int16 / Int32 / Int64, UInt16 → Int32 / Int64, and UInt32 → Int64. - Recognize Utf8 → Utf8View and Binary → LargeBinary / BinaryView. - Extend the existing `check_bigger_cast` / `is_bigger_cast` predicates to recognize these lossless conversions. ## What is the testing strategy for this PR? - Test values, nulls, integer extrema, and strict ordering properties. - Test empty values, Unicode, long strings, and non-UTF8 binary values. - Test statistics propagation with exact, inexact, and missing bounds. - Verify unsafe reverse conversions remain excluded. Validation on the latest revision: 25 relevant unit tests passed; one existing test is ignored. Formatting, all-target/all-feature Clippy, and the standard `rust_lint.sh` suite passed. ## Are there any user-facing changes? More lossless casts retain statistics and strict ordering information, enabling existing optimizer rules. Backport provenance: - Upstream PR: apache#25500 - Upstream commit: b172f73 - Retain the upstream matcher and its complete self-tests; only the absent fork example-doc context is adjusted. - Original upstream tests are retained; fork-only adaptation, if any, follows separately. Signed-off-by: discord9 <[email protected]>
1 parent 390e174 commit 38ea24d

2 files changed

Lines changed: 257 additions & 7 deletions

File tree

‎datafusion/physical-expr/src/expressions/cast.rs‎

Lines changed: 182 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -250,7 +250,7 @@ impl CastExpr {
250250
}
251251

252252
/// Check if casting from the specified source type to the target type is a
253-
/// widening cast (e.g. from `Int8` to `Int16`).
253+
/// lossless and strictly order-preserving cast (e.g. from `UInt8` to `Int16`).
254254
pub fn check_bigger_cast(cast_type: &DataType, src: &DataType) -> bool {
255255
if cast_type.eq(src) {
256256
return true;
@@ -260,12 +260,13 @@ impl CastExpr {
260260
(Int8, Int16 | Int32 | Int64)
261261
| (Int16, Int32 | Int64)
262262
| (Int32, Int64)
263-
| (UInt8, UInt16 | UInt32 | UInt64)
264-
| (UInt16, UInt32 | UInt64)
265-
| (UInt32, UInt64)
263+
| (UInt8, UInt16 | UInt32 | UInt64 | Int16 | Int32 | Int64)
264+
| (UInt16, UInt32 | UInt64 | Int32 | Int64)
265+
| (UInt32, UInt64 | Int64)
266266
| (Int8 | Int16 | UInt8 | UInt16, Float32)
267267
| (Int8 | Int16 | Int32 | UInt8 | UInt16 | UInt32, Float64)
268-
| (Utf8, LargeUtf8)
268+
| (Utf8, LargeUtf8 | Utf8View)
269+
| (Binary, LargeBinary | BinaryView)
269270
)
270271
}
271272

@@ -1480,10 +1481,184 @@ mod tests {
14801481
assert!(!CastExpr::check_bigger_cast(&Float64, &Int64));
14811482
assert!(!CastExpr::check_bigger_cast(&Float64, &UInt64));
14821483

1483-
// Signed <-> Unsigned conversions should return false (not order-preserving due to negative values)
1484+
// Signed-to-unsigned and unsigned-to-signed casts whose target cannot
1485+
// represent the entire source range are not lossless for all values.
14841486
assert!(!CastExpr::check_bigger_cast(&UInt16, &Int8));
14851487
assert!(!CastExpr::check_bigger_cast(&UInt32, &Int16));
1486-
assert!(!CastExpr::check_bigger_cast(&Int16, &UInt8));
1488+
assert!(!CastExpr::check_bigger_cast(&Int8, &UInt8));
1489+
assert!(!CastExpr::check_bigger_cast(&Int16, &UInt16));
1490+
assert!(!CastExpr::check_bigger_cast(&Int32, &UInt32));
1491+
assert!(!CastExpr::check_bigger_cast(&Int64, &UInt64));
1492+
assert!(!CastExpr::check_bigger_cast(&Int8, &UInt16));
1493+
}
1494+
#[test]
1495+
fn test_byte_representation_cast_preserves_values_and_ordering() -> Result<()> {
1496+
use arrow::array::{
1497+
BinaryArray, BinaryViewArray, LargeBinaryArray, StringViewArray,
1498+
};
1499+
use arrow::compute::SortOptions;
1500+
use datafusion_expr_common::sort_properties::SortProperties;
1501+
1502+
// Cover nulls, empty values, inline/long views, Unicode, and non-UTF8 bytes.
1503+
let strings = vec![
1504+
None,
1505+
Some(""),
1506+
Some("a"),
1507+
Some("a longer shared string"),
1508+
Some("a longer shared string"),
1509+
Some("🦀"),
1510+
];
1511+
let bytes: Vec<Option<&[u8]>> = vec![
1512+
None,
1513+
Some(b""),
1514+
Some(b"\0"),
1515+
Some(b"a longer shared byte string"),
1516+
Some(b"a longer shared byte string"),
1517+
Some(b"\xff"),
1518+
];
1519+
let binary: ArrayRef = Arc::new(BinaryArray::from(bytes.clone()));
1520+
let cases: [(ArrayRef, ArrayRef); 3] = [
1521+
(
1522+
Arc::new(StringArray::from(strings.clone())),
1523+
Arc::new(StringViewArray::from(strings)),
1524+
),
1525+
(
1526+
Arc::clone(&binary),
1527+
Arc::new(LargeBinaryArray::from(bytes.clone())),
1528+
),
1529+
(binary, Arc::new(BinaryViewArray::from(bytes))),
1530+
];
1531+
for (input, expected) in cases {
1532+
let schema = Arc::new(Schema::new(vec![Field::new(
1533+
"a",
1534+
input.data_type().clone(),
1535+
true,
1536+
)]));
1537+
let expr =
1538+
CastExpr::new(col("a", &schema)?, expected.data_type().clone(), None);
1539+
assert!(expr.is_bigger_cast(input.data_type()));
1540+
for descending in [false, true] {
1541+
for nulls_first in [false, true] {
1542+
let child = ExprProperties::new_unknown()
1543+
.with_range(Interval::make_unbounded(input.data_type())?)
1544+
.with_order(SortProperties::Ordered(SortOptions {
1545+
descending,
1546+
nulls_first,
1547+
}))
1548+
.with_strictly_order_preserving(true);
1549+
let properties = expr.get_properties(std::slice::from_ref(&child))?;
1550+
assert_eq!(properties.sort_properties, child.sort_properties);
1551+
assert!(properties.strictly_order_preserving);
1552+
assert_eq!(properties.range.data_type(), *expected.data_type());
1553+
}
1554+
}
1555+
let batch = RecordBatch::try_new(schema, vec![input])?;
1556+
let actual = expr.evaluate(&batch)?.into_array(batch.num_rows())?;
1557+
assert_eq!(actual.as_ref(), expected.as_ref());
1558+
}
1559+
for (source, target) in [
1560+
(Utf8View, Utf8),
1561+
(LargeBinary, Binary),
1562+
(BinaryView, Binary),
1563+
] {
1564+
assert!(!CastExpr::check_bigger_cast(&target, &source));
1565+
}
1566+
Ok(())
1567+
}
1568+
1569+
#[test]
1570+
fn test_unsigned_to_signed_cast_preserves_values_and_ordering() {
1571+
use arrow::array::{UInt8Array, UInt16Array, UInt32Array};
1572+
use arrow::compute::SortOptions;
1573+
use datafusion_expr_common::sort_properties::SortProperties;
1574+
1575+
let inputs: [(ArrayRef, Vec<DataType>, i64); 3] = [
1576+
(
1577+
Arc::new(UInt8Array::from(vec![
1578+
None,
1579+
Some(0),
1580+
Some(1),
1581+
Some(u8::MAX),
1582+
])),
1583+
vec![Int16, Int32, Int64],
1584+
i64::from(u8::MAX),
1585+
),
1586+
(
1587+
Arc::new(UInt16Array::from(vec![
1588+
None,
1589+
Some(0),
1590+
Some(1),
1591+
Some(u16::MAX),
1592+
])),
1593+
vec![Int32, Int64],
1594+
i64::from(u16::MAX),
1595+
),
1596+
(
1597+
Arc::new(UInt32Array::from(vec![
1598+
None,
1599+
Some(0),
1600+
Some(1),
1601+
Some(u32::MAX),
1602+
])),
1603+
vec![Int64],
1604+
i64::from(u32::MAX),
1605+
),
1606+
];
1607+
for (input, target_types, max) in inputs {
1608+
let schema = Arc::new(Schema::new(vec![Field::new(
1609+
"a",
1610+
input.data_type().clone(),
1611+
true,
1612+
)]));
1613+
let batch =
1614+
RecordBatch::try_new(Arc::clone(&schema), vec![Arc::clone(&input)])
1615+
.expect("valid input batch");
1616+
for target_type in target_types {
1617+
let expr =
1618+
CastExpr::new(col("a", &schema).unwrap(), target_type.clone(), None);
1619+
let actual = expr
1620+
.evaluate(&batch)
1621+
.unwrap()
1622+
.into_array(batch.num_rows())
1623+
.unwrap();
1624+
for (index, value) in
1625+
[None, Some(0), Some(1), Some(max)].into_iter().enumerate()
1626+
{
1627+
let expected = match target_type {
1628+
Int16 => {
1629+
ScalarValue::Int16(value.map(|v| i16::try_from(v).unwrap()))
1630+
}
1631+
Int32 => {
1632+
ScalarValue::Int32(value.map(|v| i32::try_from(v).unwrap()))
1633+
}
1634+
Int64 => ScalarValue::Int64(value),
1635+
_ => unreachable!(),
1636+
};
1637+
assert_eq!(
1638+
ScalarValue::try_from_array(&actual, index).unwrap(),
1639+
expected
1640+
);
1641+
}
1642+
for descending in [false, true] {
1643+
for nulls_first in [false, true] {
1644+
let child = ExprProperties::new_unknown()
1645+
.with_range(
1646+
Interval::make_unbounded(input.data_type()).unwrap(),
1647+
)
1648+
.with_order(SortProperties::Ordered(SortOptions {
1649+
descending,
1650+
nulls_first,
1651+
}))
1652+
.with_strictly_order_preserving(true);
1653+
let properties =
1654+
expr.get_properties(std::slice::from_ref(&child)).unwrap();
1655+
assert_eq!(properties.sort_properties, child.sort_properties);
1656+
assert!(properties.strictly_order_preserving);
1657+
assert_eq!(properties.range.data_type(), target_type);
1658+
}
1659+
}
1660+
}
1661+
}
14871662
}
14881663
}
14891664

‎datafusion/physical-expr/src/projection.rs‎

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2490,6 +2490,81 @@ pub(crate) mod tests {
24902490
Ok(())
24912491
}
24922492

2493+
#[test]
2494+
fn test_project_statistics_lossless_cast() {
2495+
use Precision::{Absent, Exact, Inexact};
2496+
2497+
for (lower, upper, targets) in [
2498+
(
2499+
ScalarValue::UInt8(Some(0)),
2500+
ScalarValue::UInt8(Some(u8::MAX)),
2501+
vec![DataType::Int16, DataType::Int32, DataType::Int64],
2502+
),
2503+
(
2504+
ScalarValue::UInt16(Some(0)),
2505+
ScalarValue::UInt16(Some(u16::MAX)),
2506+
vec![DataType::Int32, DataType::Int64],
2507+
),
2508+
(
2509+
ScalarValue::UInt32(Some(0)),
2510+
ScalarValue::UInt32(Some(u32::MAX)),
2511+
vec![DataType::Int64],
2512+
),
2513+
(
2514+
ScalarValue::Utf8(Some(String::new())),
2515+
ScalarValue::Utf8(Some("🦀".to_string())),
2516+
vec![DataType::Utf8View],
2517+
),
2518+
(
2519+
ScalarValue::Binary(Some(vec![])),
2520+
ScalarValue::Binary(Some(vec![0xff])),
2521+
vec![DataType::LargeBinary, DataType::BinaryView],
2522+
),
2523+
] {
2524+
for target in targets {
2525+
// A globally safe cast does not need two exact extrema. Preserve
2526+
// whichever bounds are available without upgrading their precision.
2527+
for (min_value, max_value) in [
2528+
(Exact(lower.clone()), Exact(upper.clone())),
2529+
(Exact(lower.clone()), Absent),
2530+
(Absent, Exact(upper.clone())),
2531+
(Inexact(lower.clone()), Inexact(upper.clone())),
2532+
(Inexact(lower.clone()), Absent),
2533+
(Absent, Inexact(upper.clone())),
2534+
] {
2535+
let input = ColumnStatistics {
2536+
min_value: min_value.clone(),
2537+
max_value: max_value.clone(),
2538+
null_count: Exact(1),
2539+
distinct_count: Inexact(3),
2540+
sum_value: Absent,
2541+
byte_size: Absent,
2542+
};
2543+
let expr = CastExpr::new(
2544+
Arc::new(Column::new("a", 0)),
2545+
target.clone(),
2546+
None,
2547+
);
2548+
let output =
2549+
project_column_statistics_through_expr(&expr, &[input], None);
2550+
assert_eq!(
2551+
output,
2552+
ColumnStatistics {
2553+
min_value: min_value.cast_to(&target).unwrap(),
2554+
max_value: max_value.cast_to(&target).unwrap(),
2555+
null_count: Exact(1),
2556+
distinct_count: Inexact(3),
2557+
sum_value: Absent,
2558+
byte_size: Absent,
2559+
},
2560+
"{} -> {target}, {min_value:?}..{max_value:?}",
2561+
lower.data_type()
2562+
);
2563+
}
2564+
}
2565+
}
2566+
}
2567+
24932568
#[test]
24942569
fn test_column_indices_duplicates() -> Result<()> {
24952570
// Test that duplicate column indices appear only once

0 commit comments

Comments
 (0)