Skip to content

Commit b172f73

Browse files
authored
feat: recognize more lossless casts for statistics and ordering (apache#25500)
## Which issue does this PR close? Follow-up to apache#25227 and [its discussion](apache#25227 (comment)). ## Rationale for this change Some lossless casts unnecessarily discard column statistics or fail to preserve strict ordering information. For example, UInt32 → Int64 currently requires both bounds to be exact to propagate statistics, even though every UInt32 value fits in Int64. Recognizing this conversion as globally safe preserves available statistics even when one bound is absent or bounds are inexact. ## What changes are included in this PR? - Recognize unsigned-to-signed widening casts: UInt8 → Int16 / Int32 / Int64, UInt16 → Int32 / Int64, and UInt32 → Int64. - Recognize Utf8 → Utf8View and Binary → LargeBinary / BinaryView. - Extend the existing `check_bigger_cast` / `is_bigger_cast` predicates to recognize these lossless conversions. ## What is the testing strategy for this PR? - Test values, nulls, integer extrema, and strict ordering properties. - Test empty values, Unicode, long strings, and non-UTF8 binary values. - Test statistics propagation with exact, inexact, and missing bounds. - Verify unsafe reverse conversions remain excluded. Validation on the latest revision: 25 relevant unit tests passed; one existing test is ignored. Formatting, all-target/all-feature Clippy, and the standard `rust_lint.sh` suite passed. ## Are there any user-facing changes? More lossless casts retain statistics and strict ordering information, enabling existing optimizer rules.
1 parent 9951f0b commit b172f73

2 files changed

Lines changed: 258 additions & 7 deletions

File tree

‎datafusion/physical-expr/src/expressions/cast.rs‎

Lines changed: 183 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -250,7 +250,7 @@ impl CastExpr {
250250

251251
/// Check if casting from the source type to the target type is known to be
252252
/// lossless and strictly order-preserving for all source values, preserving nulls.
253-
/// This includes widening casts (e.g. `Int8` to `Int16`) and representation
253+
/// This includes widening casts (e.g. `Int8` or `UInt8` to `Int16`) and representation
254254
/// conversions such as `Int32` to `Date32`, which interprets the same integer
255255
/// as days since the epoch, or `Int64` to `Date64`, which interprets the same
256256
/// integer as milliseconds since the epoch.
@@ -267,12 +267,13 @@ impl CastExpr {
267267
| (Date32, Int32)
268268
| (Int64, Date64)
269269
| (Date64, Int64)
270-
| (UInt8, UInt16 | UInt32 | UInt64)
271-
| (UInt16, UInt32 | UInt64)
272-
| (UInt32, UInt64)
270+
| (UInt8, UInt16 | UInt32 | UInt64 | Int16 | Int32 | Int64)
271+
| (UInt16, UInt32 | UInt64 | Int32 | Int64)
272+
| (UInt32, UInt64 | Int64)
273273
| (Int8 | Int16 | UInt8 | UInt16, Float32)
274274
| (Int8 | Int16 | Int32 | UInt8 | UInt16 | UInt32, Float64)
275-
| (Utf8, LargeUtf8)
275+
| (Utf8, LargeUtf8 | Utf8View)
276+
| (Binary, LargeBinary | BinaryView)
276277
)
277278
}
278279

@@ -1729,6 +1730,176 @@ mod tests {
17291730
}
17301731
}
17311732

1733+
#[test]
1734+
fn test_byte_representation_cast_preserves_values_and_ordering() -> Result<()> {
1735+
use arrow::array::{
1736+
BinaryArray, BinaryViewArray, LargeBinaryArray, StringViewArray,
1737+
};
1738+
use arrow::compute::SortOptions;
1739+
use datafusion_expr_common::sort_properties::SortProperties;
1740+
1741+
// Cover nulls, empty values, inline/long views, Unicode, and non-UTF8 bytes.
1742+
let strings = vec![
1743+
None,
1744+
Some(""),
1745+
Some("a"),
1746+
Some("a longer shared string"),
1747+
Some("a longer shared string"),
1748+
Some("🦀"),
1749+
];
1750+
let bytes: Vec<Option<&[u8]>> = vec![
1751+
None,
1752+
Some(b""),
1753+
Some(b"\0"),
1754+
Some(b"a longer shared byte string"),
1755+
Some(b"a longer shared byte string"),
1756+
Some(b"\xff"),
1757+
];
1758+
let binary: ArrayRef = Arc::new(BinaryArray::from(bytes.clone()));
1759+
let cases: [(ArrayRef, ArrayRef); 3] = [
1760+
(
1761+
Arc::new(StringArray::from(strings.clone())),
1762+
Arc::new(StringViewArray::from(strings)),
1763+
),
1764+
(
1765+
Arc::clone(&binary),
1766+
Arc::new(LargeBinaryArray::from(bytes.clone())),
1767+
),
1768+
(binary, Arc::new(BinaryViewArray::from(bytes))),
1769+
];
1770+
for (input, expected) in cases {
1771+
let schema = Arc::new(Schema::new(vec![Field::new(
1772+
"a",
1773+
input.data_type().clone(),
1774+
true,
1775+
)]));
1776+
let expr =
1777+
CastExpr::new(col("a", &schema)?, expected.data_type().clone(), None);
1778+
assert!(expr.is_bigger_cast(input.data_type()));
1779+
for descending in [false, true] {
1780+
for nulls_first in [false, true] {
1781+
let child = ExprProperties::new_unknown()
1782+
.with_range(Interval::make_unbounded(input.data_type())?)
1783+
.with_order(SortProperties::Ordered(SortOptions {
1784+
descending,
1785+
nulls_first,
1786+
}))
1787+
.with_strictly_order_preserving(true);
1788+
let properties = expr.get_properties(std::slice::from_ref(&child))?;
1789+
assert_eq!(properties.sort_properties, child.sort_properties);
1790+
assert!(properties.strictly_order_preserving);
1791+
assert_eq!(properties.range.data_type(), *expected.data_type());
1792+
}
1793+
}
1794+
let batch = RecordBatch::try_new(schema, vec![input])?;
1795+
let actual = expr.evaluate(&batch)?.into_array(batch.num_rows())?;
1796+
assert_eq!(actual.as_ref(), expected.as_ref());
1797+
}
1798+
for (source, target) in [
1799+
(Utf8View, Utf8),
1800+
(LargeBinary, Binary),
1801+
(BinaryView, Binary),
1802+
] {
1803+
assert!(!CastExpr::check_bigger_cast(&target, &source));
1804+
}
1805+
Ok(())
1806+
}
1807+
1808+
#[test]
1809+
fn test_unsigned_to_signed_cast_preserves_values_and_ordering() {
1810+
use arrow::array::{UInt8Array, UInt16Array, UInt32Array};
1811+
use arrow::compute::SortOptions;
1812+
use datafusion_expr_common::sort_properties::SortProperties;
1813+
1814+
let inputs: [(ArrayRef, Vec<DataType>, i64); 3] = [
1815+
(
1816+
Arc::new(UInt8Array::from(vec![
1817+
None,
1818+
Some(0),
1819+
Some(1),
1820+
Some(u8::MAX),
1821+
])),
1822+
vec![Int16, Int32, Int64],
1823+
i64::from(u8::MAX),
1824+
),
1825+
(
1826+
Arc::new(UInt16Array::from(vec![
1827+
None,
1828+
Some(0),
1829+
Some(1),
1830+
Some(u16::MAX),
1831+
])),
1832+
vec![Int32, Int64],
1833+
i64::from(u16::MAX),
1834+
),
1835+
(
1836+
Arc::new(UInt32Array::from(vec![
1837+
None,
1838+
Some(0),
1839+
Some(1),
1840+
Some(u32::MAX),
1841+
])),
1842+
vec![Int64],
1843+
i64::from(u32::MAX),
1844+
),
1845+
];
1846+
for (input, target_types, max) in inputs {
1847+
let schema = Arc::new(Schema::new(vec![Field::new(
1848+
"a",
1849+
input.data_type().clone(),
1850+
true,
1851+
)]));
1852+
let batch =
1853+
RecordBatch::try_new(Arc::clone(&schema), vec![Arc::clone(&input)])
1854+
.expect("valid input batch");
1855+
for target_type in target_types {
1856+
let expr =
1857+
CastExpr::new(col("a", &schema).unwrap(), target_type.clone(), None);
1858+
let actual = expr
1859+
.evaluate(&batch)
1860+
.unwrap()
1861+
.into_array(batch.num_rows())
1862+
.unwrap();
1863+
for (index, value) in
1864+
[None, Some(0), Some(1), Some(max)].into_iter().enumerate()
1865+
{
1866+
let expected = match target_type {
1867+
Int16 => {
1868+
ScalarValue::Int16(value.map(|v| i16::try_from(v).unwrap()))
1869+
}
1870+
Int32 => {
1871+
ScalarValue::Int32(value.map(|v| i32::try_from(v).unwrap()))
1872+
}
1873+
Int64 => ScalarValue::Int64(value),
1874+
_ => unreachable!(),
1875+
};
1876+
assert_eq!(
1877+
ScalarValue::try_from_array(&actual, index).unwrap(),
1878+
expected
1879+
);
1880+
}
1881+
for descending in [false, true] {
1882+
for nulls_first in [false, true] {
1883+
let child = ExprProperties::new_unknown()
1884+
.with_range(
1885+
Interval::make_unbounded(input.data_type()).unwrap(),
1886+
)
1887+
.with_order(SortProperties::Ordered(SortOptions {
1888+
descending,
1889+
nulls_first,
1890+
}))
1891+
.with_strictly_order_preserving(true);
1892+
let properties =
1893+
expr.get_properties(std::slice::from_ref(&child)).unwrap();
1894+
assert_eq!(properties.sort_properties, child.sort_properties);
1895+
assert!(properties.strictly_order_preserving);
1896+
assert_eq!(properties.range.data_type(), target_type);
1897+
}
1898+
}
1899+
}
1900+
}
1901+
}
1902+
17321903
#[test]
17331904
fn test_temporal_cast_ordering() {
17341905
use TimeUnit::*;
@@ -1913,10 +2084,15 @@ mod tests {
19132084
assert!(!CastExpr::check_bigger_cast(&Float64, &Int64));
19142085
assert!(!CastExpr::check_bigger_cast(&Float64, &UInt64));
19152086

1916-
// Signed <-> Unsigned conversions should return false (not order-preserving due to negative values)
2087+
// Signed-to-unsigned and unsigned-to-signed casts whose target cannot
2088+
// represent the entire source range are not lossless for all values.
19172089
assert!(!CastExpr::check_bigger_cast(&UInt16, &Int8));
19182090
assert!(!CastExpr::check_bigger_cast(&UInt32, &Int16));
1919-
assert!(!CastExpr::check_bigger_cast(&Int16, &UInt8));
2091+
assert!(!CastExpr::check_bigger_cast(&Int8, &UInt8));
2092+
assert!(!CastExpr::check_bigger_cast(&Int16, &UInt16));
2093+
assert!(!CastExpr::check_bigger_cast(&Int32, &UInt32));
2094+
assert!(!CastExpr::check_bigger_cast(&Int64, &UInt64));
2095+
assert!(!CastExpr::check_bigger_cast(&Int8, &UInt16));
19202096
}
19212097
}
19222098

‎datafusion/physical-expr/src/projection.rs‎

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2493,6 +2493,81 @@ pub(crate) mod tests {
24932493
}
24942494
}
24952495

2496+
#[test]
2497+
fn test_project_statistics_lossless_cast() {
2498+
use Precision::{Absent, Exact, Inexact};
2499+
2500+
for (lower, upper, targets) in [
2501+
(
2502+
ScalarValue::UInt8(Some(0)),
2503+
ScalarValue::UInt8(Some(u8::MAX)),
2504+
vec![DataType::Int16, DataType::Int32, DataType::Int64],
2505+
),
2506+
(
2507+
ScalarValue::UInt16(Some(0)),
2508+
ScalarValue::UInt16(Some(u16::MAX)),
2509+
vec![DataType::Int32, DataType::Int64],
2510+
),
2511+
(
2512+
ScalarValue::UInt32(Some(0)),
2513+
ScalarValue::UInt32(Some(u32::MAX)),
2514+
vec![DataType::Int64],
2515+
),
2516+
(
2517+
ScalarValue::Utf8(Some(String::new())),
2518+
ScalarValue::Utf8(Some("🦀".to_string())),
2519+
vec![DataType::Utf8View],
2520+
),
2521+
(
2522+
ScalarValue::Binary(Some(vec![])),
2523+
ScalarValue::Binary(Some(vec![0xff])),
2524+
vec![DataType::LargeBinary, DataType::BinaryView],
2525+
),
2526+
] {
2527+
for target in targets {
2528+
// A globally safe cast does not need two exact extrema. Preserve
2529+
// whichever bounds are available without upgrading their precision.
2530+
for (min_value, max_value) in [
2531+
(Exact(lower.clone()), Exact(upper.clone())),
2532+
(Exact(lower.clone()), Absent),
2533+
(Absent, Exact(upper.clone())),
2534+
(Inexact(lower.clone()), Inexact(upper.clone())),
2535+
(Inexact(lower.clone()), Absent),
2536+
(Absent, Inexact(upper.clone())),
2537+
] {
2538+
let input = ColumnStatistics {
2539+
min_value: min_value.clone(),
2540+
max_value: max_value.clone(),
2541+
null_count: Exact(1),
2542+
distinct_count: Inexact(3),
2543+
sum_value: Absent,
2544+
byte_size: Absent,
2545+
};
2546+
let expr = CastExpr::new(
2547+
Arc::new(Column::new("a", 0)),
2548+
target.clone(),
2549+
None,
2550+
);
2551+
let output =
2552+
project_column_statistics_through_expr(&expr, &[input], None);
2553+
assert_eq!(
2554+
output,
2555+
ColumnStatistics {
2556+
min_value: min_value.cast_to(&target).unwrap(),
2557+
max_value: max_value.cast_to(&target).unwrap(),
2558+
null_count: Exact(1),
2559+
distinct_count: Inexact(3),
2560+
sum_value: Absent,
2561+
byte_size: Absent,
2562+
},
2563+
"{} -> {target}, {min_value:?}..{max_value:?}",
2564+
lower.data_type()
2565+
);
2566+
}
2567+
}
2568+
}
2569+
}
2570+
24962571
#[test]
24972572
fn test_project_statistics_narrowing_cast_requires_safe_bounds() {
24982573
let schema = Schema::new(vec![Field::new("a", DataType::Int32, true)]);

0 commit comments

Comments
 (0)