Skip to content

Commit 87589b7

Browse files
committed
fix(decimal-byte-parts): guard the nullability cast rule
The DecimalByteParts cast rule cast the most significant part for every target nullability, even when `Validity::trivially_cast_nullability` could not prove the cast safe. The rule now declines in that case, and a new `CastKernel` handles it at execution with `cast_nullability`. Casts that change the precision or the scale stay unsupported. The cast tests now use a session that registers the crate, so they reach the new kernel. Signed-off-by: Connor Tsui <[email protected]>
1 parent 3974ee0 commit 87589b7

2 files changed

Lines changed: 219 additions & 10 deletions

File tree

‎encodings/decimal-byte-parts/src/decimal_byte_parts/compute/cast.rs‎

Lines changed: 212 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -3,34 +3,72 @@
33

44
use vortex_array::ArrayRef;
55
use vortex_array::ArrayView;
6+
use vortex_array::ExecutionCtx;
67
use vortex_array::IntoArray;
78
use vortex_array::builtins::ArrayBuiltins;
89
use vortex_array::dtype::DType;
10+
use vortex_array::dtype::Nullability;
11+
use vortex_array::scalar_fn::fns::cast::CastKernel;
912
use vortex_array::scalar_fn::fns::cast::CastReduce;
1013
use vortex_error::VortexResult;
1114

1215
use crate::DecimalByteParts;
1316
use crate::decimal_byte_parts::DecimalBytePartsArrayExt;
1417
use crate::decimal_byte_parts::DecimalBytePartsArraySlotsExt;
1518

19+
/// Rebuilds `array` over its MSP cast to `nullability`.
20+
///
21+
/// The MSP carries the validity of a DBP array, so the cast of the MSP sets the validity of the
22+
/// result. The lower parts are non-nullable and stay as they are.
23+
fn with_msp_nullability(
24+
array: ArrayView<'_, DecimalByteParts>,
25+
nullability: Nullability,
26+
) -> VortexResult<ArrayRef> {
27+
let msp = array.msp();
28+
let new_msp = msp.cast(msp.dtype().with_nullability(nullability))?;
29+
30+
Ok(array.with_msp(new_msp)?.into_array())
31+
}
32+
1633
impl CastReduce for DecimalByteParts {
1734
fn cast(array: ArrayView<'_, Self>, dtype: &DType) -> VortexResult<Option<ArrayRef>> {
18-
// Check if this is just a nullability change.
1935
// TODO(mk): Support non-nullability changes as well, e.g. precision.
2036
if !dtype.eq_ignore_nullability(array.dtype()) {
2137
return Ok(None);
2238
}
23-
// DecimalBytePartsArray can only have Decimal dtype.
24-
let DType::Decimal(_, target_nullability) = dtype else {
39+
40+
let nullability = dtype.nullability();
41+
if array
42+
.validity()?
43+
.trivially_cast_nullability(nullability, array.len())?
44+
.is_none()
45+
{
46+
return Ok(None);
47+
}
48+
49+
with_msp_nullability(array, nullability).map(Some)
50+
}
51+
}
52+
53+
impl CastKernel for DecimalByteParts {
54+
fn cast(
55+
array: ArrayView<'_, Self>,
56+
dtype: &DType,
57+
ctx: &mut ExecutionCtx,
58+
) -> VortexResult<Option<ArrayRef>> {
59+
if !dtype.eq_ignore_nullability(array.dtype()) {
2560
return Ok(None);
26-
};
61+
}
2762

28-
// Cast the msp array to handle nullability change
29-
let new_msp = array
30-
.msp()
31-
.cast(array.msp().dtype().with_nullability(*target_nullability))?;
63+
// The call to `cast_nullability` fails when the target is non-nullable and a row is null.
64+
// The `Validity` that it returns is discarded, because the MSP stores the validity of a
65+
// DBP array and the cast of the MSP in `with_msp_nullability` sets the new validity.
66+
let nullability = dtype.nullability();
67+
array
68+
.validity()?
69+
.cast_nullability(nullability, array.len(), ctx)?;
3270

33-
array.with_msp(new_msp).map(|a| Some(a.into_array()))
71+
with_msp_nullability(array, nullability).map(Some)
3472
}
3573
}
3674

@@ -42,6 +80,7 @@ mod tests {
4280
use vortex_array::Canonical;
4381
use vortex_array::IntoArray;
4482
use vortex_array::VortexSessionExecute;
83+
use vortex_array::arrays::BoolArray;
4584
use vortex_array::arrays::DecimalArray;
4685
use vortex_array::arrays::PrimitiveArray;
4786
use vortex_array::assert_arrays_eq;
@@ -50,14 +89,177 @@ mod tests {
5089
use vortex_array::dtype::DType;
5190
use vortex_array::dtype::DecimalDType;
5291
use vortex_array::dtype::Nullability;
92+
use vortex_array::scalar_fn::fns::cast::CastKernel;
93+
use vortex_array::scalar_fn::fns::cast::CastReduce;
5394
use vortex_array::validity::Validity;
5495
use vortex_buffer::buffer;
96+
use vortex_error::VortexResult;
97+
use vortex_error::vortex_err;
5598
use vortex_session::VortexSession;
5699

57100
use crate::DecimalByteParts;
58101
use crate::DecimalBytePartsArray;
59102

60-
static SESSION: LazyLock<VortexSession> = LazyLock::new(VortexSession::empty);
103+
static SESSION: LazyLock<VortexSession> = LazyLock::new(|| {
104+
let session = VortexSession::empty();
105+
crate::initialize(&session);
106+
session
107+
});
108+
109+
/// A DBP array at `DecimalDType::new(10, 2)` over the `i32` MSP `[100, 200, 300]` with
110+
/// `validity`.
111+
fn fixture(validity: Validity) -> VortexResult<DecimalBytePartsArray> {
112+
DecimalByteParts::try_new(
113+
PrimitiveArray::new(buffer![100i32, 200, 300], validity).into_array(),
114+
DecimalDType::new(10, 2),
115+
)
116+
}
117+
118+
#[rstest]
119+
#[case::with_nulls(Validity::Array(BoolArray::from_iter([true, false, true]).into_array()))]
120+
#[case::without_nulls(Validity::Array(BoolArray::from_iter([true, true, true]).into_array()))]
121+
#[case::only_nulls(Validity::Array(BoolArray::from_iter([false, false, false]).into_array()))]
122+
fn test_cast_rule_declines_validity_array(#[case] validity: Validity) -> VortexResult<()> {
123+
let array = fixture(validity)?;
124+
assert!(matches!(array.validity()?, Validity::Array(_)));
125+
126+
let dtype = DType::Decimal(DecimalDType::new(10, 2), Nullability::NonNullable);
127+
assert!(<DecimalByteParts as CastReduce>::cast(array.as_view(), &dtype)?.is_none());
128+
129+
Ok(())
130+
}
131+
132+
#[rstest]
133+
#[case::with_nulls(Validity::Array(BoolArray::from_iter([true, false, true]).into_array()))]
134+
#[case::only_nulls(Validity::Array(BoolArray::from_iter([false, false, false]).into_array()))]
135+
fn test_cast_kernel_rejects_nulls(#[case] validity: Validity) -> VortexResult<()> {
136+
const NULL_ROW_ERROR: &str = "Cannot cast array with invalid values to non-nullable type.";
137+
138+
let array = fixture(validity)?;
139+
assert!(matches!(array.validity()?, Validity::Array(_)));
140+
141+
let mut ctx = SESSION.create_execution_ctx();
142+
let dtype = DType::Decimal(DecimalDType::new(10, 2), Nullability::NonNullable);
143+
let kernel_error =
144+
<DecimalByteParts as CastKernel>::cast(array.as_view(), &dtype, &mut ctx)
145+
.expect_err("the kernel rejects a cast to non-nullable when a row is null");
146+
assert!(kernel_error.to_string().contains(NULL_ROW_ERROR));
147+
148+
let casted = array.into_array().cast(dtype)?;
149+
let execute_error = casted
150+
.execute::<Canonical>(&mut ctx)
151+
.expect_err("execution rejects a cast to non-nullable when a row is null");
152+
assert!(execute_error.to_string().contains(NULL_ROW_ERROR));
153+
154+
Ok(())
155+
}
156+
157+
#[test]
158+
fn test_cast_kernel_without_nulls() -> VortexResult<()> {
159+
let validity = Validity::Array(BoolArray::from_iter([true, true, true]).into_array());
160+
let array = fixture(validity)?;
161+
assert!(matches!(array.validity()?, Validity::Array(_)));
162+
163+
let mut ctx = SESSION.create_execution_ctx();
164+
let decimal_dtype = DecimalDType::new(10, 2);
165+
let dtype = DType::Decimal(decimal_dtype, Nullability::NonNullable);
166+
let kernel_result =
167+
<DecimalByteParts as CastKernel>::cast(array.as_view(), &dtype, &mut ctx)?.ok_or_else(
168+
|| vortex_err!("the kernel must handle a cast that changes only the nullability"),
169+
)?;
170+
assert!(kernel_result.is::<DecimalByteParts>());
171+
assert_eq!(kernel_result.dtype(), &dtype);
172+
173+
let casted = array.into_array().cast(dtype)?;
174+
assert_arrays_eq!(
175+
casted,
176+
DecimalArray::new(
177+
buffer![100i32, 200, 300],
178+
decimal_dtype,
179+
Validity::NonNullable
180+
),
181+
&mut ctx
182+
);
183+
184+
Ok(())
185+
}
186+
187+
#[test]
188+
fn test_cast_rule_rejects_all_invalid() -> VortexResult<()> {
189+
let array = fixture(Validity::AllInvalid)?;
190+
191+
let dtype = DType::Decimal(DecimalDType::new(10, 2), Nullability::NonNullable);
192+
let error = array
193+
.into_array()
194+
.cast(dtype)
195+
.expect_err("the rule rejects a cast of all nulls to non-nullable");
196+
assert!(
197+
error
198+
.to_string()
199+
.contains("Cannot cast AllInvalid to NonNullable")
200+
);
201+
202+
Ok(())
203+
}
204+
205+
#[rstest]
206+
#[case::nullable_all_valid_to_non_nullable(Validity::AllValid, Nullability::NonNullable)]
207+
#[case::non_nullable_to_nullable(Validity::NonNullable, Nullability::Nullable)]
208+
fn test_cast_rule_reduces(
209+
#[case] validity: Validity,
210+
#[case] target: Nullability,
211+
) -> VortexResult<()> {
212+
let decimal_dtype = DecimalDType::new(10, 2);
213+
let casted = fixture(validity)?
214+
.into_array()
215+
.cast(DType::Decimal(decimal_dtype, target))?;
216+
assert!(casted.is::<DecimalByteParts>());
217+
218+
let mut ctx = SESSION.create_execution_ctx();
219+
assert_arrays_eq!(
220+
casted,
221+
DecimalArray::new(
222+
buffer![100i32, 200, 300],
223+
decimal_dtype,
224+
Validity::from(target)
225+
),
226+
&mut ctx
227+
);
228+
229+
Ok(())
230+
}
231+
232+
#[test]
233+
fn test_cast_kernel_keeps_lower_parts() -> VortexResult<()> {
234+
let validity = Validity::Array(BoolArray::from_iter([true, true, true]).into_array());
235+
let msp = PrimitiveArray::new(buffer![1i64, 0, -1], validity).into_array();
236+
let lower_parts = vec![buffer![5u64, 6, 7].into_array()];
237+
let decimal_dtype = DecimalDType::new(38, 2);
238+
let array = DecimalByteParts::try_new_with_lower_parts(msp, lower_parts, decimal_dtype)?;
239+
assert!(matches!(array.validity()?, Validity::Array(_)));
240+
241+
let mut ctx = SESSION.create_execution_ctx();
242+
let dtype = DType::Decimal(decimal_dtype, Nullability::NonNullable);
243+
let kernel_result =
244+
<DecimalByteParts as CastKernel>::cast(array.as_view(), &dtype, &mut ctx)?.ok_or_else(
245+
|| vortex_err!("the kernel must handle a cast that changes only the nullability"),
246+
)?;
247+
assert!(kernel_result.is::<DecimalByteParts>());
248+
assert_eq!(kernel_result.dtype(), &dtype);
249+
250+
let casted = array.into_array().cast(dtype)?;
251+
assert_arrays_eq!(
252+
casted,
253+
DecimalArray::new(
254+
buffer![(1i128 << 64) + 5, 6, (-1i128 << 64) + 7],
255+
decimal_dtype,
256+
Validity::NonNullable
257+
),
258+
&mut ctx
259+
);
260+
261+
Ok(())
262+
}
61263

62264
#[test]
63265
fn test_cast_decimal_byte_parts_nullability() {

‎encodings/decimal-byte-parts/src/decimal_byte_parts/compute/kernel.rs‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,8 @@ use vortex_array::scalar_fn::fns::between::Between;
77
use vortex_array::scalar_fn::fns::between::BetweenExecuteAdaptor;
88
use vortex_array::scalar_fn::fns::binary::Binary;
99
use vortex_array::scalar_fn::fns::binary::CompareExecuteAdaptor;
10+
use vortex_array::scalar_fn::fns::cast::Cast;
11+
use vortex_array::scalar_fn::fns::cast::CastExecuteAdaptor;
1012
use vortex_session::VortexSession;
1113

1214
use crate::DecimalByteParts;
@@ -23,4 +25,9 @@ pub(crate) fn initialize(session: &VortexSession) {
2325
DecimalByteParts,
2426
BetweenExecuteAdaptor(DecimalByteParts),
2527
);
28+
kernels.register_execute_parent_kernel(
29+
Cast.id(),
30+
DecimalByteParts,
31+
CastExecuteAdaptor(DecimalByteParts),
32+
);
2633
}

0 commit comments

Comments
 (0)