Skip to content

Commit 999dd0b

Browse files
committed
Keep lossless primitive casts on dictionary values
Share the primitive cast kernel’s lossless conversion check with the dictionary rewrite so widening preserves the codes even when value coverage is unknown. Exercise the rewrites with the session optimizer now that constructors are lazy. Signed-off-by: blaginin <[email protected]>
1 parent b4ff723 commit 999dd0b

4 files changed

Lines changed: 60 additions & 43 deletions

File tree

‎vortex-array/src/arrays/dict/compute/cast.rs‎

Lines changed: 22 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -16,15 +16,20 @@ use crate::scalar_fn::fns::cast::CastReduce;
1616

1717
impl CastReduce for Dict {
1818
fn cast(array: ArrayView<'_, Dict>, dtype: &DType) -> VortexResult<Option<ArrayRef>> {
19-
// A value conversion can fail on an unreferenced dictionary entry. Only change outer
20-
// nullability unless every value is referenced; its validity is checked below.
21-
if !array.has_all_values_referenced() && array.dtype().as_nullable() != dtype.as_nullable()
22-
{
23-
return Ok(None);
19+
// Casting unused entries must not introduce errors that a cast over the rows would skip.
20+
if !array.has_all_values_referenced() {
21+
let safe_for_unused_values = match (array.dtype(), dtype) {
22+
(DType::Primitive(from, _), DType::Primitive(to, _)) => {
23+
from.casts_losslessly_to(*to)
24+
}
25+
(from, to) => from.as_nullable() == to.as_nullable(),
26+
};
27+
if !safe_for_unused_values {
28+
return Ok(None);
29+
}
2430
}
2531

26-
// Can have un-reference null values making the cast of values fail without a possible mask.
27-
// TODO(joe): optimize this, could look at accessible values and fill_null not those?
32+
// An unused null entry must not make a cast to a non-nullable dtype fail.
2833
if !dtype.is_nullable() && !array.values().validity()?.definitely_no_nulls() {
2934
return Ok(None);
3035
}
@@ -245,10 +250,11 @@ mod tests {
245250
fn test_cast_dict_after_slice_ignores_unreferenced_values() -> VortexResult<()> {
246251
let ctx = &mut SESSION.create_execution_ctx();
247252
let dict = dict_encode(&buffer![1000i64, 1, 1, 1].into_array(), ctx)?;
248-
let rows = dict.into_array().slice(1..4)?;
253+
let rows = dict.into_array().slice(1..4)?.optimize_ctx(&SESSION)?;
249254

250255
let casted = rows
251256
.cast(DType::Primitive(PType::I8, Nullability::NonNullable))?
257+
.optimize_ctx(&SESSION)?
252258
.execute::<Canonical>(ctx)?;
253259

254260
assert_arrays_eq!(casted, PrimitiveArray::from_iter([1i8, 1, 1]), ctx);
@@ -264,9 +270,13 @@ mod tests {
264270
)?
265271
.into_array();
266272

267-
let nullable = dict.cast(DType::Primitive(PType::I64, Nullability::Nullable))?;
273+
let nullable = dict
274+
.cast(DType::Primitive(PType::I64, Nullability::Nullable))?
275+
.optimize_ctx(&SESSION)?;
268276
assert!(nullable.is::<Dict>());
269-
let non_nullable = nullable.cast(DType::Primitive(PType::I64, Nullability::NonNullable))?;
277+
let non_nullable = nullable
278+
.cast(DType::Primitive(PType::I64, Nullability::NonNullable))?
279+
.optimize_ctx(&SESSION)?;
270280
assert!(non_nullable.is::<Dict>());
271281

272282
assert_arrays_eq!(non_nullable, PrimitiveArray::from_iter([1i64, 1, 1]), ctx);
@@ -279,7 +289,8 @@ mod tests {
279289
let dict = dict_encode(&buffer![1000i64, 1, 1, 1].into_array(), ctx)?;
280290
let casted = dict
281291
.into_array()
282-
.cast(DType::Primitive(PType::I8, Nullability::NonNullable))?;
292+
.cast(DType::Primitive(PType::I8, Nullability::NonNullable))?
293+
.optimize_ctx(&SESSION)?;
283294

284295
assert!(casted.is::<Dict>());
285296
let error = casted.execute::<Canonical>(ctx).unwrap_err();

‎vortex-array/src/arrays/dict/compute/rules.rs‎

Lines changed: 11 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -464,16 +464,20 @@ mod tests {
464464
buffer![7i32, 8].into_array(),
465465
)?
466466
.into_array()
467-
.slice(1..4)?;
467+
.slice(1..4)?
468+
.optimize_ctx(&SESSION)?;
468469
let divisor = DictArray::try_new(
469470
buffer![0u8, 1, 1, 1].into_array(),
470471
buffer![0i32, 2].into_array(),
471472
)?
472473
.into_array()
473-
.slice(1..4)?;
474+
.slice(1..4)?
475+
.optimize_ctx(&SESSION)?;
474476
let expr = Binary::try_new(dividend, divisor, Operator::Div)?.into_array();
475477

476-
let quotient = expr.optimize()?.execute::<Canonical>(&mut ctx)?;
478+
let quotient = expr
479+
.optimize_ctx(&SESSION)?
480+
.execute::<Canonical>(&mut ctx)?;
477481

478482
assert_arrays_eq!(quotient, PrimitiveArray::from_iter([4i32, 4, 4]), &mut ctx);
479483
Ok(())
@@ -496,7 +500,9 @@ mod tests {
496500
.into_array();
497501
let rhs = DictArray::try_new(codes, buffer![2i32, 9].into_array())?.into_array();
498502

499-
let optimized = Binary::try_new(lhs, rhs, operator)?.into_array().optimize()?;
503+
let optimized = Binary::try_new(lhs, rhs, operator)?
504+
.into_array()
505+
.optimize_ctx(&SESSION)?;
500506
assert!(optimized.is::<Dict>());
501507
let mut ctx = SESSION.create_execution_ctx();
502508
let result = optimized.execute::<Canonical>(&mut ctx)?;
@@ -513,7 +519,7 @@ mod tests {
513519
let expr = Binary::try_new(dividend, divisor, Operator::Div)?.into_array();
514520

515521
let error = expr
516-
.optimize()?
522+
.optimize_ctx(&SESSION)?
517523
.execute::<Canonical>(&mut SESSION.create_execution_ctx())
518524
.unwrap_err();
519525
assert!(error.to_string().contains("division by zero"));

‎vortex-array/src/arrays/primitive/compute/cast.rs‎

Lines changed: 1 addition & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -519,35 +519,9 @@ where
519519
)
520520
};
521521

522-
// Returns `true` if every value of `from` is representable in `to` without loss.
523-
fn casts_losslessly_to(from: PType, to: PType) -> bool {
524-
if from == to {
525-
return true;
526-
}
527-
if (from.is_unsigned_int() && to.is_unsigned_int())
528-
|| (from.is_signed_int() && to.is_signed_int())
529-
|| (from.is_float() && to.is_float())
530-
{
531-
return from.byte_width() <= to.byte_width();
532-
}
533-
if from.is_unsigned_int() && to.is_signed_int() {
534-
return from.byte_width() < to.byte_width();
535-
}
536-
if from.is_int() && to.is_float() {
537-
let minimum_float_width = match from.byte_width() {
538-
1 => 2,
539-
2 => 4,
540-
4 => 8,
541-
_ => return false,
542-
};
543-
return to.byte_width() >= minimum_float_width;
544-
}
545-
false
546-
}
547-
548522
// Skip the fallible kernel when type widening or (cached) min/max prove every value fits.
549523
let target_dtype = DType::Primitive(T::PTYPE, Nullability::NonNullable);
550-
let infallible = casts_losslessly_to(F::PTYPE, T::PTYPE)
524+
let infallible = F::PTYPE.casts_losslessly_to(T::PTYPE)
551525
|| cached_values_fit_in(array, &target_dtype).unwrap_or(false);
552526

553527
let len = array.len();

‎vortex-array/src/dtype/ptype.rs‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -827,6 +827,32 @@ impl PType {
827827
self.byte_width() * 8
828828
}
829829

830+
/// Returns whether every value of this type is representable in `to` without loss.
831+
pub(crate) fn casts_losslessly_to(self, to: Self) -> bool {
832+
if self == to {
833+
return true;
834+
}
835+
if (self.is_unsigned_int() && to.is_unsigned_int())
836+
|| (self.is_signed_int() && to.is_signed_int())
837+
|| (self.is_float() && to.is_float())
838+
{
839+
return self.byte_width() <= to.byte_width();
840+
}
841+
if self.is_unsigned_int() && to.is_signed_int() {
842+
return self.byte_width() < to.byte_width();
843+
}
844+
if self.is_int() && to.is_float() {
845+
let minimum_float_width = match self.byte_width() {
846+
1 => 2,
847+
2 => 4,
848+
4 => 8,
849+
_ => return false,
850+
};
851+
return to.byte_width() >= minimum_float_width;
852+
}
853+
false
854+
}
855+
830856
/// Returns the maximum value of this PType if it is an integer type
831857
/// Returns `u64::MAX` if the value is too large to fit in a `u64`
832858
#[inline]

0 commit comments

Comments
 (0)