Skip to content

Commit 2c715e8

Browse files
committed
Widen dictionary codes in Dict::fill_null
When the fill value is in the dictionary values but its index does not fit the codes type, the Dict fill_null kernel declined and the generic path decoded the whole array. Since the kernel only rewrites the null codes, cast the codes to the smallest unsigned type that holds the fill index and keep the result dictionary-encoded instead. Signed-off-by: Vincent <[email protected]>
1 parent dcb4f50 commit 2c715e8

1 file changed

Lines changed: 50 additions & 9 deletions

File tree

‎vortex-array/src/arrays/dict/compute/fill_null.rs‎

Lines changed: 50 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,9 @@ use crate::arrays::bool::BoolArrayExt;
1616
use crate::arrays::dict::DictArrayExt;
1717
use crate::arrays::dict::DictArraySlotsExt;
1818
use crate::builtins::ArrayBuiltins;
19+
use crate::dtype::DType;
20+
use crate::dtype::Nullability;
21+
use crate::dtype::PType;
1922
use crate::scalar::Scalar;
2023
use crate::scalar_fn::fns::fill_null::FillNullKernel;
2124
use crate::scalar_fn::fns::operators::Operator;
@@ -54,19 +57,22 @@ impl FillNullKernel for Dict {
5457

5558
// A dictionary can hold more values than its codes type can index, so the index of an
5659
// unreferenced fill value can exceed the range of the codes. Non-nullable codes need no
57-
// fill code. For nullable codes, decline in that case, so that the generic path fills the
58-
// decoded array. The first match has the smallest index, so no later match fits either.
60+
// fill code. For nullable codes, widen the codes to the smallest unsigned type that can
61+
// hold the fill index in that case. Codes are never negative, so the cast keeps them.
5962
let codes = array.codes();
6063
let codes = if codes.dtype().is_nullable() {
61-
let codes_dtype = codes.dtype().as_nonnullable();
6264
let fill_index = existing_fill_value_index as u64;
63-
if fill_index > codes_dtype.as_ptype().max_value_as_u64() {
64-
return Ok(None);
65-
}
65+
let codes = if fill_index > codes.dtype().as_ptype().max_value_as_u64() {
66+
let wide_ptype = PType::min_unsigned_ptype_for_value(fill_index);
67+
codes
68+
.clone()
69+
.cast(DType::Primitive(wide_ptype, Nullability::Nullable))?
70+
} else {
71+
codes.clone()
72+
};
6673

67-
codes
68-
.clone()
69-
.fill_null(Scalar::from(fill_index).cast(&codes_dtype)?)?
74+
let codes_dtype = codes.dtype().as_nonnullable();
75+
codes.fill_null(Scalar::from(fill_index).cast(&codes_dtype)?)?
7076
} else {
7177
codes.clone()
7278
};
@@ -99,12 +105,16 @@ mod tests {
99105

100106
use crate::IntoArray;
101107
use crate::VortexSessionExecute;
108+
use crate::arrays::Dict;
102109
use crate::arrays::DictArray;
103110
use crate::arrays::PrimitiveArray;
111+
use crate::arrays::dict::DictArraySlotsExt;
104112
use crate::assert_arrays_eq;
105113
use crate::builtins::ArrayBuiltins;
106114
use crate::dtype::Nullability;
115+
use crate::dtype::PType;
107116
use crate::scalar::Scalar;
117+
use crate::scalar_fn::fns::fill_null::FillNullKernel;
108118
use crate::validity::Validity;
109119

110120
static SESSION: LazyLock<VortexSession> = LazyLock::new(VortexSession::empty);
@@ -131,6 +141,37 @@ mod tests {
131141
Ok(())
132142
}
133143

144+
#[rstest]
145+
#[case::u8_to_u16(PrimitiveArray::from_option_iter([Some(0u8), None]), 256, PType::U16)]
146+
#[case::u8_to_u16_truncation(PrimitiveArray::from_option_iter([Some(0u8), None]), 300, PType::U16)]
147+
#[case::i8_to_u8(PrimitiveArray::from_option_iter([Some(0i8), None]), 128, PType::U8)]
148+
#[case::i8_to_u16(PrimitiveArray::from_option_iter([Some(0i8), None]), 256, PType::U16)]
149+
#[case::u16_to_u32(PrimitiveArray::from_option_iter([Some(0u16), None]), 65_536, PType::U32)]
150+
fn fill_unreferenced_value_widens_codes(
151+
#[case] codes: PrimitiveArray,
152+
#[case] fill_value: i32,
153+
#[case] wide_ptype: PType,
154+
) -> VortexResult<()> {
155+
let mut ctx = SESSION.create_execution_ctx();
156+
let dict = DictArray::try_new(
157+
codes.into_array(),
158+
PrimitiveArray::from_iter(0..=fill_value).into_array(),
159+
)?;
160+
161+
let filled = <Dict as FillNullKernel>::fill_null(
162+
dict.as_view(),
163+
&Scalar::from(fill_value),
164+
&mut ctx,
165+
)?
166+
.vortex_expect("kernel should fill the dictionary codes");
167+
let filled_dict = filled.as_::<Dict>();
168+
assert_eq!(filled_dict.codes().dtype().as_ptype(), wide_ptype);
169+
assert!(!filled_dict.codes().dtype().is_nullable());
170+
assert_arrays_eq!(filled, PrimitiveArray::from_iter([0, fill_value]), &mut ctx);
171+
172+
Ok(())
173+
}
174+
134175
#[test]
135176
fn non_nullable_codes_fill_unreferenced_value() -> VortexResult<()> {
136177
let mut ctx = SESSION.create_execution_ctx();

0 commit comments

Comments
 (0)