Skip to content

Commit dcb4f50

Browse files
authored
Avoid truncating dictionary fill codes (#10745)
## Summary Fixes #10623. `Dict::fill_null` converted the index of the matching dictionary value into the codes type with an unchecked `as` cast. A dictionary can hold more values than its codes type can index, so an unreferenced fill value could truncate to the wrong code (with `u8` codes, filling with value 300 returned value 44) or panic for signed codes. ## Changes The kernel now checks the index against the range of the codes type. When the index does not fit, the kernel declines and the generic `fill_null` path fills the decoded array, which is slower but correct. Non-nullable codes need no fill code, so they keep the fast path. The new tests cover `u8` and `i8` codes at and past their maximum, and non-nullable codes over nullable values. The three cases past the maximum fail without the fix. The `Narrow` work (#10733) depends on this fix, because a rule there makes narrow dictionary codes common. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Signed-off-by: Connor Tsui <[email protected]>
1 parent 9678f13 commit dcb4f50

1 file changed

Lines changed: 56 additions & 20 deletions

File tree

‎vortex-array/src/arrays/dict/compute/fill_null.rs‎

Lines changed: 56 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -16,9 +16,7 @@ use crate::arrays::bool::BoolArrayExt;
1616
use crate::arrays::dict::DictArrayExt;
1717
use crate::arrays::dict::DictArraySlotsExt;
1818
use crate::builtins::ArrayBuiltins;
19-
use crate::match_each_integer_ptype;
2019
use crate::scalar::Scalar;
21-
use crate::scalar::ScalarValue;
2220
use crate::scalar_fn::fns::fill_null::FillNullKernel;
2321
use crate::scalar_fn::fns::operators::Operator;
2422

@@ -54,26 +52,27 @@ impl FillNullKernel for Dict {
5452
));
5553
};
5654

57-
// Now we rewrite the nullable codes to point at the fill value.
55+
// A dictionary can hold more values than its codes type can index, so the index of an
56+
// unreferenced fill value can exceed the range of the codes. Non-nullable codes need no
57+
// fill code. For nullable codes, decline in that case, so that the generic path fills the
58+
// decoded array. The first match has the smallest index, so no later match fits either.
5859
let codes = array.codes();
60+
let codes = if codes.dtype().is_nullable() {
61+
let codes_dtype = codes.dtype().as_nonnullable();
62+
let fill_index = existing_fill_value_index as u64;
63+
if fill_index > codes_dtype.as_ptype().max_value_as_u64() {
64+
return Ok(None);
65+
}
5966

60-
// Cast the index to the correct unsigned integer type matching the codes' ptype.
61-
let codes_ptype = codes.dtype().as_ptype();
62-
63-
#[expect(
64-
clippy::cast_possible_truncation,
65-
reason = "The existing index must be representable by the existing ptype"
66-
)]
67-
let fill_scalar_value = match_each_integer_ptype!(codes_ptype, |P| {
68-
ScalarValue::from(existing_fill_value_index as P)
69-
});
70-
71-
// Fill nulls in both the codes and the values. Note that the precondition of this function
72-
// states that the fill value is non-null, so we do not have to worry about the nullability.
73-
let codes = codes.clone().fill_null(Scalar::try_new(
74-
codes.dtype().as_nonnullable(),
75-
Some(fill_scalar_value),
76-
)?)?;
67+
codes
68+
.clone()
69+
.fill_null(Scalar::from(fill_index).cast(&codes_dtype)?)?
70+
} else {
71+
codes.clone()
72+
};
73+
74+
// Fill the nulls of the values. Note that the precondition of this function states that the
75+
// fill value is non-null, so we do not have to worry about the nullability.
7776
let values = array.values().clone().fill_null(fill_value.clone())?;
7877

7978
// SAFETY: invariants are still satisfied after patching nulls.
@@ -91,9 +90,11 @@ impl FillNullKernel for Dict {
9190
mod tests {
9291
use std::sync::LazyLock;
9392

93+
use rstest::rstest;
9494
use vortex_buffer::BitBuffer;
9595
use vortex_buffer::buffer;
9696
use vortex_error::VortexExpect;
97+
use vortex_error::VortexResult;
9798
use vortex_session::VortexSession;
9899

99100
use crate::IntoArray;
@@ -108,6 +109,41 @@ mod tests {
108109

109110
static SESSION: LazyLock<VortexSession> = LazyLock::new(VortexSession::empty);
110111

112+
#[rstest]
113+
#[case::u8_max(PrimitiveArray::from_option_iter([Some(0u8), None]), 255)]
114+
#[case::u8_overflow(PrimitiveArray::from_option_iter([Some(0u8), None]), 256)]
115+
#[case::u8_truncation(PrimitiveArray::from_option_iter([Some(0u8), None]), 300)]
116+
#[case::i8_max(PrimitiveArray::from_option_iter([Some(0i8), None]), 127)]
117+
#[case::i8_overflow(PrimitiveArray::from_option_iter([Some(0i8), None]), 128)]
118+
fn fill_unreferenced_value(
119+
#[case] codes: PrimitiveArray,
120+
#[case] fill_value: i32,
121+
) -> VortexResult<()> {
122+
let mut ctx = SESSION.create_execution_ctx();
123+
let dict = DictArray::try_new(
124+
codes.into_array(),
125+
PrimitiveArray::from_iter(0..=fill_value).into_array(),
126+
)?;
127+
128+
let filled = dict.into_array().fill_null(fill_value)?;
129+
assert_arrays_eq!(filled, PrimitiveArray::from_iter([0, fill_value]), &mut ctx);
130+
131+
Ok(())
132+
}
133+
134+
#[test]
135+
fn non_nullable_codes_fill_unreferenced_value() -> VortexResult<()> {
136+
let mut ctx = SESSION.create_execution_ctx();
137+
let values =
138+
PrimitiveArray::from_option_iter((0..=300).map(|value| (value != 1).then_some(value)));
139+
let dict = DictArray::try_new(buffer![0u8, 1].into_array(), values.into_array())?;
140+
141+
let filled = dict.into_array().fill_null(300)?;
142+
assert_arrays_eq!(filled, PrimitiveArray::from_iter([0, 300]), &mut ctx);
143+
144+
Ok(())
145+
}
146+
111147
#[test]
112148
fn nullable_codes_fill_in_values() {
113149
let mut ctx = SESSION.create_execution_ctx();

0 commit comments

Comments
 (0)