@@ -16,6 +16,9 @@ use crate::arrays::bool::BoolArrayExt;
1616use crate :: arrays:: dict:: DictArrayExt ;
1717use crate :: arrays:: dict:: DictArraySlotsExt ;
1818use crate :: builtins:: ArrayBuiltins ;
19+ use crate :: dtype:: DType ;
20+ use crate :: dtype:: Nullability ;
21+ use crate :: dtype:: PType ;
1922use crate :: scalar:: Scalar ;
2023use crate :: scalar_fn:: fns:: fill_null:: FillNullKernel ;
2124use crate :: scalar_fn:: fns:: operators:: Operator ;
@@ -54,19 +57,22 @@ impl FillNullKernel for Dict {
5457
5558 // A dictionary can hold more values than its codes type can index, so the index of an
5659 // unreferenced fill value can exceed the range of the codes. Non-nullable codes need no
57- // fill code. For nullable codes, decline in that case, so that the generic path fills the
58- // decoded array. The first match has the smallest index , so no later match fits either .
60+ // fill code. For nullable codes, widen the codes to the smallest unsigned type that can
61+ // hold the fill index in that case. Codes are never negative , so the cast keeps them .
5962 let codes = array. codes ( ) ;
6063 let codes = if codes. dtype ( ) . is_nullable ( ) {
61- let codes_dtype = codes. dtype ( ) . as_nonnullable ( ) ;
6264 let fill_index = existing_fill_value_index as u64 ;
63- if fill_index > codes_dtype. as_ptype ( ) . max_value_as_u64 ( ) {
64- return Ok ( None ) ;
65- }
65+ let codes = if fill_index > codes. dtype ( ) . as_ptype ( ) . max_value_as_u64 ( ) {
66+ let wide_ptype = PType :: min_unsigned_ptype_for_value ( fill_index) ;
67+ codes
68+ . clone ( )
69+ . cast ( DType :: Primitive ( wide_ptype, Nullability :: Nullable ) ) ?
70+ } else {
71+ codes. clone ( )
72+ } ;
6673
67- codes
68- . clone ( )
69- . fill_null ( Scalar :: from ( fill_index) . cast ( & codes_dtype) ?) ?
74+ let codes_dtype = codes. dtype ( ) . as_nonnullable ( ) ;
75+ codes. fill_null ( Scalar :: from ( fill_index) . cast ( & codes_dtype) ?) ?
7076 } else {
7177 codes. clone ( )
7278 } ;
@@ -99,12 +105,16 @@ mod tests {
99105
100106 use crate :: IntoArray ;
101107 use crate :: VortexSessionExecute ;
108+ use crate :: arrays:: Dict ;
102109 use crate :: arrays:: DictArray ;
103110 use crate :: arrays:: PrimitiveArray ;
111+ use crate :: arrays:: dict:: DictArraySlotsExt ;
104112 use crate :: assert_arrays_eq;
105113 use crate :: builtins:: ArrayBuiltins ;
106114 use crate :: dtype:: Nullability ;
115+ use crate :: dtype:: PType ;
107116 use crate :: scalar:: Scalar ;
117+ use crate :: scalar_fn:: fns:: fill_null:: FillNullKernel ;
108118 use crate :: validity:: Validity ;
109119
110120 static SESSION : LazyLock < VortexSession > = LazyLock :: new ( VortexSession :: empty) ;
@@ -131,6 +141,37 @@ mod tests {
131141 Ok ( ( ) )
132142 }
133143
144+ #[ rstest]
145+ #[ case:: u8_to_u16( PrimitiveArray :: from_option_iter( [ Some ( 0u8 ) , None ] ) , 256 , PType :: U16 ) ]
146+ #[ case:: u8_to_u16_truncation( PrimitiveArray :: from_option_iter( [ Some ( 0u8 ) , None ] ) , 300 , PType :: U16 ) ]
147+ #[ case:: i8_to_u8( PrimitiveArray :: from_option_iter( [ Some ( 0i8 ) , None ] ) , 128 , PType :: U8 ) ]
148+ #[ case:: i8_to_u16( PrimitiveArray :: from_option_iter( [ Some ( 0i8 ) , None ] ) , 256 , PType :: U16 ) ]
149+ #[ case:: u16_to_u32( PrimitiveArray :: from_option_iter( [ Some ( 0u16 ) , None ] ) , 65_536 , PType :: U32 ) ]
150+ fn fill_unreferenced_value_widens_codes (
151+ #[ case] codes : PrimitiveArray ,
152+ #[ case] fill_value : i32 ,
153+ #[ case] wide_ptype : PType ,
154+ ) -> VortexResult < ( ) > {
155+ let mut ctx = SESSION . create_execution_ctx ( ) ;
156+ let dict = DictArray :: try_new (
157+ codes. into_array ( ) ,
158+ PrimitiveArray :: from_iter ( 0 ..=fill_value) . into_array ( ) ,
159+ ) ?;
160+
161+ let filled = <Dict as FillNullKernel >:: fill_null (
162+ dict. as_view ( ) ,
163+ & Scalar :: from ( fill_value) ,
164+ & mut ctx,
165+ ) ?
166+ . vortex_expect ( "kernel should fill the dictionary codes" ) ;
167+ let filled_dict = filled. as_ :: < Dict > ( ) ;
168+ assert_eq ! ( filled_dict. codes( ) . dtype( ) . as_ptype( ) , wide_ptype) ;
169+ assert ! ( !filled_dict. codes( ) . dtype( ) . is_nullable( ) ) ;
170+ assert_arrays_eq ! ( filled, PrimitiveArray :: from_iter( [ 0 , fill_value] ) , & mut ctx) ;
171+
172+ Ok ( ( ) )
173+ }
174+
134175 #[ test]
135176 fn non_nullable_codes_fill_unreferenced_value ( ) -> VortexResult < ( ) > {
136177 let mut ctx = SESSION . create_execution_ctx ( ) ;
0 commit comments