Skip to content

Commit 6a3fb19

Browse files
authored
Preserve dict referenced values metadata (#8499)
Mark LIKE as semantically infallible and preserve all_values_referenced when generic dictionary scalar pushdown rewrites dictionary values. Signed-off-by: "Nicholas Gates" <[email protected]>
1 parent ca96791 commit 6a3fb19

3 files changed

Lines changed: 110 additions & 8 deletions

File tree

‎vortex-array/src/arrays/dict/compute/rules.rs‎

Lines changed: 48 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ use crate::arrays::Dict;
1515
use crate::arrays::DictArray;
1616
use crate::arrays::ScalarFn;
1717
use crate::arrays::ScalarFnArray;
18+
use crate::arrays::dict::DictArrayExt;
1819
use crate::arrays::dict::DictArraySlotsExt;
1920
use crate::arrays::filter::FilterReduceAdaptor;
2021
use crate::arrays::scalar_fn::AnyScalarFn;
@@ -136,13 +137,19 @@ impl ArrayParentReduceRule<Dict> for DictionaryScalarFnValuesPushDownRule {
136137
// back to nullable if needed.
137138
if sig.is_null_sensitive() && array.codes().dtype().is_nullable() {
138139
let new_codes = array.codes().cast(array.codes().dtype().as_nonnullable())?;
139-
let new_dict = unsafe { DictArray::new_unchecked(new_codes, new_values) }.into_array();
140+
let new_dict = unsafe {
141+
DictArray::new_unchecked(new_codes, new_values)
142+
.set_all_values_referenced(array.has_all_values_referenced())
143+
}
144+
.into_array();
140145
return Ok(Some(new_dict.cast(parent.dtype().clone())?));
141146
}
142147

143-
Ok(Some(
144-
unsafe { DictArray::new_unchecked(array.codes().clone(), new_values) }.into_array(),
145-
))
148+
Ok(Some(unsafe {
149+
DictArray::new_unchecked(array.codes().clone(), new_values)
150+
.set_all_values_referenced(array.has_all_values_referenced())
151+
.into_array()
152+
}))
146153
}
147154
}
148155

@@ -201,3 +208,40 @@ impl ArrayParentReduceRule<Dict> for DictionaryScalarFnCodesPullUpRule {
201208
Ok(Some(new_dict))
202209
}
203210
}
211+
212+
#[cfg(test)]
213+
mod tests {
214+
use vortex_buffer::buffer;
215+
use vortex_error::VortexResult;
216+
217+
use crate::IntoArray;
218+
use crate::arrays::BoolArray;
219+
use crate::arrays::Dict;
220+
use crate::arrays::DictArray;
221+
use crate::arrays::dict::DictArrayExt;
222+
use crate::arrays::scalar_fn::ScalarFnFactoryExt;
223+
use crate::optimizer::ArrayOptimizer;
224+
use crate::scalar_fn::EmptyOptions;
225+
use crate::scalar_fn::fns::not::Not;
226+
227+
#[test]
228+
fn scalar_fn_values_pushdown_preserves_all_values_referenced() -> VortexResult<()> {
229+
let dict = unsafe {
230+
DictArray::try_new(
231+
buffer![0u8, 1, 0, 1].into_array(),
232+
BoolArray::from_iter([true, false]).into_array(),
233+
)?
234+
.set_all_values_referenced(true)
235+
}
236+
.into_array();
237+
238+
let result = Not
239+
.try_new_array(dict.len(), EmptyOptions, [dict])?
240+
.optimize()?;
241+
let result = result.as_::<Dict>();
242+
243+
assert!(result.has_all_values_referenced());
244+
245+
Ok(())
246+
}
247+
}

‎vortex-array/src/arrays/primitive/compute/take/mod.rs‎

Lines changed: 51 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -10,10 +10,12 @@ use vortex_buffer::Buffer;
1010
use vortex_buffer::BufferMut;
1111
use vortex_error::VortexResult;
1212
use vortex_error::vortex_bail;
13+
use vortex_mask::Mask;
1314

1415
use crate::ArrayRef;
1516
use crate::IntoArray;
1617
use crate::array::ArrayView;
18+
use crate::arrays::ConstantArray;
1719
use crate::arrays::Primitive;
1820
use crate::arrays::PrimitiveArray;
1921
use crate::arrays::dict::TakeExecute;
@@ -23,6 +25,7 @@ use crate::dtype::IntegerPType;
2325
use crate::executor::ExecutionCtx;
2426
use crate::match_each_integer_ptype;
2527
use crate::match_each_native_ptype;
28+
use crate::scalar::Scalar;
2629
use crate::validity::Validity;
2730

2831
// Kernel selection happens on the first call to `take` and uses a combination of compile-time
@@ -80,19 +83,36 @@ impl TakeExecute for Primitive {
8083
vortex_bail!("Invalid indices dtype: {}", indices.dtype())
8184
};
8285

86+
let indices_validity = indices.validity()?;
87+
// Null index lanes are semantically ignored, but their physical values may be out of
88+
// bounds. Redirect those lanes to zero for the cast/gather, then restore the original index
89+
// validity below.
90+
let indices_nulls_zeroed = match indices_validity.execute_mask(indices.len(), ctx)? {
91+
Mask::AllTrue(_) => indices.clone(),
92+
Mask::AllFalse(_) => {
93+
return Ok(Some(
94+
ConstantArray::new(Scalar::null(array.dtype().as_nullable()), indices.len())
95+
.into_array(),
96+
));
97+
}
98+
Mask::Values(_) => indices
99+
.clone()
100+
.fill_null(Scalar::from(0).cast(indices.dtype())?)?,
101+
};
102+
83103
let unsigned_indices = if ptype.is_unsigned_int() {
84-
indices.clone().execute::<PrimitiveArray>(ctx)?
104+
indices_nulls_zeroed.execute::<PrimitiveArray>(ctx)?
85105
} else {
86106
// This will fail if all values cannot be converted to unsigned
87-
indices
88-
.clone()
107+
indices_nulls_zeroed
89108
.cast(DType::Primitive(ptype.to_unsigned(), *null))?
90109
.execute::<PrimitiveArray>(ctx)?
91110
};
92111

93112
let validity = array
94113
.validity()?
95-
.take(&unsigned_indices.clone().into_array())?;
114+
.take(&unsigned_indices.clone().into_array())?
115+
.and(indices_validity)?;
96116
// Delegate to the best kernel based on the target CPU
97117
{
98118
let unsigned_indices = unsigned_indices.as_view();
@@ -196,3 +216,30 @@ mod test {
196216
test_take_conformance(&array.into_array());
197217
}
198218
}
219+
220+
#[cfg(test)]
221+
mod tests {
222+
use vortex_buffer::buffer;
223+
224+
use crate::IntoArray;
225+
use crate::arrays::BoolArray;
226+
use crate::arrays::PrimitiveArray;
227+
use crate::assert_arrays_eq;
228+
use crate::validity::Validity;
229+
230+
#[test]
231+
fn take_null_index_skips_out_of_bounds_value() {
232+
let values = PrimitiveArray::from_iter([10i32, 20, 30]);
233+
let indices = PrimitiveArray::new(
234+
buffer![1u64, 3],
235+
Validity::Array(BoolArray::from_iter([true, false]).into_array()),
236+
);
237+
238+
let taken = values.take(indices.into_array()).unwrap();
239+
240+
assert_arrays_eq!(
241+
taken,
242+
PrimitiveArray::from_option_iter([Some(20i32), None]).into_array()
243+
);
244+
}
245+
}

‎vortex-array/src/scalar_fn/fns/like/mod.rs‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -157,6 +157,10 @@ impl ScalarFnVTable for Like {
157157
fn is_null_sensitive(&self, _instance: &Self::Options) -> bool {
158158
false
159159
}
160+
161+
fn is_fallible(&self, _options: &Self::Options) -> bool {
162+
false
163+
}
160164
}
161165

162166
/// Implementation of LIKE using the Arrow crate.
@@ -272,6 +276,13 @@ mod tests {
272276
);
273277
}
274278

279+
#[test]
280+
fn signature() {
281+
let like_expr = like(root(), lit("%test%"));
282+
assert!(!like_expr.signature().is_null_sensitive());
283+
assert!(!like_expr.signature().is_fallible());
284+
}
285+
275286
#[test]
276287
fn test_display() {
277288
let expr = like(get_item("name", root()), lit("%john%"));

0 commit comments

Comments
 (0)