Skip to content

Commit 5605eab

Browse files
Execute filter scalar fns' inputs iteratively with execute_step (#11154)
Signed-off-by: Joe Isaacs <[email protected]>
1 parent db0c25f commit 5605eab

8 files changed

Lines changed: 258 additions & 8 deletions

File tree

‎vortex-array/src/executor.rs‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -919,6 +919,25 @@ macro_rules! require_child {
919919
}};
920920
}
921921

922+
/// The scalar function counterpart of [`require_child!`]: require that input `$idx` of `$args`
923+
/// matches `$M`. If it does not, early-returns a
924+
/// [`ScalarFnExecutionResult`](crate::scalar_fn::ScalarFnExecutionResult) asking the executor to
925+
/// execute that input until it matches, so the function is stepped again with the executed input.
926+
///
927+
/// ```ignore
928+
/// require_input!(args, 0 => AnyColumnar);
929+
/// ```
930+
#[macro_export]
931+
macro_rules! require_input {
932+
($args:expr, $idx:expr => $M:ty) => {
933+
if !$args.input_matches($idx, <$M as $crate::matcher::Matcher>::matches)? {
934+
return Ok($crate::scalar_fn::ScalarFnExecutionResult::execute_input::<
935+
$M,
936+
>($idx));
937+
}
938+
};
939+
}
940+
922941
/// Like [`require_child!`], but for optional children. If the child is `None`, this is a no-op.
923942
/// If the child is `Some` but does not match `$M`, early-returns an [`ExecutionResult`] requesting
924943
/// execution of child `$idx`.

‎vortex-array/src/scalar_fn/fns/between/kernel.rs‎

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -66,9 +66,10 @@ where
6666
let scalar_fn_array = parent
6767
.as_opt::<ScalarFn>()
6868
.vortex_expect("ExactScalarFn matcher confirmed ScalarFnArray");
69-
let children = scalar_fn_array.children();
70-
let lower = &children[1];
71-
let upper = &children[2];
69+
// Read only the bounds: the executor may hand over the parent with the input's slot
70+
// detached while it executes that input.
71+
let lower = scalar_fn_array.get_child(1);
72+
let upper = scalar_fn_array.get_child(2);
7273
let arr = array.array().clone();
7374
if let Some(result) = short_circuit(&arr, lower, upper, parent.options)? {
7475
return Ok(Some(result));
@@ -101,9 +102,10 @@ where
101102
let scalar_fn_array = parent
102103
.as_opt::<ScalarFn>()
103104
.vortex_expect("ExactScalarFn matcher confirmed ScalarFnArray");
104-
let children = scalar_fn_array.children();
105-
let lower = &children[1];
106-
let upper = &children[2];
105+
// Read only the bounds: the executor may hand over the parent with the input's slot
106+
// detached while it executes that input.
107+
let lower = scalar_fn_array.get_child(1);
108+
let upper = scalar_fn_array.get_child(2);
107109
let arr = array.array().clone();
108110
if let Some(result) = short_circuit(&arr, lower, upper, parent.options)? {
109111
// TODO(joe): return the lazy array directly, blocked on the same executor support as

‎vortex-array/src/scalar_fn/fns/between/mod.rs‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ use vortex_error::vortex_bail;
1313
use vortex_session::VortexSession;
1414
use vortex_session::registry::CachedId;
1515

16+
use crate::AnyCanonical;
1617
use crate::ArrayRef;
1718
use crate::Canonical;
1819
use crate::ExecutionCtx;
@@ -27,10 +28,12 @@ use crate::dtype::DType::Bool;
2728
use crate::dtype::NativePType;
2829
use crate::expr::display::ExprDisplay;
2930
use crate::proto::expr as pb;
31+
use crate::require_input;
3032
use crate::scalar::Scalar;
3133
use crate::scalar_fn::Arity;
3234
use crate::scalar_fn::ChildName;
3335
use crate::scalar_fn::ExecutionArgs;
36+
use crate::scalar_fn::ScalarFnExecutionResult;
3437
use crate::scalar_fn::ScalarFnId;
3538
use crate::scalar_fn::ScalarFnVTable;
3639
use crate::scalar_fn::ScalarFnVTableExt;
@@ -343,6 +346,17 @@ impl ScalarFnVTable for Between {
343346
between_canonical(&arr, &lower, &upper, options, ctx)
344347
}
345348

349+
fn execute_step(
350+
&self,
351+
options: &Self::Options,
352+
args: &dyn ExecutionArgs,
353+
ctx: &mut ExecutionCtx,
354+
) -> VortexResult<ScalarFnExecutionResult> {
355+
require_input!(args, 0 => AnyCanonical);
356+
self.execute(options, args, ctx)
357+
.map(ScalarFnExecutionResult::done)
358+
}
359+
346360
fn is_strict(&self, _options: &Self::Options) -> bool {
347361
false // NULL AND false = false
348362
}
@@ -785,4 +799,31 @@ mod tests {
785799

786800
assert_eq!(to_int_indices(result, ctx).unwrap(), expected_indices);
787801
}
802+
803+
/// A non-canonical input is executed through the executor before the between kernel runs.
804+
#[test]
805+
fn executes_non_canonical_input_iteratively() -> VortexResult<()> {
806+
let mut ctx = SESSION.create_execution_ctx();
807+
let chunks = vec![
808+
PrimitiveArray::from_iter([1i32, 5]).into_array(),
809+
PrimitiveArray::from_iter([9i32, 12]).into_array(),
810+
];
811+
let dtype = chunks[0].dtype().clone();
812+
let chunked = crate::arrays::ChunkedArray::try_new(chunks, dtype)?.into_array();
813+
let result = chunked.between(
814+
ConstantArray::new(5i32, 4).into_array(),
815+
ConstantArray::new(10i32, 4).into_array(),
816+
BetweenOptions {
817+
lower_strict: StrictComparison::NonStrict,
818+
upper_strict: StrictComparison::Strict,
819+
},
820+
)?;
821+
822+
assert_arrays_eq!(
823+
result,
824+
BoolArray::from_iter([false, true, true, false]),
825+
&mut ctx
826+
);
827+
Ok(())
828+
}
788829
}

‎vortex-array/src/scalar_fn/fns/binary/mod.rs‎

Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,19 +20,24 @@ use vortex_error::vortex_bail;
2020
use vortex_session::VortexSession;
2121
use vortex_session::registry::CachedId;
2222

23+
use crate::AnyColumnar;
2324
use crate::ArrayRef;
2425
use crate::ExecutionCtx;
26+
use crate::arrays::Constant;
2527
use crate::arrays::ScalarFnArray;
2628
use crate::dtype::DType;
2729
use crate::dtype::Nullability;
2830
use crate::expr::BoundExpression;
2931
use crate::expr::bound;
3032
use crate::expr::display::ExprDisplay;
33+
use crate::matcher::Matcher;
3134
use crate::proto::expr as pb;
35+
use crate::require_input;
3236
use crate::scalar_fn::Arity;
3337
use crate::scalar_fn::ChildName;
3438
use crate::scalar_fn::ExecutionArgs;
3539
use crate::scalar_fn::ReduceNode;
40+
use crate::scalar_fn::ScalarFnExecutionResult;
3641
use crate::scalar_fn::ScalarFnId;
3742
use crate::scalar_fn::ScalarFnVTable;
3843
use crate::scalar_fn::ScalarFnVTableExt;
@@ -315,6 +320,38 @@ impl ScalarFnVTable for Binary {
315320
))
316321
}
317322

323+
fn execute_step(
324+
&self,
325+
op: &Operator,
326+
args: &dyn ExecutionArgs,
327+
ctx: &mut ExecutionCtx,
328+
) -> VortexResult<ScalarFnExecutionResult> {
329+
let columnar = |idx| args.input_matches(idx, AnyColumnar::matches);
330+
let ready = (columnar(0)? && columnar(1)?)
331+
// A constant operand can decide the result, such as `false AND x` or `x < null`, so
332+
// `execute` does not need the other operand executed.
333+
|| match op {
334+
Operator::And | Operator::Or => {
335+
let constant = |idx| args.input_matches(idx, |array| array.is::<Constant>());
336+
constant(0)? || constant(1)?
337+
}
338+
_ => {
339+
let null = |idx| {
340+
args.input_matches(idx, |array| {
341+
array.as_constant().is_some_and(|scalar| scalar.is_null())
342+
})
343+
};
344+
null(0)? || null(1)?
345+
}
346+
};
347+
if !ready {
348+
require_input!(args, 0 => AnyColumnar);
349+
require_input!(args, 1 => AnyColumnar);
350+
}
351+
self.execute(op, args, ctx)
352+
.map(ScalarFnExecutionResult::done)
353+
}
354+
318355
fn is_strict(&self, operator: &Operator) -> bool {
319356
!matches!(operator, Operator::And | Operator::Or)
320357
}
@@ -344,6 +381,7 @@ mod tests {
344381
use vortex_session::VortexSession;
345382

346383
use super::*;
384+
use crate::Canonical;
347385
use crate::IntoArray;
348386
use crate::VortexSessionExecute;
349387
use crate::arrays::Bool;
@@ -750,4 +788,49 @@ mod tests {
750788
}
751789
Ok(())
752790
}
791+
792+
/// Comparisons over arithmetic, joined by `AND`: every input is a scalar function, so each
793+
/// operator executes its inputs through the executor.
794+
#[test]
795+
fn executes_non_columnar_inputs_iteratively() -> VortexResult<()> {
796+
let mut ctx = SESSION.create_execution_ctx();
797+
let a = PrimitiveArray::from_iter([1i32, 5, 9, 12]).into_array();
798+
let b = PrimitiveArray::from_iter([2i32, 2, 2, 2]).into_array();
799+
let sum = a.binary(b.clone(), Operator::Add)?;
800+
let difference = a.binary(b, Operator::Sub)?;
801+
let ten = ConstantArray::new(10i32, 4).into_array();
802+
let result = sum
803+
.binary(ten.clone(), Operator::Gt)?
804+
.binary(difference.binary(ten, Operator::Lt)?, Operator::And)?;
805+
806+
assert_arrays_eq!(
807+
result,
808+
BoolArray::from_iter([false, false, true, false]),
809+
&mut ctx
810+
);
811+
Ok(())
812+
}
813+
814+
/// A constant `false` decides `AND` without executing the other operand, which here fails
815+
/// when executed.
816+
#[test]
817+
fn deciding_constant_skips_other_operand() -> VortexResult<()> {
818+
let mut ctx = SESSION.create_execution_ctx();
819+
let failing = PrimitiveArray::from_iter([1i32, 2, 3])
820+
.into_array()
821+
.binary(ConstantArray::new(0i32, 3).into_array(), Operator::Div)?
822+
.binary(ConstantArray::new(1i32, 3).into_array(), Operator::Gt)?;
823+
assert!(failing.clone().execute::<Canonical>(&mut ctx).is_err());
824+
825+
let result = ConstantArray::new(false, 3)
826+
.into_array()
827+
.binary(failing, Operator::And)?;
828+
829+
assert_arrays_eq!(
830+
result,
831+
BoolArray::from_iter([false, false, false]),
832+
&mut ctx
833+
);
834+
Ok(())
835+
}
753836
}

‎vortex-array/src/scalar_fn/fns/fill_null/mod.rs‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,13 +27,15 @@ use crate::arrays::VarBinView;
2727
use crate::dtype::DType;
2828
use crate::expr::BoundExpression;
2929
use crate::optimizer::ArrayOptimizer;
30+
use crate::require_input;
3031
use crate::scalar::Scalar;
3132
use crate::scalar_fn::Arity;
3233
use crate::scalar_fn::ChildName;
3334
use crate::scalar_fn::EmptyOptions;
3435
use crate::scalar_fn::ExecutionArgs;
3536
use crate::scalar_fn::ReduceNode;
3637
use crate::scalar_fn::ReduceNodeValidity;
38+
use crate::scalar_fn::ScalarFnExecutionResult;
3739
use crate::scalar_fn::ScalarFnId;
3840
use crate::scalar_fn::ScalarFnVTable;
3941
use crate::scalar_fn::ScalarFnVTableExt;
@@ -168,6 +170,21 @@ impl ScalarFnVTable for FillNull {
168170
}
169171
}
170172

173+
fn execute_step(
174+
&self,
175+
options: &Self::Options,
176+
args: &dyn ExecutionArgs,
177+
ctx: &mut ExecutionCtx,
178+
) -> VortexResult<ScalarFnExecutionResult> {
179+
// Only once the fill is constant: `execute` materializes a lazy fill first, so the input's
180+
// FillNull kernels can still apply before the input is decoded.
181+
if args.input_matches(1, |fill| fill.is::<Constant>())? {
182+
require_input!(args, 0 => AnyColumnar);
183+
}
184+
self.execute(options, args, ctx)
185+
.map(ScalarFnExecutionResult::done)
186+
}
187+
171188
fn simplify(
172189
&self,
173190
_options: &Self::Options,
@@ -247,6 +264,7 @@ mod tests {
247264
use crate::Canonical;
248265
use crate::IntoArray;
249266
use crate::VortexSessionExecute;
267+
use crate::arrays::BoolArray;
250268
use crate::arrays::Chunked;
251269
use crate::arrays::ChunkedArray;
252270
use crate::arrays::Constant;
@@ -523,4 +541,18 @@ mod tests {
523541
let expr = fill_null(get_item("value", root()), lit(0i32));
524542
assert_eq!(expr.to_string(), "vortex.fill_null($.value, 0i32)");
525543
}
544+
545+
/// A scalar function input is executed through the executor before its nulls are filled.
546+
#[test]
547+
fn executes_non_columnar_input_iteratively() -> VortexResult<()> {
548+
let mut ctx = SESSION.create_execution_ctx();
549+
let input = PrimitiveArray::from_option_iter([Some(1i32), None, Some(3)])
550+
.into_array()
551+
.binary(ConstantArray::new(2i32, 3).into_array(), Operator::Gt)?;
552+
553+
let result = input.fill_null(Scalar::from(false))?;
554+
555+
assert_arrays_eq!(result, BoolArray::from_iter([false, false, true]), &mut ctx);
556+
Ok(())
557+
}
526558
}

‎vortex-array/src/scalar_fn/fns/like/mod.rs‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ use vortex_error::vortex_err;
1919
use vortex_session::VortexSession;
2020
use vortex_session::registry::CachedId;
2121

22+
use crate::AnyColumnar;
2223
use crate::ArrayRef;
2324
use crate::Canonical;
2425
use crate::ExecutionCtx;
@@ -33,10 +34,12 @@ use crate::dtype::DType;
3334
use crate::dtype::Nullability;
3435
use crate::expr::display::ExprDisplay;
3536
use crate::proto::expr as pb;
37+
use crate::require_input;
3638
use crate::scalar::Scalar;
3739
use crate::scalar_fn::Arity;
3840
use crate::scalar_fn::ChildName;
3941
use crate::scalar_fn::ExecutionArgs;
42+
use crate::scalar_fn::ScalarFnExecutionResult;
4043
use crate::scalar_fn::ScalarFnId;
4144
use crate::scalar_fn::ScalarFnVTable;
4245
use crate::scalar_fn::ScalarFnVTableExt;
@@ -171,6 +174,25 @@ impl ScalarFnVTable for Like {
171174
execute_like(&child, &pattern, *options, ctx)
172175
}
173176

177+
fn execute_step(
178+
&self,
179+
options: &Self::Options,
180+
args: &dyn ExecutionArgs,
181+
ctx: &mut ExecutionCtx,
182+
) -> VortexResult<ScalarFnExecutionResult> {
183+
// A null pattern decides every row without reading the input.
184+
if !args.input_matches(1, |pattern| {
185+
pattern
186+
.as_constant()
187+
.is_some_and(|pattern| pattern.is_null())
188+
})? {
189+
require_input!(args, 0 => AnyColumnar);
190+
require_input!(args, 1 => AnyColumnar);
191+
}
192+
self.execute(options, args, ctx)
193+
.map(ScalarFnExecutionResult::done)
194+
}
195+
174196
fn is_strict(&self, _instance: &Self::Options) -> bool {
175197
true
176198
}
@@ -699,4 +721,25 @@ mod tests {
699721
let array = VarBinViewArray::from_iter_str(values);
700722
assert_eq!(is_ascii(&array.resolved_views()), expected);
701723
}
724+
725+
/// A non-columnar input is executed through the executor before the pattern is matched.
726+
#[test]
727+
fn executes_non_columnar_input_iteratively() -> vortex_error::VortexResult<()> {
728+
let mut ctx = SESSION.create_execution_ctx();
729+
let chunks = vec![
730+
VarBinViewArray::from_iter_str(["alpha", "beta"]).into_array(),
731+
VarBinViewArray::from_iter_str(["alphabet"]).into_array(),
732+
];
733+
let dtype = chunks[0].dtype().clone();
734+
let chunked = crate::arrays::ChunkedArray::try_new(chunks, dtype)?.into_array();
735+
let result = Like::try_new(
736+
chunked,
737+
ConstantArray::new("alpha%", 3).into_array(),
738+
LikeOptions::default(),
739+
)?
740+
.into_array();
741+
742+
assert_arrays_eq!(result, BoolArray::from_iter([true, false, true]), &mut ctx);
743+
Ok(())
744+
}
702745
}

‎vortex-array/src/scalar_fn/unstable/row/batch/args.rs‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -78,4 +78,14 @@ impl ExecutionArgs for BorrowedRowFnArgs<'_> {
7878
fn row_count(&self) -> usize {
7979
self.row_count
8080
}
81+
82+
fn input_matches(&self, index: usize, predicate: fn(&ArrayRef) -> bool) -> VortexResult<bool> {
83+
let input = self.arrays.get(index).ok_or_else(|| {
84+
vortex_err!(
85+
"row-function input index must be less than {}, got {index}",
86+
self.arrays.len(),
87+
)
88+
})?;
89+
Ok(predicate(input))
90+
}
8191
}

0 commit comments

Comments
 (0)