Skip to content

Commit 96fc43d

Browse files
authored
Add execution path consistency assertions (#11013)
## Summary Checks batch, row, and constant execution against an explicit expected result, so agreement between paths cannot hide a shared wrong answer. ## Changes Adds test-harness assertions with coverage for nulls, nesting, errors, and divergent paths. Empty arrays check batch execution only. Scoped Clippy, pinned-nightly formatting, and nextest pass (5,251 passed, 1 skipped). Signed-off-by: Connor Tsui <[email protected]>
1 parent 493967c commit 96fc43d

3 files changed

Lines changed: 359 additions & 0 deletions

File tree

Lines changed: 151 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,151 @@
1+
// SPDX-License-Identifier: Apache-2.0
2+
// SPDX-FileCopyrightText: Copyright the Vortex contributors
3+
4+
//! Assertions for batch, row, and constant execution of the same operation.
5+
//!
6+
//! Each path uses a fresh operation and an explicit expected result, so agreement alone cannot
7+
//! conceal a shared mistake. Empty arrays have no rows to probe and only exercise batch execution.
8+
9+
use std::fmt::Display;
10+
11+
use vortex_error::VortexExpect;
12+
use vortex_error::VortexResult;
13+
use vortex_error::vortex_panic;
14+
15+
use crate::ArrayRef;
16+
use crate::ExecutionCtx;
17+
use crate::IntoArray;
18+
use crate::RecursiveCanonical;
19+
use crate::arrays::ConstantArray;
20+
use crate::assert_arrays_eq;
21+
22+
/// The result that every applicable execution path must produce.
23+
#[derive(Debug)]
24+
pub enum ExpectedOutcome {
25+
/// The expected dtype, length, nulls, and values.
26+
Array(ArrayRef),
27+
/// An error from construction or execution, including each individual row.
28+
Error,
29+
}
30+
31+
impl ExpectedOutcome {
32+
fn row(&self, index: usize) -> Self {
33+
match self {
34+
Self::Array(array) => Self::Array(
35+
array
36+
.slice(index..index + 1)
37+
.vortex_expect("expected output length matches the input length"),
38+
),
39+
Self::Error => Self::Error,
40+
}
41+
}
42+
}
43+
44+
/// Assert that an operation matches `expected` on batch, row, and constant inputs.
45+
///
46+
/// `inputs` must contain at least one array, and every input must have the same length. The
47+
/// operation must preserve that length. Input rows must be readable independently of the operation.
48+
///
49+
/// Batch execution recursively materializes the result before comparison. Row execution reads a
50+
/// fresh lazy result with [`ArrayRef::execute_scalar`]. Constant execution builds another operation
51+
/// for each row, with all inputs replaced by one-row [`ConstantArray`]s, then materializes it.
52+
/// [`ExpectedOutcome::Error`] requires every observed row to fail, even when batch execution fails.
53+
/// For empty arrays, only the batch path is checked because no row or constant input exists.
54+
///
55+
/// # Panics
56+
///
57+
/// Panics if the input requirements do not hold or an execution path differs from `expected`.
58+
#[track_caller]
59+
pub fn assert_execution_paths(
60+
inputs: &[ArrayRef],
61+
operation: impl Fn(&[ArrayRef]) -> VortexResult<ArrayRef>,
62+
expected: ExpectedOutcome,
63+
ctx: &mut ExecutionCtx,
64+
) {
65+
let first = inputs
66+
.first()
67+
.vortex_expect("execution path assertions require at least one input");
68+
let len = first.len();
69+
assert!(
70+
inputs.iter().all(|input| input.len() == len),
71+
"input arrays must have equal lengths"
72+
);
73+
74+
if let ExpectedOutcome::Array(expected) = &expected {
75+
assert_eq!(
76+
expected.len(),
77+
len,
78+
"expected output length must match the inputs"
79+
);
80+
}
81+
82+
let batch = operation(inputs).and_then(|array| execute_batch(array, ctx));
83+
assert_outcome(batch, &expected, "batch", ctx);
84+
85+
if len == 0 {
86+
return;
87+
}
88+
89+
match operation(inputs) {
90+
Ok(array) => {
91+
assert_eq!(array.len(), len, "row output length must match the inputs");
92+
if let ExpectedOutcome::Array(expected) = &expected {
93+
assert_eq!(array.dtype(), expected.dtype());
94+
}
95+
96+
for index in 0..len {
97+
let row = array
98+
.execute_scalar(index, ctx)
99+
.map(|scalar| ConstantArray::new(scalar, 1).into_array());
100+
assert_outcome(row, &expected.row(index), format_args!("row {index}"), ctx);
101+
}
102+
}
103+
Err(error) => assert_outcome(Err(error), &expected, "row construction", ctx),
104+
}
105+
106+
for index in 0..len {
107+
let constants = inputs
108+
.iter()
109+
.map(|input| {
110+
let scalar = input.execute_scalar(index, ctx).vortex_expect(
111+
"test input rows must be readable before applying the operation",
112+
);
113+
ConstantArray::new(scalar, 1).into_array()
114+
})
115+
.collect::<Vec<_>>();
116+
let constant = operation(&constants).and_then(|array| execute_batch(array, ctx));
117+
assert_outcome(
118+
constant,
119+
&expected.row(index),
120+
format_args!("constant row {index}"),
121+
ctx,
122+
);
123+
}
124+
}
125+
126+
fn execute_batch(array: ArrayRef, ctx: &mut ExecutionCtx) -> VortexResult<ArrayRef> {
127+
Ok(array.execute::<RecursiveCanonical>(ctx)?.0.into_array())
128+
}
129+
130+
#[track_caller]
131+
fn assert_outcome(
132+
actual: VortexResult<ArrayRef>,
133+
expected: &ExpectedOutcome,
134+
path: impl Display,
135+
ctx: &mut ExecutionCtx,
136+
) {
137+
match (actual, expected) {
138+
(Ok(actual), ExpectedOutcome::Array(expected)) => assert_arrays_eq!(actual, expected, ctx),
139+
(Err(_), ExpectedOutcome::Error) => {}
140+
(Err(error), ExpectedOutcome::Array(_)) => vortex_panic!("{path} path failed: {error}"),
141+
(Ok(actual), ExpectedOutcome::Error) => {
142+
vortex_panic!(
143+
"{path} path must return an error, got {} rows",
144+
actual.len()
145+
);
146+
}
147+
}
148+
}
149+
150+
#[cfg(test)]
151+
mod tests;
Lines changed: 200 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,200 @@
1+
// SPDX-License-Identifier: Apache-2.0
2+
// SPDX-FileCopyrightText: Copyright the Vortex contributors
3+
4+
use std::cell::Cell;
5+
use std::slice;
6+
use std::sync::LazyLock;
7+
8+
use rstest::rstest;
9+
use vortex_buffer::buffer;
10+
use vortex_error::VortexResult;
11+
use vortex_error::vortex_err;
12+
use vortex_session::VortexSession;
13+
14+
use super::ExpectedOutcome;
15+
use super::assert_execution_paths;
16+
use crate::ArrayRef;
17+
use crate::IntoArray;
18+
use crate::VortexSessionExecute;
19+
use crate::arrays::BoolArray;
20+
use crate::arrays::Constant;
21+
use crate::arrays::PrimitiveArray;
22+
use crate::arrays::StructArray;
23+
use crate::builtins::ArrayBuiltins;
24+
use crate::dtype::DType;
25+
use crate::dtype::Nullability;
26+
use crate::dtype::PType;
27+
use crate::scalar_fn::fns::operators::Operator;
28+
29+
static SESSION: LazyLock<VortexSession> = LazyLock::new(VortexSession::empty);
30+
31+
#[track_caller]
32+
fn assert_cast_paths(input: ArrayRef, expected: ArrayRef) {
33+
let dtype = expected.dtype().clone();
34+
assert_execution_paths(
35+
&[input],
36+
|inputs| inputs[0].cast(dtype.clone()),
37+
ExpectedOutcome::Array(expected),
38+
&mut SESSION.create_execution_ctx(),
39+
);
40+
}
41+
42+
#[rstest]
43+
#[case::non_nullable(buffer![1i32, 2, 3].into_array(), buffer![1i64, 2, 3].into_array())]
44+
#[case::partially_valid(
45+
PrimitiveArray::from_option_iter([Some(1i32), None, Some(3)]).into_array(),
46+
PrimitiveArray::from_option_iter([Some(1i64), None, Some(3)]).into_array()
47+
)]
48+
#[case::empty(
49+
PrimitiveArray::empty::<i32>(Nullability::NonNullable).into_array(),
50+
PrimitiveArray::empty::<i64>(Nullability::NonNullable).into_array()
51+
)]
52+
fn test_cast_paths(#[case] input: ArrayRef, #[case] expected: ArrayRef) {
53+
assert_cast_paths(input, expected);
54+
}
55+
56+
#[test]
57+
fn test_nested_cast_paths() -> VortexResult<()> {
58+
let input = StructArray::from_fields(&[("value", buffer![1i32, 2, 3].into_array())])?;
59+
let expected = StructArray::from_fields(&[("value", buffer![1i64, 2, 3].into_array())])?;
60+
61+
assert_cast_paths(input.into_array(), expected.into_array());
62+
Ok(())
63+
}
64+
65+
#[test]
66+
fn test_binary_paths() {
67+
assert_execution_paths(
68+
&[
69+
buffer![1i32, 3, 2].into_array(),
70+
buffer![2i32, 1, 2].into_array(),
71+
],
72+
|inputs| inputs[0].binary(inputs[1].clone(), Operator::Lt),
73+
ExpectedOutcome::Array(BoolArray::from_iter([true, false, false]).into_array()),
74+
&mut SESSION.create_execution_ctx(),
75+
);
76+
}
77+
78+
#[test]
79+
fn test_construction_errors() {
80+
assert_execution_paths(
81+
&[buffer![1i32, 2].into_array()],
82+
|_inputs| Err(vortex_err!("operation is not supported")),
83+
ExpectedOutcome::Error,
84+
&mut SESSION.create_execution_ctx(),
85+
);
86+
}
87+
88+
#[test]
89+
fn test_execution_errors() {
90+
assert_execution_paths(
91+
&[buffer![1000i32, 2000].into_array()],
92+
|inputs| inputs[0].cast(DType::Primitive(PType::I8, Nullability::NonNullable)),
93+
ExpectedOutcome::Error,
94+
&mut SESSION.create_execution_ctx(),
95+
);
96+
}
97+
98+
#[test]
99+
fn test_empty_inputs_only_run_batch() {
100+
let calls = Cell::new(0);
101+
let input = PrimitiveArray::empty::<i32>(Nullability::NonNullable).into_array();
102+
103+
assert_execution_paths(
104+
slice::from_ref(&input),
105+
|inputs| {
106+
calls.set(calls.get() + 1);
107+
Ok(inputs[0].clone())
108+
},
109+
ExpectedOutcome::Array(input.clone()),
110+
&mut SESSION.create_execution_ctx(),
111+
);
112+
assert_eq!(calls.get(), 1);
113+
}
114+
115+
#[rstest]
116+
#[case::value(buffer![1i64, 3].into_array())]
117+
#[case::null(PrimitiveArray::from_option_iter([Some(1i64), None]).into_array())]
118+
#[should_panic(expected = "arrays do not match")]
119+
fn test_rejects_incorrect_expected_values(#[case] expected: ArrayRef) {
120+
assert_cast_paths(buffer![1i32, 2].into_array(), expected);
121+
}
122+
123+
#[test]
124+
#[should_panic(expected = "arrays differ in type")]
125+
fn test_rejects_incorrect_expected_dtype() {
126+
assert_execution_paths(
127+
&[buffer![1i32].into_array()],
128+
|inputs| Ok(inputs[0].clone()),
129+
ExpectedOutcome::Array(buffer![1i64].into_array()),
130+
&mut SESSION.create_execution_ctx(),
131+
);
132+
}
133+
134+
#[test]
135+
#[should_panic(expected = "row construction path failed")]
136+
fn test_rejects_mixed_success_and_error() {
137+
let calls = Cell::new(0);
138+
assert_execution_paths(
139+
&[buffer![1i32, 2].into_array()],
140+
|inputs| {
141+
let call = calls.get();
142+
calls.set(call + 1);
143+
if call == 1 {
144+
return Err(vortex_err!("row construction failed"));
145+
}
146+
147+
Ok(inputs[0].clone())
148+
},
149+
ExpectedOutcome::Array(buffer![1i32, 2].into_array()),
150+
&mut SESSION.create_execution_ctx(),
151+
);
152+
}
153+
154+
#[test]
155+
#[should_panic(expected = "arrays do not match")]
156+
fn test_rejects_divergent_row_values() {
157+
let calls = Cell::new(0);
158+
assert_execution_paths(
159+
&[buffer![1i32, 2].into_array()],
160+
|inputs| {
161+
let call = calls.get();
162+
calls.set(call + 1);
163+
if call == 1 {
164+
return Ok(buffer![1i32, 3].into_array());
165+
}
166+
167+
Ok(inputs[0].clone())
168+
},
169+
ExpectedOutcome::Array(buffer![1i32, 2].into_array()),
170+
&mut SESSION.create_execution_ctx(),
171+
);
172+
}
173+
174+
#[test]
175+
#[should_panic(expected = "arrays do not match")]
176+
fn test_rejects_divergent_constant_values() {
177+
assert_execution_paths(
178+
&[buffer![1i32, 2].into_array()],
179+
|inputs| {
180+
if inputs[0].is::<Constant>() {
181+
return Ok(buffer![9i32].into_array());
182+
}
183+
184+
Ok(inputs[0].clone())
185+
},
186+
ExpectedOutcome::Array(buffer![1i32, 2].into_array()),
187+
&mut SESSION.create_execution_ctx(),
188+
);
189+
}
190+
191+
#[test]
192+
#[should_panic(expected = "batch path must return an error")]
193+
fn test_rejects_success_when_error_expected() {
194+
assert_execution_paths(
195+
&[buffer![1i32].into_array()],
196+
|inputs| Ok(inputs[0].clone()),
197+
ExpectedOutcome::Error,
198+
&mut SESSION.create_execution_ctx(),
199+
);
200+
}

‎vortex-array/src/test_harness/mod.rs‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,10 @@
11
// SPDX-License-Identifier: Apache-2.0
22
// SPDX-FileCopyrightText: Copyright the Vortex contributors
33

4+
//! Shared assertions and fixtures for array tests.
5+
//!
6+
//! These helpers are available to this crate's tests and to crates that enable `_test-harness`.
7+
48
use std::io::Write;
59

610
use goldenfile::Mint;
@@ -12,6 +16,10 @@ use crate::ExecutionCtx;
1216
use crate::arrays::BoolArray;
1317
use crate::arrays::bool::BoolArrayExt;
1418

19+
mod execution_paths;
20+
pub use execution_paths::ExpectedOutcome;
21+
pub use execution_paths::assert_execution_paths;
22+
1523
#[cfg(not(codspeed))]
1624
pub mod trace;
1725

0 commit comments

Comments
 (0)