Skip to content

Commit b6ff4dc

Browse files
committed
perf: preserve default checked visitor code generation
Signed-off-by: Connor Tsui <[email protected]>
1 parent 10a8626 commit b6ff4dc

2 files changed

Lines changed: 43 additions & 26 deletions

File tree

‎vortex-array/src/scalar_fn/fns/binary/numeric/primitive.rs‎

Lines changed: 0 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,6 @@
66
use std::ops::BitOrAssign;
77

88
use crate::dtype::NativePType;
9-
use crate::dtype::PType;
109
use crate::dtype::half::f16;
1110

1211
/// Checked addition, failing on integer overflow.
@@ -38,11 +37,6 @@ pub(super) trait CheckedPrimitiveOp<T: NativePType>: 'static + Sized {
3837
/// How this operation reports a failing row. See [`Failure`].
3938
type Fail: Failure;
4039

41-
/// Select the dense reduction loop for this operation and primitive type.
42-
fn use_chunked_loop() -> bool {
43-
false
44-
}
45-
4640
/// The result of this operation, paired with evidence of whether the row failed.
4741
fn apply(lhs: T, rhs: T) -> (T, Self::Fail);
4842
}
@@ -74,13 +68,6 @@ impl<T: CheckedArithmetic> CheckedPrimitiveOp<T> for CheckedMul {
7468

7569
type Fail = T::MulFailure;
7670

77-
#[inline]
78-
fn use_chunked_loop() -> bool {
79-
// Fixed-size reductions help LLVM combine signed i16/i32 overflow checks. Other widths
80-
// and unsigned operations can lose efficient instruction selection with the same loop.
81-
matches!(T::PTYPE, PType::I16 | PType::I32)
82-
}
83-
8471
#[inline]
8572
fn apply(lhs: T, rhs: T) -> (T, T::MulFailure) {
8673
(lhs.mul_value(rhs), lhs.mul_failure(rhs))

‎vortex-array/src/scalar_fn/fns/binary/numeric/row.rs‎

Lines changed: 43 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,10 @@ impl RowFn for NumericBinary {
6363
ScalarFnVTable::id(&Binary)
6464
}
6565

66+
#[allow(
67+
clippy::cognitive_complexity,
68+
reason = "the primitive-type macro repeats the same operator dispatch for each type"
69+
)]
6670
fn dispatch<V: RowVisitor>(
6771
&self,
6872
op: &Self::Options,
@@ -74,11 +78,20 @@ impl RowFn for NumericBinary {
7478
.ok_or_else(|| vortex_err!("a numeric operator takes two operands, got none"))?,
7579
)?;
7680

81+
// Keep this choice inside the type macro. An outer operator/type match loses ARM i64/u64
82+
// vectorization by changing CGU placement. See the settings at `visit_checked_chunked`.
7783
match_each_native_ptype!(ptype, |T| {
7884
match op {
7985
NumericOperator::Add => visit_checked::<T, CheckedAdd, V>(visitor),
8086
NumericOperator::Sub => visit_checked::<T, CheckedSub, V>(visitor),
81-
NumericOperator::Mul => visit_checked::<T, CheckedMul, V>(visitor),
87+
NumericOperator::Mul => {
88+
// Chunking improves these signed widths across the measured x86 and ARM targets.
89+
if matches!(T::PTYPE, PType::I16 | PType::I32) {
90+
visit_checked_chunked::<T, CheckedMul, V>(visitor)
91+
} else {
92+
visit_checked::<T, CheckedMul, V>(visitor)
93+
}
94+
}
8295
NumericOperator::Div => visit_div::<T, V>(visitor),
8396
}
8497
})
@@ -91,20 +104,37 @@ where
91104
Op: CheckedPrimitiveOp<T>,
92105
V: RowVisitor,
93106
{
94-
let apply = |(lhs, rhs)| Op::apply(lhs, rhs);
95-
let finish_failure = |failure| {
96-
if failure != <Op::Fail as Default>::default() {
97-
return Err(numeric_error(Op::ERROR));
98-
}
107+
visitor.visit_deferred::<(T, T), T, Op::Fail>(
108+
|(lhs, rhs)| Op::apply(lhs, rhs),
109+
|failure| {
110+
if failure != <Op::Fail as Default>::default() {
111+
return Err(numeric_error(Op::ERROR));
112+
}
99113

100-
Ok(())
101-
};
114+
Ok(())
115+
},
116+
)
117+
}
102118

103-
if Op::use_chunked_loop() {
104-
visitor.visit_deferred_chunked::<(T, T), T, Op::Fail>(apply, finish_failure)
105-
} else {
106-
visitor.visit_deferred::<(T, T), T, Op::Fail>(apply, finish_failure)
107-
}
119+
// Keep the visit bodies separate. With rustc 1.98.0, LLVM 22.1.8, 16 CGUs, LTO off, and +neon,
120+
// sharing callback construction and dispatch here loses the two-lane dense i64/u64 loops.
121+
// Recheck binary_ops multiplication IR and timings before combining these helpers.
122+
fn visit_checked_chunked<T, Op, V>(visitor: V) -> VortexResult<V::VisitResult>
123+
where
124+
T: NativePType,
125+
Op: CheckedPrimitiveOp<T>,
126+
V: RowVisitor,
127+
{
128+
visitor.visit_deferred_chunked::<(T, T), T, Op::Fail>(
129+
|(lhs, rhs)| Op::apply(lhs, rhs),
130+
|failure| {
131+
if failure != <Op::Fail as Default>::default() {
132+
return Err(numeric_error(Op::ERROR));
133+
}
134+
135+
Ok(())
136+
},
137+
)
108138
}
109139

110140
fn visit_div<T, V>(visitor: V) -> VortexResult<V::VisitResult>

0 commit comments

Comments
 (0)