@@ -63,6 +63,10 @@ impl RowFn for NumericBinary {
6363 ScalarFnVTable :: id ( & Binary )
6464 }
6565
66+ #[ allow(
67+ clippy:: cognitive_complexity,
68+ reason = "the primitive-type macro repeats the same operator dispatch for each type"
69+ ) ]
6670 fn dispatch < V : RowVisitor > (
6771 & self ,
6872 op : & Self :: Options ,
@@ -74,11 +78,20 @@ impl RowFn for NumericBinary {
7478 . ok_or_else ( || vortex_err ! ( "a numeric operator takes two operands, got none" ) ) ?,
7579 ) ?;
7680
81+ // Keep this choice inside the type macro. An outer operator/type match loses ARM i64/u64
82+ // vectorization by changing CGU placement. See the settings at `visit_checked_chunked`.
7783 match_each_native_ptype ! ( ptype, |T | {
7884 match op {
7985 NumericOperator :: Add => visit_checked:: <T , CheckedAdd , V >( visitor) ,
8086 NumericOperator :: Sub => visit_checked:: <T , CheckedSub , V >( visitor) ,
81- NumericOperator :: Mul => visit_checked:: <T , CheckedMul , V >( visitor) ,
87+ NumericOperator :: Mul => {
88+ // Chunking improves these signed widths across the measured x86 and ARM targets.
89+ if matches!( T :: PTYPE , PType :: I16 | PType :: I32 ) {
90+ visit_checked_chunked:: <T , CheckedMul , V >( visitor)
91+ } else {
92+ visit_checked:: <T , CheckedMul , V >( visitor)
93+ }
94+ }
8295 NumericOperator :: Div => visit_div:: <T , V >( visitor) ,
8396 }
8497 } )
@@ -91,20 +104,37 @@ where
91104 Op : CheckedPrimitiveOp < T > ,
92105 V : RowVisitor ,
93106{
94- let apply = |( lhs, rhs) | Op :: apply ( lhs, rhs) ;
95- let finish_failure = |failure| {
96- if failure != <Op :: Fail as Default >:: default ( ) {
97- return Err ( numeric_error ( Op :: ERROR ) ) ;
98- }
107+ visitor. visit_deferred :: < ( T , T ) , T , Op :: Fail > (
108+ |( lhs, rhs) | Op :: apply ( lhs, rhs) ,
109+ |failure| {
110+ if failure != <Op :: Fail as Default >:: default ( ) {
111+ return Err ( numeric_error ( Op :: ERROR ) ) ;
112+ }
99113
100- Ok ( ( ) )
101- } ;
114+ Ok ( ( ) )
115+ } ,
116+ )
117+ }
102118
103- if Op :: use_chunked_loop ( ) {
104- visitor. visit_deferred_chunked :: < ( T , T ) , T , Op :: Fail > ( apply, finish_failure)
105- } else {
106- visitor. visit_deferred :: < ( T , T ) , T , Op :: Fail > ( apply, finish_failure)
107- }
119+ // Keep the visit bodies separate. With rustc 1.98.0, LLVM 22.1.8, 16 CGUs, LTO off, and +neon,
120+ // sharing callback construction and dispatch here loses the two-lane dense i64/u64 loops.
121+ // Recheck binary_ops multiplication IR and timings before combining these helpers.
122+ fn visit_checked_chunked < T , Op , V > ( visitor : V ) -> VortexResult < V :: VisitResult >
123+ where
124+ T : NativePType ,
125+ Op : CheckedPrimitiveOp < T > ,
126+ V : RowVisitor ,
127+ {
128+ visitor. visit_deferred_chunked :: < ( T , T ) , T , Op :: Fail > (
129+ |( lhs, rhs) | Op :: apply ( lhs, rhs) ,
130+ |failure| {
131+ if failure != <Op :: Fail as Default >:: default ( ) {
132+ return Err ( numeric_error ( Op :: ERROR ) ) ;
133+ }
134+
135+ Ok ( ( ) )
136+ } ,
137+ )
108138}
109139
110140fn visit_div < T , V > ( visitor : V ) -> VortexResult < V :: VisitResult >
0 commit comments