11// SPDX-License-Identifier: Apache-2.0
22// SPDX-FileCopyrightText: Copyright the Vortex contributors
33
4- //! Checked-lane execution for numeric kernels, driven by the shared
4+ //! Checked-lane execution for the decimal kernels, driven by the shared
55//! `vortex-compute` lane kernels.
6-
7- use std:: ops:: BitOrAssign ;
6+ //!
7+ //! The primitive widths do not come through here: they are computed one row at a time by
8+ //! [`row`](super::row), which writes a value for every row and reduces failure evidence without
9+ //! scanning the finished output.
810
911use vortex_buffer:: Buffer ;
1012use vortex_buffer:: BufferMut ;
@@ -13,33 +15,18 @@ use vortex_compute::lane_kernels::IndexedSourceExt;
1315use vortex_mask:: AllOr ;
1416use vortex_mask:: Mask ;
1517
16- /// Evidence that a lane failed, anything other than [`Default`] meaning failure.
17- ///
18- /// `bool` is the ordinary choice; [`map_checked_into`] explains why the wider members exist and
19- /// asserts the width bound that membership here does **not** imply.
20- ///
21- /// [`map_checked_into`]: IndexedSourceExt::map_checked_into
22- pub ( super ) trait Failure : Copy + Default + PartialEq + BitOrAssign { }
23-
24- impl Failure for bool { }
25- impl Failure for u8 { }
26- impl Failure for u16 { }
27- impl Failure for u32 { }
28- impl Failure for u64 { }
29-
3018/// Apply the fallible `apply` over every lane of `source`, returning
31- /// `Err(first_failing_valid_lane)` only when it returns `None` on a _valid_ lane.
19+ /// `Err(first_failing_valid_lane)` only when it returns `None` on a valid lane.
3220///
3321/// `apply` also runs on invalid lanes, whose failures are masked out and whose values are
3422/// unspecified, so it must be total: no panics or side effects on any stored lane value.
3523///
36- /// This drives the one-pass early-exit kernels, which abort at the end of the enclosing 64-lane
37- /// chunk. Use it when per-lane failure handling is cheap relative to the operation, as in integer
38- /// division. Prefer [`checked_apply_lanes`] when the failure check itself vectorizes .
24+ /// This drives the one-pass early-exit kernels: failures abort at the end of the enclosing
25+ /// 64-lane chunk. It suits an operation whose per-lane failure handling is cheap relative to the
26+ /// operation itself, which is what the decimal kernels and their per-lane casts are .
3927///
40- /// `#[inline]`: this must inline into the caller that builds the closure, so that a captured
41- /// constant operand flattens into a register rather than living behind a pointer the loop reloads
42- /// on every lane, which blocks vectorization.
28+ /// Keep this wrapper inlineable so captured constants can become loop invariants in the caller.
29+ /// The lane kernels retain their own inlining decisions.
4330#[ inline]
4431pub ( super ) fn checked_lanes < S , T , Apply > (
4532 source : S ,
6148 } ;
6249
6350 let mut values = BufferMut :: < T > :: with_capacity ( len) ;
64-
6551 let out = & mut values. spare_capacity_mut ( ) [ ..len] ;
6652 match valid_bits {
6753 None => source. try_map_into ( out, apply) ?,
7359
7460 Ok ( values. freeze ( ) )
7561}
76-
77- /// Apply the split value/failure `apply` over every lane of `source`, returning
78- /// `Err(first_failing_valid_lane)` only when it flags a _valid_ lane.
79- ///
80- /// The hot pass writes every value unconditionally and OR-reduces one piece of evidence, leaving
81- /// the loop free of per-lane selects and per-chunk exit branches. Only if a lane flagged does a
82- /// cold second pass re-run `apply` through the early-exit kernels, which drop null-lane failures
83- /// and attribute the first valid one.
84- ///
85- /// Like [`checked_lanes`], `apply` runs on invalid lanes and must be total.
86- ///
87- /// `#[inline]`: see [`checked_lanes`].
88- #[ inline]
89- pub ( super ) fn checked_apply_lanes < S , T , Fail , Apply > (
90- source : S ,
91- valid_rows : & Mask ,
92- apply : Apply ,
93- ) -> Result < Buffer < T > , usize >
94- where
95- S : IndexedSource + Copy ,
96- T : Copy + Default ,
97- Fail : Failure ,
98- Apply : Fn ( S :: Item ) -> ( T , Fail ) ,
99- {
100- let len = source. len ( ) ;
101- debug_assert_eq ! ( len, valid_rows. len( ) ) ;
102-
103- let valid_bits = match valid_rows. bit_buffer ( ) {
104- AllOr :: All => None ,
105- AllOr :: None => return Ok ( Buffer :: zeroed ( len) ) ,
106- AllOr :: Some ( valid_bits) => Some ( valid_bits) ,
107- } ;
108-
109- let mut values = BufferMut :: < T > :: with_capacity ( len) ;
110-
111- let out = & mut values. spare_capacity_mut ( ) [ ..len] ;
112- if source. map_checked_into ( out, & apply) != Fail :: default ( ) {
113- let checked = |item : S :: Item | {
114- let ( value, failure) = apply ( item) ;
115- ( failure == Fail :: default ( ) ) . then_some ( value)
116- } ;
117- match valid_bits {
118- None => source. try_map_into ( out, checked) ?,
119- Some ( valid_bits) => source. try_map_masked_into ( valid_bits, out, checked) ?,
120- }
121- }
122-
123- // SAFETY: the kernels initialize every lane in `out`.
124- unsafe { values. set_len ( len) } ;
125-
126- Ok ( values. freeze ( ) )
127- }
0 commit comments