Skip to content

Commit 91e10a8

Browse files
committed
perf(compute): tile checked lane kernels consistently
Signed-off-by: Connor Tsui <[email protected]>
1 parent 18903b1 commit 91e10a8

2 files changed

Lines changed: 56 additions & 10 deletions

File tree

‎vortex-compute/src/lane_kernels/map_into.rs‎

Lines changed: 54 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -337,20 +337,46 @@ pub trait IndexedSourceExt: IndexedSource + Sized {
337337
)
338338
};
339339

340+
#[allow(clippy::inline_always)]
341+
#[inline(always)]
342+
fn chunk<S, R, Fail, Apply>(
343+
values: &S,
344+
out: &mut [MaybeUninit<R>],
345+
apply: &Apply,
346+
base: usize,
347+
count: usize,
348+
) -> Fail
349+
where
350+
S: IndexedSource,
351+
Fail: Copy + Default + BitOrAssign,
352+
Apply: Fn(S::Item) -> (R, Fail),
353+
{
354+
let mut failed = Fail::default();
355+
for offset in 0..count {
356+
let idx = base + offset;
357+
// SAFETY: the caller proves base + count <= values.len() == out.len().
358+
let value = unsafe { values.get_unchecked(idx) };
359+
let (result, failure) = apply(value);
360+
failed |= failure;
361+
// SAFETY: the same chunk bounds prove idx < out.len().
362+
unsafe { out.get_unchecked_mut(idx).write(result) };
363+
}
364+
failed
365+
}
366+
340367
let values = self;
341368
let len = values.len();
342369
assert_eq!(out.len(), len, "out must have the same length as values");
343370

371+
// Fixed trip counts let LLVM unroll checked loops independently of the surrounding code.
372+
let chunks_count = len / CHUNK_LEN;
373+
let remainder = len % CHUNK_LEN;
344374
let mut failed = Fail::default();
345-
for idx in 0..len {
346-
// SAFETY: idx < len by the loop bound, and out.len() == len.
347-
let val = unsafe { values.get_unchecked(idx) };
348-
349-
let (result, failure) = apply(val);
350-
failed |= failure;
351-
352-
// SAFETY: idx < len == out.len().
353-
unsafe { out.get_unchecked_mut(idx).write(result) };
375+
for chunk_idx in 0..chunks_count {
376+
failed |= chunk(&values, out, &apply, chunk_idx * CHUNK_LEN, CHUNK_LEN);
377+
}
378+
if remainder != 0 {
379+
failed |= chunk(&values, out, &apply, chunks_count * CHUNK_LEN, remainder);
354380
}
355381
failed
356382
}
@@ -758,6 +784,25 @@ mod tests {
758784
assert_eq!(write_t(out), expected);
759785
}
760786

787+
#[test]
788+
fn map_checked_into_chunk_boundaries() {
789+
for len in [0usize, 1, 63, 64, 65, 127, 128, 129] {
790+
let values: Vec<u32> = (0..len as u32).collect();
791+
let mut out = vec![MaybeUninit::<u32>::uninit(); len];
792+
let failed = values.as_slice().map_checked_into(&mut out, |value| {
793+
let failure = if value % 64 == 0 {
794+
1u32 << (value / 64)
795+
} else {
796+
0
797+
};
798+
(value + 1, failure)
799+
});
800+
801+
assert_eq!(failed, (1u32 << len.div_ceil(64)) - 1, "length {len}");
802+
assert_eq!(write_t(out), (1..=len as u32).collect::<Vec<_>>());
803+
}
804+
}
805+
761806
#[test]
762807
fn map_bits_into_packs_full_and_remainder_words() {
763808
let values: Vec<u32> = (0..130).collect();

‎vortex-compute/src/lane_kernels/mod.rs‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,8 @@ pub use source::IndexedSource;
4141
pub use source::LaneZip;
4242

4343
/// Loop-tiling chunk length for the **no-mask** kernels ([`IndexedSourceExt::map_into`],
44-
/// [`IndexedSourceExt::try_map_into`], [`IndexedSinkExt::map_into_in_place`]).
44+
/// [`IndexedSourceExt::try_map_into`], [`IndexedSourceExt::map_checked_into`],
45+
/// [`IndexedSinkExt::map_into_in_place`]).
4546
///
4647
/// This is a pure tuning knob: those kernels split the lane range into
4748
/// `len / CHUNK_LEN` full chunks plus a remainder, so any value yields correct

0 commit comments

Comments
 (0)