Skip to content

Commit 1a9ed68

Browse files
authored
perf: enable POPCNT in the BMI2, AVX2 and AVX-512 kernels (#10456)
## Summary `#[target_feature(enable = "bmi2")]`, `"avx2"` and `"avx512f,..."` do not imply POPCNT, so `count_ones()` inside these kernels compiles to a SWAR sequence (with `imul` for narrow masks) in any build without `target-cpu`. The bench runners use `target-cpu=native`, so they won't show a difference. ## Changes Add `popcnt` to the target features and to the runtime detection of the `intersect_by_rank` BMI2 kernels and the x86 `simd_compress` kernels. No other `#[target_feature]` function in the tree counts bits without it (the `fearless_simd` levels already enable POPCNT). Default build, Zen 5, divan medians: | bench | before | after | | |---|---|---|---| | `intersect_by_rank` 100k random | 3.17 µs | 1.93 µs | 1.64x | | `random_rotating` 0.5/0.5 | 9.71 µs | 5.67 µs | 1.71x | | `very_sparse_mask_cached` 10%/1% | 1.91 µs | 0.81 µs | 2.36x | | `filter_fixed_width` `random_i16` 0.8 | 1.60 µs | 1.06 µs | 1.51x | | `filter_fixed_width` `random_i32` 0.5 | 1.49 µs | 1.03 µs | 1.45x | | `filter_fixed_width` `random_i64` 0.5 | 1.13 µs | 0.99 µs | 1.14x | | `filter_fixed_width` `random_i128` 0.5 (no kernel) | 1.45 µs | 1.42 µs | 1.02x | The filter numbers are the AVX-512 kernels; the AVX2 ones should show up in the CodSpeed simulation shard, where AFAIK valgrind has no AVX-512. Signed-off-by: mightsleep <[email protected]>
1 parent fdba755 commit 1a9ed68

4 files changed

Lines changed: 28 additions & 22 deletions

File tree

‎vortex-array/src/arrays/filter/execute/simd_compress/tests.rs‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -85,7 +85,7 @@ fn engages_on_supported_cpus() {
8585
let mask = mask_values(&mask).expect("alternating mask is mixed");
8686

8787
#[cfg(target_arch = "x86_64")]
88-
let expected = is_x86_feature_detected!("avx2");
88+
let expected = is_x86_feature_detected!("avx2") && is_x86_feature_detected!("popcnt");
8989
#[cfg(target_arch = "aarch64")]
9090
let expected = true;
9191

@@ -109,7 +109,7 @@ fn declines_sparse_and_short_masks() {
109109
#[cfg(all(target_arch = "x86_64", not(miri)))]
110110
#[test]
111111
fn avx2_kernels_match_scalar() {
112-
if !is_x86_feature_detected!("avx2") {
112+
if !(is_x86_feature_detected!("avx2") && is_x86_feature_detected!("popcnt")) {
113113
return;
114114
}
115115

@@ -126,15 +126,15 @@ fn avx2_kernels_match_scalar() {
126126
);
127127

128128
let mut out = vec![T::default(); mask.true_count() + SLACK_BYTES / size_of::<T>()];
129-
// SAFETY: AVX2 was detected above and the output has a vector of slack.
129+
// SAFETY: AVX2 and POPCNT were detected above and the output has a vector of slack.
130130
let written =
131131
unsafe { kernel_out_of_place(values.as_ptr().cast(), out.as_mut_ptr().cast(), mask) };
132132
assert_eq!(written, mask.true_count());
133133
assert_eq!(&out[..written], expected.as_slice());
134134

135135
let mut compacted = values.to_vec();
136136
let ptr = compacted.as_mut_ptr().cast::<u8>();
137-
// SAFETY: AVX2 was detected above; in-place compaction stays within the slice.
137+
// SAFETY: AVX2 and POPCNT were detected above; in-place compaction stays within the slice.
138138
let written = unsafe { kernel_in_place(ptr.cast_const(), ptr, mask) };
139139
assert_eq!(written, mask.true_count());
140140
assert_eq!(&compacted[..written], expected.as_slice());

‎vortex-array/src/arrays/filter/execute/simd_compress/x86.rs‎

Lines changed: 9 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -65,17 +65,18 @@ pub(super) fn select_kernel<T, const IN_PLACE: bool>(mask: &MaskValues) -> Optio
6565
}
6666

6767
fn avx512f() -> bool {
68-
is_x86_feature_detected!("avx512f")
68+
is_x86_feature_detected!("avx512f") && is_x86_feature_detected!("popcnt")
6969
}
7070

7171
fn avx512_vbmi2() -> bool {
7272
is_x86_feature_detected!("avx512f")
7373
&& is_x86_feature_detected!("avx512bw")
7474
&& is_x86_feature_detected!("avx512vbmi2")
75+
&& is_x86_feature_detected!("popcnt")
7576
}
7677

7778
fn avx2() -> bool {
78-
is_x86_feature_detected!("avx2")
79+
is_x86_feature_detected!("avx2") && is_x86_feature_detected!("popcnt")
7980
}
8081

8182
/// Generate an AVX-512 kernel for one element width.
@@ -187,7 +188,7 @@ macro_rules! avx512_compress_kernel {
187188

188189
avx512_compress_kernel!(
189190
compress_word_avx512_epi8, compress_avx512_epi8,
190-
features: "avx512f,avx512bw,avx512vbmi2",
191+
features: "avx512f,avx512bw,avx512vbmi2,popcnt",
191192
elem: i8,
192193
lanes: 64,
193194
kmask: u64,
@@ -200,7 +201,7 @@ avx512_compress_kernel!(
200201

201202
avx512_compress_kernel!(
202203
compress_word_avx512_epi16, compress_avx512_epi16,
203-
features: "avx512f,avx512bw,avx512vbmi2",
204+
features: "avx512f,avx512bw,avx512vbmi2,popcnt",
204205
elem: i16,
205206
lanes: 32,
206207
kmask: u32,
@@ -213,7 +214,7 @@ avx512_compress_kernel!(
213214

214215
avx512_compress_kernel!(
215216
compress_word_avx512_epi32, compress_avx512_epi32,
216-
features: "avx512f",
217+
features: "avx512f,popcnt",
217218
elem: i32,
218219
lanes: 16,
219220
kmask: u16,
@@ -226,7 +227,7 @@ avx512_compress_kernel!(
226227

227228
avx512_compress_kernel!(
228229
compress_word_avx512_epi64, compress_avx512_epi64,
229-
features: "avx512f",
230+
features: "avx512f,popcnt",
230231
elem: i64,
231232
lanes: 8,
232233
kmask: u8,
@@ -329,7 +330,7 @@ macro_rules! avx2_compress_kernel {
329330
clippy::cast_possible_truncation,
330331
reason = "deliberate submask narrowing"
331332
)]
332-
#[target_feature(enable = "avx2")]
333+
#[target_feature(enable = "avx2,popcnt")]
333334
#[inline]
334335
unsafe fn $word_fn<const IN_PLACE: bool>(
335336
src: *const u8,
@@ -409,7 +410,7 @@ macro_rules! avx2_compress_kernel {
409410
/// The CPU must support AVX2 and the pointer contract of
410411
/// [`filter_slice_by_bitmap`](super::filter_slice_by_bitmap) /
411412
/// [`filter_slice_mut_by_bitmap`](super::filter_slice_mut_by_bitmap) must hold.
412-
#[target_feature(enable = "avx2")]
413+
#[target_feature(enable = "avx2,popcnt")]
413414
pub(super) unsafe fn $walk_fn<const IN_PLACE: bool>(
414415
src: *const u8,
415416
dst: *mut u8,

‎vortex-mask/src/intersect_by_rank/bmi2.rs‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -45,8 +45,8 @@ unsafe fn pdep_bmi2(source: u64, mask: u64) -> u64 {
4545
/// kernel without the feature calls an out-of-line `pdep_bmi2` once per word.
4646
///
4747
/// # Safety
48-
/// Requires BMI2.
49-
#[target_feature(enable = "bmi2")]
48+
/// Requires BMI2 and POPCNT.
49+
#[target_feature(enable = "bmi2,popcnt")]
5050
pub(super) unsafe fn intersect_bit_buffers_bmi2(
5151
self_buffer: &BitBuffer,
5252
mask_buffer: &BitBuffer,
@@ -58,8 +58,8 @@ pub(super) unsafe fn intersect_bit_buffers_bmi2(
5858
/// See [`intersect_bit_buffers_bmi2`].
5959
///
6060
/// # Safety
61-
/// Requires BMI2.
62-
#[target_feature(enable = "bmi2")]
61+
/// Requires BMI2 and POPCNT.
62+
#[target_feature(enable = "bmi2,popcnt")]
6363
pub(super) unsafe fn intersect_bit_buffer_by_rank_indices_bmi2(
6464
self_buffer: &BitBuffer,
6565
mask_indices: &[usize],
@@ -70,8 +70,8 @@ pub(super) unsafe fn intersect_bit_buffer_by_rank_indices_bmi2(
7070
/// See [`intersect_bit_buffers_bmi2`].
7171
///
7272
/// # Safety
73-
/// Requires BMI2.
74-
#[target_feature(enable = "bmi2")]
73+
/// Requires BMI2 and POPCNT.
74+
#[target_feature(enable = "bmi2,popcnt")]
7575
pub(super) unsafe fn intersect_mask_driven_bmi2<I>(
7676
self_buffer: &BitBuffer,
7777
mask_indices: I,

‎vortex-mask/src/intersect_by_rank/mod.rs‎

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -443,7 +443,9 @@ type IntersectRankIndices = unsafe fn(&BitBuffer, &[usize]) -> Mask;
443443
fn select_bit_buffers<P: DepositBits>() -> IntersectBuffers {
444444
#[cfg(target_arch = "x86_64")]
445445
{
446-
if std::arch::is_x86_feature_detected!("bmi2") {
446+
if std::arch::is_x86_feature_detected!("bmi2")
447+
&& std::arch::is_x86_feature_detected!("popcnt")
448+
{
447449
return bmi2::intersect_bit_buffers_bmi2;
448450
}
449451
}
@@ -460,7 +462,9 @@ fn select_bit_buffers<P: DepositBits>() -> IntersectBuffers {
460462
fn select_rank_indices<P: DepositBits>() -> IntersectRankIndices {
461463
#[cfg(target_arch = "x86_64")]
462464
{
463-
if std::arch::is_x86_feature_detected!("bmi2") {
465+
if std::arch::is_x86_feature_detected!("bmi2")
466+
&& std::arch::is_x86_feature_detected!("popcnt")
467+
{
464468
return bmi2::intersect_bit_buffer_by_rank_indices_bmi2;
465469
}
466470
}
@@ -523,8 +527,9 @@ where
523527
I: Iterator<Item = usize>,
524528
{
525529
#[cfg(target_arch = "x86_64")]
526-
if std::arch::is_x86_feature_detected!("bmi2") {
527-
// SAFETY: BMI2 was just detected.
530+
if std::arch::is_x86_feature_detected!("bmi2") && std::arch::is_x86_feature_detected!("popcnt")
531+
{
532+
// SAFETY: BMI2 and POPCNT were just detected.
528533
return unsafe { bmi2::intersect_mask_driven_bmi2(self_buffer, mask_indices, true_count) };
529534
}
530535

0 commit comments

Comments
 (0)