Skip to content

Commit 7db5f02

Browse files
apacheGH-55573: [C++] Add RVV128/RVV256 SIMD kernels for bit-packing unpack
1 parent ce58c3a commit 7db5f02

6 files changed

Lines changed: 143 additions & 2 deletions

File tree

‎cpp/cmake_modules/SetupCxxFlags.cmake‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -168,7 +168,13 @@ elseif(ARROW_CPU_FLAG STREQUAL "aarch64")
168168
set(ARROW_SIMD_LEVEL "NEON")
169169
endif()
170170
elseif(ARROW_CPU_FLAG STREQUAL "riscv64")
171+
# RISC-V compiler flags, gcc/clang only
171172
set(ARROW_RVV_MARCH "rv64gcv")
173+
# Fixed-size RVV is compiled with a zvl extension plus -mrvv-vector-bits=zvl,
174+
# matching xsimd (see xsimd_get_riscv64_arch_flags in xsimd's Architecture.cmake).
175+
set(ARROW_RVV128_FLAGS "-march=${ARROW_RVV_MARCH}_zvl128b" "-mrvv-vector-bits=zvl")
176+
set(ARROW_RVV256_FLAGS "-march=${ARROW_RVV_MARCH}_zvl256b" "-mrvv-vector-bits=zvl")
177+
set(ARROW_RVV512_FLAGS "-march=${ARROW_RVV_MARCH}_zvl512b" "-mrvv-vector-bits=zvl")
172178
check_cxx_compiler_flag("-march=${ARROW_RVV_MARCH}" CXX_SUPPORTS_RVV)
173179
if(CXX_SUPPORTS_RVV AND ARROW_RUNTIME_SIMD_LEVEL MATCHES "^(RVV128|RVV256|RVV512|MAX)$")
174180
set(ARROW_HAVE_RUNTIME_RVV128 ON)

‎cpp/src/arrow/CMakeLists.txt‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -413,6 +413,35 @@ macro(append_runtime_sve512_src SRCS SRC)
413413
endif()
414414
endmacro()
415415

416+
macro(append_runtime_rvv128_src SRCS SRC)
417+
if(ARROW_HAVE_RUNTIME_RVV128)
418+
list(APPEND ${SRCS} ${SRC})
419+
set(_flags ${ARROW_RVV128_FLAGS})
420+
set_property(SOURCE ${SRC}
421+
APPEND
422+
PROPERTY COMPILE_OPTIONS ${_flags})
423+
endif()
424+
endmacro()
425+
426+
macro(append_runtime_rvv256_src SRCS SRC)
427+
if(ARROW_HAVE_RUNTIME_RVV256)
428+
list(APPEND ${SRCS} ${SRC})
429+
set(_flags ${ARROW_RVV256_FLAGS})
430+
set_property(SOURCE ${SRC}
431+
APPEND
432+
PROPERTY COMPILE_OPTIONS ${_flags})
433+
endif()
434+
endmacro()
435+
436+
macro(append_runtime_rvv512_src SRCS SRC)
437+
if(ARROW_HAVE_RUNTIME_RVV512)
438+
list(APPEND ${SRCS} ${SRC})
439+
set(_flags ${ARROW_RVV512_FLAGS})
440+
set_property(SOURCE ${SRC}
441+
APPEND
442+
PROPERTY COMPILE_OPTIONS ${_flags})
443+
endif()
444+
endmacro()
416445
# Write out compile-time configuration constants
417446
string(REPLACE "${CMAKE_SOURCE_DIR}" "<CMAKE_SOURCE_DIR>" REDACTED_CXX_FLAGS
418447
${CMAKE_CXX_FLAGS})
@@ -622,7 +651,14 @@ if(ARROW_CPU_FLAG STREQUAL "x86" AND ARROW_HAVE_RUNTIME_SSE4_2)
622651
"${ARROW_SSE4_2_FLAG}")
623652
endif()
624653

654+
# also provides the RVV128 kernels on riscv64, so it stays in ARROW_UTIL_SRCS
655+
if(ARROW_CPU_FLAG STREQUAL "riscv64" AND ARROW_HAVE_RUNTIME_RVV128)
656+
set_source_files_properties(util/bpacking_simd_128.cc PROPERTIES COMPILE_OPTIONS
657+
"${ARROW_RVV128_FLAGS}")
658+
endif()
659+
625660
append_runtime_sve256_src(ARROW_UTIL_SRCS util/bpacking_simd_256.cc)
661+
append_runtime_rvv256_src(ARROW_UTIL_SRCS util/bpacking_simd_256.cc)
626662

627663
if(ARROW_WITH_BROTLI)
628664
list(APPEND ARROW_UTIL_SRCS util/compression_brotli.cc)

‎cpp/src/arrow/util/bpacking.cc‎

Lines changed: 46 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,10 +16,12 @@
1616
// under the License.
1717

1818
#include <array>
19+
#include <type_traits>
1920

2021
#include "arrow/util/bpacking_internal.h"
2122
#include "arrow/util/bpacking_scalar_internal.h"
2223
#include "arrow/util/bpacking_simd_internal.h"
24+
#include "arrow/util/cpu_info.h"
2325
#include "arrow/util/dispatch_internal.h"
2426

2527
namespace arrow::internal {
@@ -32,11 +34,13 @@ struct UnpackDynamicFunction {
3234

3335
static constexpr auto targets() {
3436
return std::array{
35-
ARROW_DISPATCH_TARGET_NONE(&bpacking::unpack_scalar<Uint>) //
36-
ARROW_DISPATCH_TARGET_NEON(&bpacking::unpack_neon<Uint>) //
37+
ARROW_DISPATCH_TARGET_NONE(&bpacking::unpack_scalar<Uint>) //
38+
ARROW_DISPATCH_TARGET_NEON(&bpacking::unpack_neon<Uint>) //
39+
ARROW_DISPATCH_TARGET_RVV128(&bpacking::unpack_rvv128<Uint>) //
3740
// GH-50503: No SVE128 dispatch as it increases code size without
3841
// increasing performance vs. Neon, and can produce ODR violations.
3942
ARROW_DISPATCH_TARGET_SVE256(&bpacking::unpack_sve256<Uint>) //
43+
ARROW_DISPATCH_TARGET_RVV256(&bpacking::unpack_rvv256<Uint>) //
4044
ARROW_DISPATCH_TARGET_SSE4_2(&bpacking::unpack_sse4_2<Uint>) //
4145
ARROW_DISPATCH_TARGET_AVX2(&bpacking::unpack_avx2<Uint>) //
4246
ARROW_DISPATCH_TARGET_AVX512(&bpacking::unpack_avx512<Uint>) //
@@ -48,6 +52,46 @@ struct UnpackDynamicFunction {
4852

4953
template <typename Uint>
5054
void unpack(const uint8_t* in, Uint* out, const UnpackOptions& opts) {
55+
#if defined(ARROW_HAVE_RVV128) || defined(ARROW_HAVE_RVV256) || \
56+
defined(ARROW_HAVE_RUNTIME_RVV128) || \
57+
defined(ARROW_HAVE_RUNTIME_RVV256)
58+
// On RISC-V RVV, the medium SIMD kernel is slower than the scalar
59+
// implementation (which the compiler auto-vectorizes efficiently) for several
60+
// unpack combinations. Route them directly to the scalar path to avoid both
61+
// the slow kernel and the extra SIMD dispatch indirection.
62+
constexpr bool kIsUint64 = sizeof(Uint) == sizeof(uint64_t);
63+
// Whether the runtime dispatch would target the RVV256 kernel. CpuInfo::RVV256
64+
// is set only when the hardware VLEN is 256 (see cpu_info.cc), so this is
65+
// reliable on both VLEN=128 and VLEN=256 machines.
66+
static const bool kRvv256 = CpuInfo::GetInstance()->IsSupported(CpuInfo::RVV256);
67+
if (opts.bit_width == 1 && kIsUint64) {
68+
// Uint64/1-bit regresses on all RVV VLENs.
69+
return bpacking::unpack_scalar<Uint>(in, out, opts);
70+
}
71+
if (kRvv256) {
72+
// The following combinations only regress on VLEN=256 hardware; on VLEN=128
73+
// they are faster than scalar and must keep using the SIMD kernel.
74+
if (kIsUint64 && opts.bit_width == 2) {
75+
// Uint64/2-bit regresses on VLEN=256 only.
76+
return bpacking::unpack_scalar<Uint>(in, out, opts);
77+
}
78+
// Large bit widths on small batches have too much fixed medium-kernel
79+
// overhead to beat the scalar path on VLEN=256.
80+
if constexpr (std::is_same_v<Uint, uint16_t>) {
81+
if (opts.bit_width == 13 && opts.batch_size <= 32) {
82+
return bpacking::unpack_scalar<Uint>(in, out, opts);
83+
}
84+
} else if constexpr (std::is_same_v<Uint, uint32_t>) {
85+
if (opts.bit_width == 20 && opts.batch_size <= 32) {
86+
return bpacking::unpack_scalar<Uint>(in, out, opts);
87+
}
88+
} else if constexpr (std::is_same_v<Uint, uint64_t>) {
89+
if (opts.bit_width == 20 && opts.batch_size <= 64) {
90+
return bpacking::unpack_scalar<Uint>(in, out, opts);
91+
}
92+
}
93+
}
94+
#endif
5195
static const DynamicDispatch<UnpackDynamicFunction<Uint>> dispatch;
5296
return dispatch(in, out, opts);
5397
}

‎cpp/src/arrow/util/bpacking_simd_128.cc‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,10 @@
1818
#if defined(ARROW_HAVE_NEON)
1919
# define UNPACK_PLATFORM unpack_neon
2020
# define KERNEL_PLATFORM KernelNeon
21+
#elif defined(ARROW_HAVE_RVV128) || defined(ARROW_HAVE_RUNTIME_RVV128)
22+
# define UNPACK_PLATFORM unpack_rvv128
23+
# define KERNEL_PLATFORM KernelRvv128
24+
# define ARROW_RVV_MEDIUM_SCALAR_FALLBACK 1
2125
#elif defined(ARROW_HAVE_SSE4_2) || defined(ARROW_HAVE_RUNTIME_SSE4_2)
2226
# define UNPACK_PLATFORM unpack_sse4_2
2327
# define KERNEL_PLATFORM KernelSse42
@@ -26,6 +30,7 @@
2630
#if defined(UNPACK_PLATFORM)
2731

2832
# include "arrow/util/bpacking_dispatch_internal.h"
33+
# include "arrow/util/bpacking_scalar_internal.h"
2934
# include "arrow/util/bpacking_simd_internal.h"
3035
# include "arrow/util/bpacking_simd_kernel_internal.h"
3136

@@ -36,6 +41,18 @@ using KERNEL_PLATFORM = Kernel<UnpackedUint, kPackedBitSize, xsimd::default_arch
3641

3742
template <typename Uint>
3843
void UNPACK_PLATFORM(const uint8_t* in, Uint* out, const UnpackOptions& opts) {
44+
#if defined(ARROW_RVV_MEDIUM_SCALAR_FALLBACK)
45+
// On RISC-V RVV, unpacking single-bit packed integers into uint64_t is faster
46+
// with the compiler auto-vectorized scalar implementation than with the generic
47+
// medium kernel. The latter relies on byte swizzling plus per-lane variable
48+
// shifts, which do not map efficiently to RVV instructions and cause a
49+
// performance regression (measured on VLEN=128 and VLEN=256 hardware).
50+
if constexpr (sizeof(Uint) == sizeof(uint64_t)) {
51+
if (opts.bit_width == 1) {
52+
return unpack_scalar<Uint>(in, out, opts);
53+
}
54+
}
55+
#endif
3956
return unpack_jump<KERNEL_PLATFORM>(in, out, opts);
4057
}
4158

@@ -47,5 +64,6 @@ template void UNPACK_PLATFORM<uint64_t>(const uint8_t*, uint64_t*, const UnpackO
4764

4865
} // namespace arrow::internal::bpacking
4966

67+
# undef ARROW_RVV_MEDIUM_SCALAR_FALLBACK
5068
# undef UNPACK_PLATFORM
5169
#endif // UNPACK_PLATFORM

‎cpp/src/arrow/util/bpacking_simd_256.cc‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,10 @@
1818
#if defined(ARROW_HAVE_SVE256) || defined(ARROW_HAVE_RUNTIME_SVE256)
1919
# define UNPACK_PLATFORM unpack_sve256
2020
# define KERNEL_PLATFORM KernelSve256
21+
#elif defined(ARROW_HAVE_RVV256) || defined(ARROW_HAVE_RUNTIME_RVV256)
22+
# define UNPACK_PLATFORM unpack_rvv256
23+
# define KERNEL_PLATFORM KernelRvv256
24+
# define ARROW_RVV_MEDIUM_SCALAR_FALLBACK 1
2125
#elif defined(ARROW_HAVE_RUNTIME_AVX2)
2226
# define UNPACK_PLATFORM unpack_avx2
2327
# define KERNEL_PLATFORM KernelAvx2
@@ -27,8 +31,11 @@
2731
# error "This file must be compiled with a known SIMD micro architecture"
2832
#endif
2933

34+
#include <type_traits>
35+
3036
#include "arrow/util/bpacking_dispatch_internal.h"
3137
#include "arrow/util/bpacking_internal.h"
38+
#include "arrow/util/bpacking_scalar_internal.h"
3239
#include "arrow/util/bpacking_simd_internal.h"
3340
#include "arrow/util/bpacking_simd_kernel_internal.h"
3441

@@ -39,6 +46,31 @@ using KERNEL_PLATFORM = Kernel<UnpackedUint, kPackedBitSize, xsimd::default_arch
3946

4047
template <typename Uint>
4148
void UNPACK_PLATFORM(const uint8_t* in, Uint* out, const UnpackOptions& opts) {
49+
#if defined(ARROW_RVV_MEDIUM_SCALAR_FALLBACK)
50+
// On RISC-V RVV (VLEN=256 here), several unpack combinations are faster with
51+
// the compiler auto-vectorized scalar implementation than with the generic
52+
// medium kernel, which relies on byte swizzling plus per-lane variable shifts
53+
// that do not map efficiently to RVV instructions. Measured on VLEN=256
54+
// hardware; the VLEN=128 kernel file only falls back for Uint64/1-bit.
55+
constexpr bool kIsUint64 = sizeof(Uint) == sizeof(uint64_t);
56+
if (kIsUint64 && (opts.bit_width == 1 || opts.bit_width == 2)) {
57+
return unpack_scalar<Uint>(in, out, opts);
58+
}
59+
// Large bit widths on small batches have too much fixed medium-kernel overhead.
60+
if constexpr (std::is_same_v<Uint, uint16_t>) {
61+
if (opts.bit_width == 13 && opts.batch_size <= 32) {
62+
return unpack_scalar<Uint>(in, out, opts);
63+
}
64+
} else if constexpr (std::is_same_v<Uint, uint32_t>) {
65+
if (opts.bit_width == 20 && opts.batch_size <= 32) {
66+
return unpack_scalar<Uint>(in, out, opts);
67+
}
68+
} else if constexpr (std::is_same_v<Uint, uint64_t>) {
69+
if (opts.bit_width == 20 && opts.batch_size <= 64) {
70+
return unpack_scalar<Uint>(in, out, opts);
71+
}
72+
}
73+
#endif
4274
return unpack_jump<KERNEL_PLATFORM>(in, out, opts);
4375
}
4476

@@ -50,4 +82,5 @@ template void UNPACK_PLATFORM<uint64_t>(const uint8_t*, uint64_t*, const UnpackO
5082

5183
} // namespace arrow::internal::bpacking
5284

85+
#undef ARROW_RVV_MEDIUM_SCALAR_FALLBACK
5386
#undef UNPACK_PLATFORM

‎cpp/src/arrow/util/bpacking_simd_internal.h‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,8 @@ namespace arrow::internal::bpacking {
2626

2727
#if defined(ARROW_HAVE_NEON)
2828
# define UNPACK_ARCH128 unpack_neon
29+
#elif defined(ARROW_HAVE_RVV128) || defined(ARROW_HAVE_RUNTIME_RVV128)
30+
# define UNPACK_ARCH128 unpack_rvv128
2931
#elif defined(ARROW_HAVE_SSE4_2) || defined(ARROW_HAVE_RUNTIME_SSE4_2)
3032
# define UNPACK_ARCH128 unpack_sse4_2
3133
#endif
@@ -55,6 +57,8 @@ extern template ARROW_TEMPLATE_EXPORT void UNPACK_ARCH128<uint64_t>(
5557

5658
#if defined(ARROW_HAVE_SVE256) || defined(ARROW_HAVE_RUNTIME_SVE256)
5759
# define UNPACK_ARCH256 unpack_sve256
60+
#elif defined(ARROW_HAVE_RVV256) || defined(ARROW_HAVE_RUNTIME_RVV256)
61+
# define UNPACK_ARCH256 unpack_rvv256
5862
#elif defined(UNPACK_ARCH256) || defined(ARROW_HAVE_RUNTIME_AVX2)
5963
# define UNPACK_ARCH256 unpack_avx2
6064
#endif

0 commit comments

Comments
 (0)