Skip to content

Commit 710ce90

Browse files
borsbors
committed
Auto merge of rust-lang#128250 - Amanieu:select_unpredictable, r=nikic
Add `select_unpredictable` to force LLVM to use CMOV Since https://reviews.llvm.org/D118118, LLVM will no longer turn CMOVs into branches if it comes from a `select` marked with an `unpredictable` metadata attribute. This PR introduces `core::intrinsics::select_unpredictable` which emits such a `select` and uses it in the implementation of `binary_search_by`.
2 parents dba8e2d + 4f78f9f commit 710ce90

File tree

7 files changed

+107
-1
lines changed

7 files changed

+107
-1
lines changed

‎compiler/rustc_codegen_llvm/src/builder.rs

+10
Original file line numberDiff line numberDiff line change
@@ -1355,6 +1355,16 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> {
13551355
}
13561356
}
13571357

1358+
pub fn set_unpredictable(&mut self, inst: &'ll Value) {
1359+
unsafe {
1360+
llvm::LLVMSetMetadata(
1361+
inst,
1362+
llvm::MD_unpredictable as c_uint,
1363+
llvm::LLVMMDNodeInContext(self.cx.llcx, ptr::null(), 0),
1364+
);
1365+
}
1366+
}
1367+
13581368
pub fn minnum(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value {
13591369
unsafe { llvm::LLVMRustBuildMinNum(self.llbuilder, lhs, rhs) }
13601370
}

‎compiler/rustc_codegen_llvm/src/intrinsic.rs

+30-1
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ use rustc_codegen_ssa::base::{compare_simd_types, wants_msvc_seh, wants_wasm_eh}
44
use rustc_codegen_ssa::common::{IntPredicate, TypeKind};
55
use rustc_codegen_ssa::errors::{ExpectedPointerMutability, InvalidMonomorphization};
66
use rustc_codegen_ssa::mir::operand::{OperandRef, OperandValue};
7-
use rustc_codegen_ssa::mir::place::PlaceRef;
7+
use rustc_codegen_ssa::mir::place::{PlaceRef, PlaceValue};
88
use rustc_codegen_ssa::traits::*;
99
use rustc_hir as hir;
1010
use rustc_middle::mir::BinOp;
@@ -203,6 +203,35 @@ impl<'ll, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'_, 'll, 'tcx> {
203203
}
204204
sym::unlikely => self
205205
.call_intrinsic("llvm.expect.i1", &[args[0].immediate(), self.const_bool(false)]),
206+
sym::select_unpredictable => {
207+
let cond = args[0].immediate();
208+
assert_eq!(args[1].layout, args[2].layout);
209+
let select = |bx: &mut Self, true_val, false_val| {
210+
let result = bx.select(cond, true_val, false_val);
211+
bx.set_unpredictable(&result);
212+
result
213+
};
214+
match (args[1].val, args[2].val) {
215+
(OperandValue::Ref(true_val), OperandValue::Ref(false_val)) => {
216+
assert!(true_val.llextra.is_none());
217+
assert!(false_val.llextra.is_none());
218+
assert_eq!(true_val.align, false_val.align);
219+
let ptr = select(self, true_val.llval, false_val.llval);
220+
let selected =
221+
OperandValue::Ref(PlaceValue::new_sized(ptr, true_val.align));
222+
selected.store(self, result);
223+
return Ok(());
224+
}
225+
(OperandValue::Immediate(_), OperandValue::Immediate(_))
226+
| (OperandValue::Pair(_, _), OperandValue::Pair(_, _)) => {
227+
let true_val = args[1].immediate_or_packed_pair(self);
228+
let false_val = args[2].immediate_or_packed_pair(self);
229+
select(self, true_val, false_val)
230+
}
231+
(OperandValue::ZeroSized, OperandValue::ZeroSized) => return Ok(()),
232+
_ => span_bug!(span, "Incompatible OperandValue for select_unpredictable"),
233+
}
234+
}
206235
sym::catch_unwind => {
207236
catch_unwind_intrinsic(
208237
self,

‎compiler/rustc_codegen_llvm/src/llvm/ffi.rs

+1
Original file line numberDiff line numberDiff line change
@@ -426,6 +426,7 @@ pub enum MetadataType {
426426
MD_nontemporal = 9,
427427
MD_mem_parallel_loop_access = 10,
428428
MD_nonnull = 11,
429+
MD_unpredictable = 15,
429430
MD_align = 17,
430431
MD_type = 19,
431432
MD_vcall_visibility = 28,

‎compiler/rustc_hir_analysis/src/check/intrinsic.rs

+2
Original file line numberDiff line numberDiff line change
@@ -120,6 +120,7 @@ pub fn intrinsic_operation_unsafety(tcx: TyCtxt<'_>, intrinsic_id: LocalDefId) -
120120
| sym::type_id
121121
| sym::likely
122122
| sym::unlikely
123+
| sym::select_unpredictable
123124
| sym::ptr_guaranteed_cmp
124125
| sym::minnumf16
125126
| sym::minnumf32
@@ -488,6 +489,7 @@ pub fn check_intrinsic_type(
488489
sym::assume => (0, 0, vec![tcx.types.bool], tcx.types.unit),
489490
sym::likely => (0, 0, vec![tcx.types.bool], tcx.types.bool),
490491
sym::unlikely => (0, 0, vec![tcx.types.bool], tcx.types.bool),
492+
sym::select_unpredictable => (1, 0, vec![tcx.types.bool, param(0), param(0)], param(0)),
491493

492494
sym::read_via_copy => (1, 0, vec![Ty::new_imm_ptr(tcx, param(0))], param(0)),
493495
sym::write_via_move => {

‎compiler/rustc_span/src/symbol.rs

+1
Original file line numberDiff line numberDiff line change
@@ -1701,6 +1701,7 @@ symbols! {
17011701
saturating_add,
17021702
saturating_div,
17031703
saturating_sub,
1704+
select_unpredictable,
17041705
self_in_typedefs,
17051706
self_struct_ctor,
17061707
semitransparent,

‎library/core/src/intrinsics.rs

+28
Original file line numberDiff line numberDiff line change
@@ -1008,6 +1008,34 @@ pub const fn unlikely(b: bool) -> bool {
10081008
b
10091009
}
10101010

1011+
/// Returns either `true_val` or `false_val` depending on condition `b` with a
1012+
/// hint to the compiler that this condition is unlikely to be correctly
1013+
/// predicted by a CPU's branch predictor (e.g. a binary search).
1014+
///
1015+
/// This is otherwise functionally equivalent to `if b { true_val } else { false_val }`.
1016+
///
1017+
/// Note that, unlike most intrinsics, this is safe to call;
1018+
/// it does not require an `unsafe` block.
1019+
/// Therefore, implementations must not require the user to uphold
1020+
/// any safety invariants.
1021+
///
1022+
/// This intrinsic does not have a stable counterpart.
1023+
#[cfg(not(bootstrap))]
1024+
#[unstable(feature = "core_intrinsics", issue = "none")]
1025+
#[rustc_intrinsic]
1026+
#[rustc_nounwind]
1027+
#[miri::intrinsic_fallback_is_spec]
1028+
#[inline]
1029+
pub fn select_unpredictable<T>(b: bool, true_val: T, false_val: T) -> T {
1030+
if b { true_val } else { false_val }
1031+
}
1032+
1033+
#[cfg(bootstrap)]
1034+
#[inline]
1035+
pub fn select_unpredictable<T>(b: bool, true_val: T, false_val: T) -> T {
1036+
if b { true_val } else { false_val }
1037+
}
1038+
10111039
extern "rust-intrinsic" {
10121040
/// Executes a breakpoint trap, for inspection by a debugger.
10131041
///

‎tests/codegen/intrinsics/select_unpredictable.rs

+35
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
//@ compile-flags: -O
2+
3+
#![feature(core_intrinsics)]
4+
#![crate_type = "lib"]
5+
6+
#[no_mangle]
7+
pub fn test_int(p: bool, a: u64, b: u64) -> u64 {
8+
// CHECK-LABEL: define{{.*}} @test_int
9+
// CHECK: select i1 %p, i64 %a, i64 %b, !unpredictable
10+
core::intrinsics::select_unpredictable(p, a, b)
11+
}
12+
13+
#[no_mangle]
14+
pub fn test_pair(p: bool, a: (u64, u64), b: (u64, u64)) -> (u64, u64) {
15+
// CHECK-LABEL: define{{.*}} @test_pair
16+
// CHECK: select i1 %p, {{.*}}, !unpredictable
17+
core::intrinsics::select_unpredictable(p, a, b)
18+
}
19+
20+
struct Large {
21+
e: [u64; 100],
22+
}
23+
24+
#[no_mangle]
25+
pub fn test_struct(p: bool, a: Large, b: Large) -> Large {
26+
// CHECK-LABEL: define{{.*}} @test_struct
27+
// CHECK: select i1 %p, {{.*}}, !unpredictable
28+
core::intrinsics::select_unpredictable(p, a, b)
29+
}
30+
31+
#[no_mangle]
32+
pub fn test_zst(p: bool, a: (), b: ()) -> () {
33+
// CHECK-LABEL: define{{.*}} @test_zst
34+
core::intrinsics::select_unpredictable(p, a, b)
35+
}

0 commit comments

Comments
 (0)