Auto merge of rust-lang#120370 - x17jiri:likely_unlikely_fix, r=saethlin

bors · bors · commit 1cd1dd7c56f1 · 2024-11-17T23:57:53.000Z
Likely unlikely fix RFC 1131 ( rust-lang#26179 ) added likely/unlikely intrinsics, but they have been broken for a while: rust-lang#96276 , rust-lang#96275 , rust-lang#88767 . This PR tries to fix them. Changes: - added a new `cold_path()` intrinsic - `likely()` and `unlikely()` changed to regular functions implemented using `cold_path()`
diff --git a/core/src/intrinsics/mod.rs b/core/src/intrinsics/mod.rs
@@ -1465,6 +1465,22 @@ pub const unsafe fn assume(b: bool) {
     }
 }
 
+/// Hints to the compiler that current code path is cold.
+///
+/// Note that, unlike most intrinsics, this is safe to call;
+/// it does not require an `unsafe` block.
+/// Therefore, implementations must not require the user to uphold
+/// any safety invariants.
+///
+/// This intrinsic does not have a stable counterpart.
+#[unstable(feature = "core_intrinsics", issue = "none")]
+#[cfg_attr(not(bootstrap), rustc_intrinsic)]
+#[cfg(not(bootstrap))]
+#[rustc_nounwind]
+#[miri::intrinsic_fallback_is_spec]
+#[cold]
+pub const fn cold_path() {}
+
 /// Hints to the compiler that branch condition is likely to be true.
 /// Returns the value passed to it.
 ///
@@ -1480,13 +1496,21 @@ pub const unsafe fn assume(b: bool) {
     bootstrap,
     rustc_const_stable(feature = "const_likely", since = "CURRENT_RUSTC_VERSION")
 )]
-#[cfg_attr(not(bootstrap), rustc_const_stable_intrinsic)]
 #[unstable(feature = "core_intrinsics", issue = "none")]
-#[rustc_intrinsic]
 #[rustc_nounwind]
-#[miri::intrinsic_fallback_is_spec]
+#[inline(always)]
 pub const fn likely(b: bool) -> bool {
-    b
+    #[cfg(bootstrap)]
+    {
+        b
+    }
+    #[cfg(not(bootstrap))]
+    if b {
+        true
+    } else {
+        cold_path();
+        false
+    }
 }
 
 /// Hints to the compiler that branch condition is likely to be false.
@@ -1504,13 +1528,21 @@ pub const fn likely(b: bool) -> bool {
     bootstrap,
     rustc_const_stable(feature = "const_likely", since = "CURRENT_RUSTC_VERSION")
 )]
-#[cfg_attr(not(bootstrap), rustc_const_stable_intrinsic)]
 #[unstable(feature = "core_intrinsics", issue = "none")]
-#[rustc_intrinsic]
 #[rustc_nounwind]
-#[miri::intrinsic_fallback_is_spec]
+#[inline(always)]
 pub const fn unlikely(b: bool) -> bool {
-    b
+    #[cfg(bootstrap)]
+    {
+        b
+    }
+    #[cfg(not(bootstrap))]
+    if b {
+        cold_path();
+        true
+    } else {
+        false
+    }
 }
 
 /// Returns either `true_val` or `false_val` depending on condition `b` with a
