Auto merge of #120370 - x17jiri:likely_unlikely_fix, r=saethlin

Likely unlikely fix

RFC 1131 ( https://github.com/rust-lang/rust/issues/26179 ) added likely/unlikely intrinsics, but they have been broken for a while: https://github.com/rust-lang/rust/issues/96276 , https://github.com/rust-lang/rust/issues/96275 , https://github.com/rust-lang/rust/issues/88767 . This PR tries to fix them.

Changes:
- added a new `cold_path()` intrinsic
- `likely()` and `unlikely()` changed to regular functions implemented using `cold_path()`

5 files changed, 91 insertions, 13 deletions

diff --git a/tests/codegen/checked_math.rs b/tests/codegen/checked_math.rs
index 63f5c3d34f7..c612ddccdaa 100644
--- a/tests/codegen/checked_math.rs
+++ b/tests/codegen/checked_math.rs
@@ -90,7 +90,7 @@ pub fn checked_shr_signed(a: i32, b: u32) -> Option<i32> {
 #[no_mangle]
 pub fn checked_add_one_unwrap_unsigned(x: u32) -> u32 {
     // CHECK: %[[IS_MAX:.+]] = icmp eq i32 %x, -1
-    // CHECK: br i1 %[[IS_MAX]], label %[[NONE_BB:.+]], label %[[SOME_BB:.+]]
+    // CHECK: br i1 %[[IS_MAX]], label %[[NONE_BB:.+]], label %[[SOME_BB:.+]],
     // CHECK: [[SOME_BB]]:
     // CHECK: %[[R:.+]] = add nuw i32 %x, 1
     // CHECK: ret i32 %[[R]]
diff --git a/tests/codegen/intrinsics/cold_path.rs b/tests/codegen/intrinsics/cold_path.rs
new file mode 100644
index 00000000000..24ee84e07bf
--- /dev/null
+++ b/tests/codegen/intrinsics/cold_path.rs
@@ -0,0 +1,13 @@
+//@ compile-flags: -O
+#![crate_type = "lib"]
+#![feature(core_intrinsics)]
+
+use std::intrinsics::cold_path;
+
+#[no_mangle]
+pub fn test_cold_path(x: bool) {
+    cold_path();
+}
+
+// CHECK-LABEL: @test_cold_path(
+// CHECK-NOT: cold_path
diff --git a/tests/codegen/intrinsics/likely.rs b/tests/codegen/intrinsics/likely.rs
index 9dc31d21045..e318390db20 100644
--- a/tests/codegen/intrinsics/likely.rs
+++ b/tests/codegen/intrinsics/likely.rs
@@ -1,22 +1,35 @@
-//@ compile-flags: -C no-prepopulate-passes -Copt-level=1
-
+//@ compile-flags: -O
 #![crate_type = "lib"]
 #![feature(core_intrinsics)]
 
-use std::intrinsics::{likely, unlikely};
+use std::intrinsics::likely;
 
+#[inline(never)]
 #[no_mangle]
-pub fn check_likely(x: i32, y: i32) -> Option<i32> {
-    unsafe {
-        // CHECK: call i1 @llvm.expect.i1(i1 %{{.*}}, i1 true)
-        if likely(x == y) { None } else { Some(x + y) }
-    }
+pub fn path_a() {
+    println!("path a");
+}
+
+#[inline(never)]
+#[no_mangle]
+pub fn path_b() {
+    println!("path b");
 }
 
 #[no_mangle]
-pub fn check_unlikely(x: i32, y: i32) -> Option<i32> {
-    unsafe {
-        // CHECK: call i1 @llvm.expect.i1(i1 %{{.*}}, i1 false)
-        if unlikely(x == y) { None } else { Some(x + y) }
+pub fn test_likely(x: bool) {
+    if likely(x) {
+        path_a();
+    } else {
+        path_b();
     }
 }
+
+// CHECK-LABEL: @test_likely(
+// CHECK: br i1 %x, label %bb2, label %bb3, !prof ![[NUM:[0-9]+]]
+// CHECK: bb3:
+// CHECK-NOT: cold_path
+// CHECK: path_b
+// CHECK: bb2:
+// CHECK: path_a
+// CHECK: ![[NUM]] = !{!"branch_weights", {{(!"expected", )?}}i32 2000, i32 1}
diff --git a/tests/codegen/intrinsics/likely_assert.rs b/tests/codegen/intrinsics/likely_assert.rs
new file mode 100644
index 00000000000..0ddbd6206ae
--- /dev/null
+++ b/tests/codegen/intrinsics/likely_assert.rs
@@ -0,0 +1,17 @@
+//@ compile-flags: -O
+#![crate_type = "lib"]
+
+#[no_mangle]
+pub fn test_assert(x: bool) {
+    assert!(x);
+}
+
+// check that assert! emits branch weights
+
+// CHECK-LABEL: @test_assert(
+// CHECK: br i1 %x, label %bb2, label %bb1, !prof ![[NUM:[0-9]+]]
+// CHECK: bb1:
+// CHECK: panic
+// CHECK: bb2:
+// CHECK: ret void
+// CHECK: ![[NUM]] = !{!"branch_weights", {{(!"expected", )?}}i32 2000, i32 1}
diff --git a/tests/codegen/intrinsics/unlikely.rs b/tests/codegen/intrinsics/unlikely.rs
new file mode 100644
index 00000000000..2d776031a52
--- /dev/null
+++ b/tests/codegen/intrinsics/unlikely.rs
@@ -0,0 +1,35 @@
+//@ compile-flags: -O
+#![crate_type = "lib"]
+#![feature(core_intrinsics)]
+
+use std::intrinsics::unlikely;
+
+#[inline(never)]
+#[no_mangle]
+pub fn path_a() {
+    println!("path a");
+}
+
+#[inline(never)]
+#[no_mangle]
+pub fn path_b() {
+    println!("path b");
+}
+
+#[no_mangle]
+pub fn test_unlikely(x: bool) {
+    if unlikely(x) {
+        path_a();
+    } else {
+        path_b();
+    }
+}
+
+// CHECK-LABEL: @test_unlikely(
+// CHECK: br i1 %x, label %bb2, label %bb4, !prof ![[NUM:[0-9]+]]
+// CHECK: bb4:
+// CHECK: path_b
+// CHECK: bb2:
+// CHECK-NOT: cold_path
+// CHECK: path_a
+// CHECK: ![[NUM]] = !{!"branch_weights", {{(!"expected", )?}}i32 1, i32 2000}