Auto merge of #133852 - x17jiri:cold_path, r=saethlin

improve cold_path()

#120370 added a new instrinsic `cold_path()` and used it to fix `likely` and `unlikely`

However, in order to limit scope, the information about cold code paths is only used in 2-target switch instructions. This is sufficient for `likely` and `unlikely`, but limits usefulness of `cold_path` for idiomatic rust. For example, code like this:

```
if let Some(x) = y { ... }
```

may generate 3-target switch:

```
switch y.discriminator:
0 => true branch
1 = > false branch
_ => unreachable
```

and therefore marking a branch as cold will have no effect.

This PR improves `cold_path()` to work with arbitrary switch instructions.

Note that for 2-target switches, we can use `llvm.expect`, but for multiple targets we need to manually emit branch weights. I checked Clang and it also emits weights in this situation. The Clang's weight calculation is more complex that this PR, which I believe is mainly because `switch` in `C/C++` can have multiple cases going to the same target.

2 files changed, 123 insertions, 0 deletions

diff --git a/tests/codegen/intrinsics/cold_path2.rs b/tests/codegen/intrinsics/cold_path2.rs
new file mode 100644
index 00000000000..1e7e0478f4f
--- /dev/null
+++ b/tests/codegen/intrinsics/cold_path2.rs
@@ -0,0 +1,36 @@
+//@ compile-flags: -O
+#![crate_type = "lib"]
+#![feature(core_intrinsics)]
+
+use std::intrinsics::cold_path;
+
+#[inline(never)]
+#[no_mangle]
+pub fn path_a() {
+    println!("path a");
+}
+
+#[inline(never)]
+#[no_mangle]
+pub fn path_b() {
+    println!("path b");
+}
+
+#[no_mangle]
+pub fn test(x: Option<bool>) {
+    if let Some(_) = x {
+        path_a();
+    } else {
+        cold_path();
+        path_b();
+    }
+
+    // CHECK-LABEL: @test(
+    // CHECK: br i1 %1, label %bb2, label %bb1, !prof ![[NUM:[0-9]+]]
+    // CHECK: bb1:
+    // CHECK: path_a
+    // CHECK: bb2:
+    // CHECK: path_b
+}
+
+// CHECK: ![[NUM]] = !{!"branch_weights", {{(!"expected", )?}}i32 1, i32 2000}
diff --git a/tests/codegen/intrinsics/cold_path3.rs b/tests/codegen/intrinsics/cold_path3.rs
new file mode 100644
index 00000000000..bf3347de665
--- /dev/null
+++ b/tests/codegen/intrinsics/cold_path3.rs
@@ -0,0 +1,87 @@
+//@ compile-flags: -O
+#![crate_type = "lib"]
+#![feature(core_intrinsics)]
+
+use std::intrinsics::cold_path;
+
+#[inline(never)]
+#[no_mangle]
+pub fn path_a() {
+    println!("path a");
+}
+
+#[inline(never)]
+#[no_mangle]
+pub fn path_b() {
+    println!("path b");
+}
+
+#[inline(never)]
+#[no_mangle]
+pub fn path_c() {
+    println!("path c");
+}
+
+#[inline(never)]
+#[no_mangle]
+pub fn path_d() {
+    println!("path d");
+}
+
+#[no_mangle]
+pub fn test(x: Option<u32>) {
+    match x {
+        Some(0) => path_a(),
+        Some(1) => {
+            cold_path();
+            path_b()
+        }
+        Some(2) => path_c(),
+        Some(3) => {
+            cold_path();
+            path_d()
+        }
+        _ => path_a(),
+    }
+
+    // CHECK-LABEL: @test(
+    // CHECK: switch i32 %1, label %bb1 [
+    // CHECK: i32 0, label %bb6
+    // CHECK: i32 1, label %bb5
+    // CHECK: i32 2, label %bb4
+    // CHECK: i32 3, label %bb3
+    // CHECK: ], !prof ![[NUM1:[0-9]+]]
+}
+
+#[no_mangle]
+pub fn test2(x: Option<u32>) {
+    match x {
+        Some(10) => path_a(),
+        Some(11) => {
+            cold_path();
+            path_b()
+        }
+        Some(12) => {
+            unsafe { core::intrinsics::unreachable() };
+            path_c()
+        }
+        Some(13) => {
+            cold_path();
+            path_d()
+        }
+        _ => {
+            cold_path();
+            path_a()
+        }
+    }
+
+    // CHECK-LABEL: @test2(
+    // CHECK: switch i32 %1, label %bb1 [
+    // CHECK: i32 10, label %bb5
+    // CHECK: i32 11, label %bb4
+    // CHECK: i32 13, label %bb3
+    // CHECK: ], !prof ![[NUM2:[0-9]+]]
+}
+
+// CHECK: ![[NUM1]] = !{!"branch_weights", i32 2000, i32 2000, i32 1, i32 2000, i32 1}
+// CHECK: ![[NUM2]] = !{!"branch_weights", i32 1, i32 2000, i32 1, i32 1}