Use `preserve_mostcc` for `extern "rust-cold"`

As experimentation in 115242 has shown looks better than `coldcc`.

And *don't* use a different convention for cold on Windows, because that actually ends up making things worse.

cc tracking issue 97544

9 files changed, 71 insertions, 23 deletions

diff --git a/compiler/rustc_codegen_cranelift/src/abi/mod.rs b/compiler/rustc_codegen_cranelift/src/abi/mod.rs
index b7f56a2986c..5d775b9b532 100644
--- a/compiler/rustc_codegen_cranelift/src/abi/mod.rs
+++ b/compiler/rustc_codegen_cranelift/src/abi/mod.rs
@@ -39,7 +39,7 @@ fn clif_sig_from_fn_abi<'tcx>(
 pub(crate) fn conv_to_call_conv(sess: &Session, c: Conv, default_call_conv: CallConv) -> CallConv {
     match c {
         Conv::Rust | Conv::C => default_call_conv,
-        Conv::RustCold => CallConv::Cold,
+        Conv::Cold | Conv::PreserveMost | Conv::PreserveAll => CallConv::Cold,
         Conv::X86_64SysV => CallConv::SystemV,
         Conv::X86_64Win64 => CallConv::WindowsFastcall,
 
diff --git a/compiler/rustc_codegen_llvm/src/abi.rs b/compiler/rustc_codegen_llvm/src/abi.rs
index c6a7dc95d77..863cb7068f8 100644
--- a/compiler/rustc_codegen_llvm/src/abi.rs
+++ b/compiler/rustc_codegen_llvm/src/abi.rs
@@ -571,7 +571,9 @@ impl From<Conv> for llvm::CallConv {
             Conv::C | Conv::Rust | Conv::CCmseNonSecureCall | Conv::RiscvInterrupt { .. } => {
                 llvm::CCallConv
             }
-            Conv::RustCold => llvm::ColdCallConv,
+            Conv::Cold => llvm::ColdCallConv,
+            Conv::PreserveMost => llvm::PreserveMost,
+            Conv::PreserveAll => llvm::PreserveAll,
             Conv::AmdGpuKernel => llvm::AmdGpuKernel,
             Conv::AvrInterrupt => llvm::AvrInterrupt,
             Conv::AvrNonBlockingInterrupt => llvm::AvrNonBlockingInterrupt,
diff --git a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs
index 84157d1e25c..01cbf7d3b11 100644
--- a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs
+++ b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs
@@ -83,12 +83,17 @@ pub enum LLVMModFlagBehavior {
 // Consts for the LLVM CallConv type, pre-cast to usize.
 
 /// LLVM CallingConv::ID. Should we wrap this?
+///
+/// See <https://github.com/llvm/llvm-project/blob/main/llvm/include/llvm/IR/CallingConv.h>
 #[derive(Copy, Clone, PartialEq, Debug)]
 #[repr(C)]
 pub enum CallConv {
     CCallConv = 0,
     FastCallConv = 8,
     ColdCallConv = 9,
+    PreserveMost = 14,
+    PreserveAll = 15,
+    Tail = 18,
     X86StdcallCallConv = 64,
     X86FastcallCallConv = 65,
     ArmAapcsCallConv = 67,
diff --git a/compiler/rustc_target/src/abi/call/mod.rs b/compiler/rustc_target/src/abi/call/mod.rs
index 8fab13d5d5d..8d573def9bb 100644
--- a/compiler/rustc_target/src/abi/call/mod.rs
+++ b/compiler/rustc_target/src/abi/call/mod.rs
@@ -579,10 +579,9 @@ pub enum Conv {
     C,
     Rust,
 
-    /// For things unlikely to be called, where smaller caller codegen is
-    /// preferred over raw speed.
-    /// Stronger than just `#[cold]` because `fn` pointers might be incompatible.
-    RustCold,
+    Cold,
+    PreserveMost,
+    PreserveAll,
 
     // Target-specific calling conventions.
     ArmAapcs,
@@ -605,9 +604,7 @@ pub enum Conv {
     AvrInterrupt,
     AvrNonBlockingInterrupt,
 
-    RiscvInterrupt {
-        kind: RiscvInterruptKind,
-    },
+    RiscvInterrupt { kind: RiscvInterruptKind },
 }
 
 #[derive(Copy, Clone, PartialEq, Eq, Hash, Debug, HashStable_Generic)]
diff --git a/compiler/rustc_target/src/json.rs b/compiler/rustc_target/src/json.rs
index af455b6432f..c6135149081 100644
--- a/compiler/rustc_target/src/json.rs
+++ b/compiler/rustc_target/src/json.rs
@@ -96,7 +96,9 @@ impl ToJson for crate::abi::call::Conv {
         let s = match self {
             Self::C => "C",
             Self::Rust => "Rust",
-            Self::RustCold => "RustCold",
+            Self::Cold => "Cold",
+            Self::PreserveMost => "PreserveMost",
+            Self::PreserveAll => "PreserveAll",
             Self::ArmAapcs => "ArmAapcs",
             Self::CCmseNonSecureCall => "CCmseNonSecureCall",
             Self::Msp430Intr => "Msp430Intr",
diff --git a/compiler/rustc_target/src/spec/abi.rs b/compiler/rustc_target/src/spec/abi.rs
index 550cdf6bda6..956a5cb5c2f 100644
--- a/compiler/rustc_target/src/spec/abi.rs
+++ b/compiler/rustc_target/src/spec/abi.rs
@@ -14,15 +14,33 @@ pub enum Abi {
     // hashing tests. These are used in many places, so giving them stable values reduces test
     // churn. The specific values are meaningless.
     Rust,
-    C { unwind: bool },
-    Cdecl { unwind: bool },
-    Stdcall { unwind: bool },
-    Fastcall { unwind: bool },
-    Vectorcall { unwind: bool },
-    Thiscall { unwind: bool },
-    Aapcs { unwind: bool },
-    Win64 { unwind: bool },
-    SysV64 { unwind: bool },
+    C {
+        unwind: bool,
+    },
+    Cdecl {
+        unwind: bool,
+    },
+    Stdcall {
+        unwind: bool,
+    },
+    Fastcall {
+        unwind: bool,
+    },
+    Vectorcall {
+        unwind: bool,
+    },
+    Thiscall {
+        unwind: bool,
+    },
+    Aapcs {
+        unwind: bool,
+    },
+    Win64 {
+        unwind: bool,
+    },
+    SysV64 {
+        unwind: bool,
+    },
     PtxKernel,
     Msp430Interrupt,
     X86Interrupt,
@@ -32,11 +50,16 @@ pub enum Abi {
     AvrNonBlockingInterrupt,
     CCmseNonSecureCall,
     Wasm,
-    System { unwind: bool },
+    System {
+        unwind: bool,
+    },
     RustIntrinsic,
     RustCall,
     PlatformIntrinsic,
     Unadjusted,
+    /// For things unlikely to be called, where reducing register pressure in
+    /// `extern "Rust"` callers is worth paying extra cost in the callee.
+    /// Stronger than just `#[cold]` because `fn` pointers might be incompatible.
     RustCold,
     RiscvInterruptM,
     RiscvInterruptS,
diff --git a/compiler/rustc_target/src/spec/mod.rs b/compiler/rustc_target/src/spec/mod.rs
index f8d40c4142d..b3cf76d2257 100644
--- a/compiler/rustc_target/src/spec/mod.rs
+++ b/compiler/rustc_target/src/spec/mod.rs
@@ -2276,6 +2276,13 @@ impl Target {
             Abi::Vectorcall { .. } if ["x86", "x86_64"].contains(&&self.arch[..]) => abi,
             Abi::Fastcall { unwind } | Abi::Vectorcall { unwind } => Abi::C { unwind },
 
+            // The Windows x64 calling convention we use for `extern "Rust"`
+            // <https://learn.microsoft.com/en-us/cpp/build/x64-software-conventions#register-volatility-and-preservation>
+            // expects the callee to save `xmm6` through `xmm15`, but `PreserveMost`
+            // (that we use by default for `extern "rust-cold"`) doesn't save any of those.
+            // So to avoid bloating callers, just use the Rust convention here.
+            Abi::RustCold if self.is_like_windows && self.arch == "x86_64" => Abi::Rust,
+
             abi => abi,
         }
     }
diff --git a/compiler/rustc_ty_utils/src/abi.rs b/compiler/rustc_ty_utils/src/abi.rs
index 4d0b847533b..a03da41652c 100644
--- a/compiler/rustc_ty_utils/src/abi.rs
+++ b/compiler/rustc_ty_utils/src/abi.rs
@@ -172,7 +172,10 @@ fn conv_from_spec_abi(tcx: TyCtxt<'_>, abi: SpecAbi) -> Conv {
     use rustc_target::spec::abi::Abi::*;
     match tcx.sess.target.adjust_abi(abi) {
         RustIntrinsic | PlatformIntrinsic | Rust | RustCall => Conv::Rust,
-        RustCold => Conv::RustCold,
+
+        // This is intentionally not using `Conv::Cold`, as that has to preserve
+        // even SIMD registers, which is generally not a good trade-off.
+        RustCold => Conv::PreserveMost,
 
         // It's the ABI's job to select this, not ours.
         System { .. } => bug!("system abi should be selected elsewhere"),
diff --git a/tests/codegen/cold-call-declare-and-call.rs b/tests/codegen/cold-call-declare-and-call.rs
index 71d49478bfc..572dc407f51 100644
--- a/tests/codegen/cold-call-declare-and-call.rs
+++ b/tests/codegen/cold-call-declare-and-call.rs
@@ -1,12 +1,21 @@
+// revisions: NORMAL WINDOWS
 // compile-flags: -C no-prepopulate-passes
+//[NORMAL] ignore-windows
+//[WINDOWS] only-windows
+//[WINDOWS] only-x86_64
 
 #![crate_type = "lib"]
 #![feature(rust_cold_cc)]
 
 // wasm marks the definition as `dso_local`, so allow that as optional.
 
-// CHECK: define{{( dso_local)?}} coldcc void @this_should_never_happen(i16
-// CHECK: call coldcc void @this_should_never_happen(i16
+// NORMAL: define{{( dso_local)?}} preserve_mostcc void @this_should_never_happen(i16
+// NORMAL: call preserve_mostcc void @this_should_never_happen(i16
+
+// See the comment in `Target::adjust_abi` for why this differs
+
+// WINDOWS: define void @this_should_never_happen(i16
+// WINDOWS: call void @this_should_never_happen(i16
 
 #[no_mangle]
 pub extern "rust-cold" fn this_should_never_happen(x: u16) {}