Use `preserve_mostcc` for `extern "rust-cold"`

As experimentation in 115242 has shown looks better than `coldcc`.

And *don't* use a different convention for cold on Windows, because that actually ends up making things worse.

cc tracking issue 97544

4 files changed, 47 insertions, 18 deletions

diff --git a/compiler/rustc_target/src/abi/call/mod.rs b/compiler/rustc_target/src/abi/call/mod.rs
index 8fab13d5d5d..8d573def9bb 100644
--- a/compiler/rustc_target/src/abi/call/mod.rs
+++ b/compiler/rustc_target/src/abi/call/mod.rs
@@ -579,10 +579,9 @@ pub enum Conv {
     C,
     Rust,
 
-    /// For things unlikely to be called, where smaller caller codegen is
-    /// preferred over raw speed.
-    /// Stronger than just `#[cold]` because `fn` pointers might be incompatible.
-    RustCold,
+    Cold,
+    PreserveMost,
+    PreserveAll,
 
     // Target-specific calling conventions.
     ArmAapcs,
@@ -605,9 +604,7 @@ pub enum Conv {
     AvrInterrupt,
     AvrNonBlockingInterrupt,
 
-    RiscvInterrupt {
-        kind: RiscvInterruptKind,
-    },
+    RiscvInterrupt { kind: RiscvInterruptKind },
 }
 
 #[derive(Copy, Clone, PartialEq, Eq, Hash, Debug, HashStable_Generic)]
diff --git a/compiler/rustc_target/src/json.rs b/compiler/rustc_target/src/json.rs
index af455b6432f..c6135149081 100644
--- a/compiler/rustc_target/src/json.rs
+++ b/compiler/rustc_target/src/json.rs
@@ -96,7 +96,9 @@ impl ToJson for crate::abi::call::Conv {
         let s = match self {
             Self::C => "C",
             Self::Rust => "Rust",
-            Self::RustCold => "RustCold",
+            Self::Cold => "Cold",
+            Self::PreserveMost => "PreserveMost",
+            Self::PreserveAll => "PreserveAll",
             Self::ArmAapcs => "ArmAapcs",
             Self::CCmseNonSecureCall => "CCmseNonSecureCall",
             Self::Msp430Intr => "Msp430Intr",
diff --git a/compiler/rustc_target/src/spec/abi.rs b/compiler/rustc_target/src/spec/abi.rs
index 550cdf6bda6..956a5cb5c2f 100644
--- a/compiler/rustc_target/src/spec/abi.rs
+++ b/compiler/rustc_target/src/spec/abi.rs
@@ -14,15 +14,33 @@ pub enum Abi {
     // hashing tests. These are used in many places, so giving them stable values reduces test
     // churn. The specific values are meaningless.
     Rust,
-    C { unwind: bool },
-    Cdecl { unwind: bool },
-    Stdcall { unwind: bool },
-    Fastcall { unwind: bool },
-    Vectorcall { unwind: bool },
-    Thiscall { unwind: bool },
-    Aapcs { unwind: bool },
-    Win64 { unwind: bool },
-    SysV64 { unwind: bool },
+    C {
+        unwind: bool,
+    },
+    Cdecl {
+        unwind: bool,
+    },
+    Stdcall {
+        unwind: bool,
+    },
+    Fastcall {
+        unwind: bool,
+    },
+    Vectorcall {
+        unwind: bool,
+    },
+    Thiscall {
+        unwind: bool,
+    },
+    Aapcs {
+        unwind: bool,
+    },
+    Win64 {
+        unwind: bool,
+    },
+    SysV64 {
+        unwind: bool,
+    },
     PtxKernel,
     Msp430Interrupt,
     X86Interrupt,
@@ -32,11 +50,16 @@ pub enum Abi {
     AvrNonBlockingInterrupt,
     CCmseNonSecureCall,
     Wasm,
-    System { unwind: bool },
+    System {
+        unwind: bool,
+    },
     RustIntrinsic,
     RustCall,
     PlatformIntrinsic,
     Unadjusted,
+    /// For things unlikely to be called, where reducing register pressure in
+    /// `extern "Rust"` callers is worth paying extra cost in the callee.
+    /// Stronger than just `#[cold]` because `fn` pointers might be incompatible.
     RustCold,
     RiscvInterruptM,
     RiscvInterruptS,
diff --git a/compiler/rustc_target/src/spec/mod.rs b/compiler/rustc_target/src/spec/mod.rs
index f8d40c4142d..b3cf76d2257 100644
--- a/compiler/rustc_target/src/spec/mod.rs
+++ b/compiler/rustc_target/src/spec/mod.rs
@@ -2276,6 +2276,13 @@ impl Target {
             Abi::Vectorcall { .. } if ["x86", "x86_64"].contains(&&self.arch[..]) => abi,
             Abi::Fastcall { unwind } | Abi::Vectorcall { unwind } => Abi::C { unwind },
 
+            // The Windows x64 calling convention we use for `extern "Rust"`
+            // <https://learn.microsoft.com/en-us/cpp/build/x64-software-conventions#register-volatility-and-preservation>
+            // expects the callee to save `xmm6` through `xmm15`, but `PreserveMost`
+            // (that we use by default for `extern "rust-cold"`) doesn't save any of those.
+            // So to avoid bloating callers, just use the Rust convention here.
+            Abi::RustCold if self.is_like_windows && self.arch == "x86_64" => Abi::Rust,
+
             abi => abi,
         }
     }