Auto merge of #76986 - jonas-schievink:ret-in-reg, r=nagisa

Return values up to 128 bits in registers

This fixes https://github.com/rust-lang/rust/issues/26494#issuecomment-619506345 by making Rust's default ABI pass return values up to 128 bits in size in registers, just like the System V ABI.

The result is that these methods from the comment linked above now generate the same code, making the Rust ABI as efficient as the `"C"` ABI:

```rust
pub struct Stats { x: u32, y: u32, z: u32, }

pub extern "C" fn sum_c(a: &Stats, b: &Stats) -> Stats {
    return Stats {x: a.x + b.x, y: a.y + b.y, z: a.z + b.z };
}

pub fn sum_rust(a: &Stats, b: &Stats) -> Stats {
    return Stats {x: a.x + b.x, y: a.y + b.y, z: a.z + b.z };
}
```

```asm
sum_rust:
	movl	(%rsi), %eax
	addl	(%rdi), %eax
	movl	4(%rsi), %ecx
	addl	4(%rdi), %ecx
	movl	8(%rsi), %edx
	addl	8(%rdi), %edx
	shlq	$32, %rcx
	orq	%rcx, %rax
	retq
```

3 files changed, 52 insertions, 4 deletions

diff --git a/compiler/rustc_middle/src/ty/layout.rs b/compiler/rustc_middle/src/ty/layout.rs
index 0fda1473f64..ee669ed2289 100644
--- a/compiler/rustc_middle/src/ty/layout.rs
+++ b/compiler/rustc_middle/src/ty/layout.rs
@@ -2735,6 +2735,7 @@ where
             can_unwind: fn_can_unwind(cx.tcx().sess.panic_strategy(), codegen_fn_attr_flags, conv),
         };
         fn_abi.adjust_for_abi(cx, sig.abi);
+        debug!("FnAbi::new_internal = {:?}", fn_abi);
         fn_abi
     }
 
@@ -2748,7 +2749,7 @@ where
             || abi == SpecAbi::RustIntrinsic
             || abi == SpecAbi::PlatformIntrinsic
         {
-            let fixup = |arg: &mut ArgAbi<'tcx, Ty<'tcx>>| {
+            let fixup = |arg: &mut ArgAbi<'tcx, Ty<'tcx>>, is_ret: bool| {
                 if arg.is_ignore() {
                     return;
                 }
@@ -2786,8 +2787,11 @@ where
                     _ => return,
                 }
 
+                let max_by_val_size =
+                    if is_ret { call::max_ret_by_val(cx) } else { Pointer.size(cx) };
                 let size = arg.layout.size;
-                if arg.layout.is_unsized() || size > Pointer.size(cx) {
+
+                if arg.layout.is_unsized() || size > max_by_val_size {
                     arg.make_indirect();
                 } else {
                     // We want to pass small aggregates as immediates, but using
@@ -2796,9 +2800,9 @@ where
                     arg.cast_to(Reg { kind: RegKind::Integer, size });
                 }
             };
-            fixup(&mut self.ret);
+            fixup(&mut self.ret, true);
             for arg in &mut self.args {
-                fixup(arg);
+                fixup(arg, false);
             }
             if let PassMode::Indirect(ref mut attrs, _) = self.ret.mode {
                 attrs.set(ArgAttribute::StructRet);
diff --git a/compiler/rustc_target/src/abi/call/mod.rs b/compiler/rustc_target/src/abi/call/mod.rs
index 8f7e2bba5aa..602c424a043 100644
--- a/compiler/rustc_target/src/abi/call/mod.rs
+++ b/compiler/rustc_target/src/abi/call/mod.rs
@@ -610,3 +610,15 @@ impl<'a, Ty> FnAbi<'a, Ty> {
         Ok(())
     }
 }
+
+/// Returns the maximum size of return values to be passed by value in the Rust ABI.
+///
+/// Return values beyond this size will use an implicit out-pointer instead.
+pub fn max_ret_by_val<C: HasTargetSpec + HasDataLayout>(spec: &C) -> Size {
+    match spec.target_spec().arch.as_str() {
+        // System-V will pass return values up to 128 bits in RAX/RDX.
+        "x86_64" => Size::from_bits(128),
+
+        _ => spec.data_layout().pointer_size,
+    }
+}
diff --git a/src/test/codegen/return-value-in-reg.rs b/src/test/codegen/return-value-in-reg.rs
new file mode 100644
index 00000000000..4bc0136c5e3
--- /dev/null
+++ b/src/test/codegen/return-value-in-reg.rs
@@ -0,0 +1,32 @@
+//! This test checks that types of up to 128 bits are returned by-value instead of via out-pointer.
+
+// compile-flags: -C no-prepopulate-passes -O
+// only-x86_64
+
+#![crate_type = "lib"]
+
+pub struct S {
+    a: u64,
+    b: u32,
+    c: u32,
+}
+
+// CHECK: define i128 @modify(%S* noalias nocapture dereferenceable(16) %s)
+#[no_mangle]
+pub fn modify(s: S) -> S {
+    S { a: s.a + s.a, b: s.b + s.b, c: s.c + s.c }
+}
+
+#[repr(packed)]
+pub struct TooBig {
+    a: u64,
+    b: u32,
+    c: u32,
+    d: u8,
+}
+
+// CHECK: define void @m_big(%TooBig* [[ATTRS:.*sret.*]], %TooBig* [[ATTRS2:.*]] %s)
+#[no_mangle]
+pub fn m_big(s: TooBig) -> TooBig {
+    TooBig { a: s.a + s.a, b: s.b + s.b, c: s.c + s.c, d: s.d + s.d }
+}