Auto merge of #123886 - scottmcm:more-rvalue-operands, r=matthewjasper

Avoid `alloca`s in codegen for simple `mir::Aggregate` statements

The core idea here is to remove the abstraction penalty of simple newtypes in codegen.

Even something simple like constructing a
```rust
#[repr(transparent)] struct Foo(u32);
```
forces an `alloca` to be generated in nightly right now.

Certainly LLVM can optimize that away, but it would be nice if it didn't have to.

Quick example:
```rust
#[repr(transparent)]
pub struct Transparent32(u32);

#[no_mangle]
pub fn make_transparent(x: u32) -> Transparent32 {
    let a = Transparent32(x);
    a
}
```
on nightly we produce <https://rust.godbolt.org/z/zcvoM79ae>
```llvm
define noundef i32 `@make_transparent(i32` noundef %x) unnamed_addr #0 {
  %a = alloca i32, align 4
  store i32 %x, ptr %a, align 4
  %0 = load i32, ptr %a, align 4, !noundef !3
  ret i32 %0
}
```
but after this PR we produce
```llvm
define noundef i32 `@make_transparent(i32` noundef %x) unnamed_addr #0 {
start:
  ret i32 %x
}
```
(even before the optimizer runs).

2 files changed, 103 insertions, 13 deletions

diff --git a/compiler/rustc_codegen_ssa/src/mir/operand.rs b/compiler/rustc_codegen_ssa/src/mir/operand.rs
index a2fce1275c2..0479dd11ed0 100644
--- a/compiler/rustc_codegen_ssa/src/mir/operand.rs
+++ b/compiler/rustc_codegen_ssa/src/mir/operand.rs
@@ -14,6 +14,9 @@ use rustc_target::abi::{self, Abi, Align, Size};
 
 use std::fmt;
 
+use arrayvec::ArrayVec;
+use either::Either;
+
 /// The representation of a Rust value. The enum variant is in fact
 /// uniquely determined by the value's type, but is kept as a
 /// safety check.
@@ -58,6 +61,33 @@ pub enum OperandValue<V> {
     ZeroSized,
 }
 
+impl<V> OperandValue<V> {
+    /// If this is ZeroSized/Immediate/Pair, return an array of the 0/1/2 values.
+    /// If this is Ref, return the place.
+    #[inline]
+    pub fn immediates_or_place(self) -> Either<ArrayVec<V, 2>, PlaceValue<V>> {
+        match self {
+            OperandValue::ZeroSized => Either::Left(ArrayVec::new()),
+            OperandValue::Immediate(a) => Either::Left(ArrayVec::from_iter([a])),
+            OperandValue::Pair(a, b) => Either::Left([a, b].into()),
+            OperandValue::Ref(p) => Either::Right(p),
+        }
+    }
+
+    /// Given an array of 0/1/2 immediate values, return ZeroSized/Immediate/Pair.
+    #[inline]
+    pub fn from_immediates(immediates: ArrayVec<V, 2>) -> Self {
+        let mut it = immediates.into_iter();
+        let Some(a) = it.next() else {
+            return OperandValue::ZeroSized;
+        };
+        let Some(b) = it.next() else {
+            return OperandValue::Immediate(a);
+        };
+        OperandValue::Pair(a, b)
+    }
+}
+
 /// An `OperandRef` is an "SSA" reference to a Rust value, along with
 /// its type.
 ///
diff --git a/compiler/rustc_codegen_ssa/src/mir/rvalue.rs b/compiler/rustc_codegen_ssa/src/mir/rvalue.rs
index 1c2049f4326..2976ca14c92 100644
--- a/compiler/rustc_codegen_ssa/src/mir/rvalue.rs
+++ b/compiler/rustc_codegen_ssa/src/mir/rvalue.rs
@@ -8,14 +8,16 @@ use crate::traits::*;
 use crate::MemFlags;
 
 use rustc_hir as hir;
-use rustc_middle::mir::{self, AggregateKind, Operand};
+use rustc_middle::mir;
 use rustc_middle::ty::cast::{CastTy, IntTy};
 use rustc_middle::ty::layout::{HasTyCtxt, LayoutOf, TyAndLayout};
 use rustc_middle::ty::{self, adjustment::PointerCoercion, Instance, Ty, TyCtxt};
 use rustc_middle::{bug, span_bug};
 use rustc_session::config::OptLevel;
 use rustc_span::{Span, DUMMY_SP};
-use rustc_target::abi::{self, FIRST_VARIANT};
+use rustc_target::abi::{self, FieldIdx, FIRST_VARIANT};
+
+use arrayvec::ArrayVec;
 
 impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
     #[instrument(level = "trace", skip(self, bx))]
@@ -579,7 +581,9 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                 self.codegen_place_to_pointer(bx, place, mk_ref)
             }
 
-            mir::Rvalue::CopyForDeref(place) => self.codegen_operand(bx, &Operand::Copy(place)),
+            mir::Rvalue::CopyForDeref(place) => {
+                self.codegen_operand(bx, &mir::Operand::Copy(place))
+            }
             mir::Rvalue::AddressOf(mutability, place) => {
                 let mk_ptr =
                     move |tcx: TyCtxt<'tcx>, ty: Ty<'tcx>| Ty::new_ptr(tcx, ty, mutability);
@@ -736,11 +740,41 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                     _ => bug!("RawPtr operands {data:?} {meta:?}"),
                 }
             }
-            mir::Rvalue::Repeat(..) | mir::Rvalue::Aggregate(..) => {
-                // According to `rvalue_creates_operand`, only ZST
-                // aggregate rvalues are allowed to be operands.
+            mir::Rvalue::Repeat(..) => bug!("{rvalue:?} in codegen_rvalue_operand"),
+            mir::Rvalue::Aggregate(_, ref fields) => {
                 let ty = rvalue.ty(self.mir, self.cx.tcx());
-                OperandRef::zero_sized(self.cx.layout_of(self.monomorphize(ty)))
+                let ty = self.monomorphize(ty);
+                let layout = self.cx.layout_of(ty);
+
+                // `rvalue_creates_operand` has arranged that we only get here if
+                // we can build the aggregate immediate from the field immediates.
+                let mut inputs = ArrayVec::<Bx::Value, 2>::new();
+                let mut input_scalars = ArrayVec::<abi::Scalar, 2>::new();
+                for field_idx in layout.fields.index_by_increasing_offset() {
+                    let field_idx = FieldIdx::from_usize(field_idx);
+                    let op = self.codegen_operand(bx, &fields[field_idx]);
+                    let values = op.val.immediates_or_place().left_or_else(|p| {
+                        bug!("Field {field_idx:?} is {p:?} making {layout:?}");
+                    });
+                    inputs.extend(values);
+                    let scalars = self.value_kind(op.layout).scalars().unwrap();
+                    input_scalars.extend(scalars);
+                }
+
+                let output_scalars = self.value_kind(layout).scalars().unwrap();
+                itertools::izip!(&mut inputs, input_scalars, output_scalars).for_each(
+                    |(v, in_s, out_s)| {
+                        if in_s != out_s {
+                            // We have to be really careful about bool here, because
+                            // `(bool,)` stays i1 but `Cell<bool>` becomes i8.
+                            *v = bx.from_immediate(*v);
+                            *v = bx.to_immediate_scalar(*v, out_s);
+                        }
+                    },
+                );
+
+                let val = OperandValue::from_immediates(inputs);
+                OperandRef { val, layout }
             }
             mir::Rvalue::ShallowInitBox(ref operand, content_ty) => {
                 let operand = self.codegen_operand(bx, operand);
@@ -1047,14 +1081,29 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
             mir::Rvalue::ThreadLocalRef(_) |
             mir::Rvalue::Use(..) => // (*)
                 true,
-            // This always produces a `ty::RawPtr`, so will be Immediate or Pair
-            mir::Rvalue::Aggregate(box AggregateKind::RawPtr(..), ..) => true,
-            mir::Rvalue::Repeat(..) |
-            mir::Rvalue::Aggregate(..) => {
+            // Arrays are always aggregates, so it's not worth checking anything here.
+            // (If it's really `[(); N]` or `[T; 0]` and we use the place path, fine.)
+            mir::Rvalue::Repeat(..) => false,
+            mir::Rvalue::Aggregate(ref kind, _) => {
+                let allowed_kind = match **kind {
+                    // This always produces a `ty::RawPtr`, so will be Immediate or Pair
+                    mir::AggregateKind::RawPtr(..) => true,
+                    mir::AggregateKind::Array(..) => false,
+                    mir::AggregateKind::Tuple => true,
+                    mir::AggregateKind::Adt(def_id, ..) => {
+                        let adt_def = self.cx.tcx().adt_def(def_id);
+                        adt_def.is_struct() && !adt_def.repr().simd()
+                    }
+                    mir::AggregateKind::Closure(..) => true,
+                    // FIXME: Can we do this for simple coroutines too?
+                    mir::AggregateKind::Coroutine(..) | mir::AggregateKind::CoroutineClosure(..) => false,
+                };
+                allowed_kind && {
                 let ty = rvalue.ty(self.mir, self.cx.tcx());
                 let ty = self.monomorphize(ty);
-                // For ZST this can be `OperandValueKind::ZeroSized`.
-                self.cx.spanned_layout_of(ty, span).is_zst()
+                    let layout = self.cx.spanned_layout_of(ty, span);
+                    !self.cx.is_backend_ref(layout)
+                }
             }
         }
 
@@ -1096,3 +1145,14 @@ enum OperandValueKind {
     Pair(abi::Scalar, abi::Scalar),
     ZeroSized,
 }
+
+impl OperandValueKind {
+    fn scalars(self) -> Option<ArrayVec<abi::Scalar, 2>> {
+        Some(match self {
+            OperandValueKind::ZeroSized => ArrayVec::new(),
+            OperandValueKind::Immediate(a) => ArrayVec::from_iter([a]),
+            OperandValueKind::Pair(a, b) => [a, b].into(),
+            OperandValueKind::Ref => return None,
+        })
+    }
+}