Let `rvalue_creates_operand` return true for *all* `Rvalue::Aggregate`s

Inspired by <https://github.com/rust-lang/rust/pull/138759#discussion_r2156375342> where I noticed that we were nearly at this point, plus the comments I was writing in 143410 that reminded me a type-dependent `true` is fine.

This PR splits the `OperandRef::builder` logic out to a separate type, with the updates needed to handle SIMD as well.  In doing so, that makes the existing `Aggregate` path in `codegen_rvalue_operand` capable of handing SIMD values just fine.

As a result, we no longer need to do layout calculations for aggregate result types when running the analysis to determine which things can be SSA in codegen.

3 files changed, 135 insertions, 9 deletions

diff --git a/tests/codegen/enum/enum-aggregate.rs b/tests/codegen/enum/enum-aggregate.rs
index b6a9b8dd814..0161e5f3fa1 100644
--- a/tests/codegen/enum/enum-aggregate.rs
+++ b/tests/codegen/enum/enum-aggregate.rs
@@ -112,17 +112,14 @@ fn make_uninhabited_err_indirectly(n: Never) -> Result<u32, Never> {
 
 #[no_mangle]
 fn make_fully_uninhabited_result(v: u32, n: Never) -> Result<(u32, Never), (Never, u32)> {
-    // We don't try to do this in SSA form since the whole type is uninhabited.
+    // Actually reaching this would be UB, so we don't actually build a result.
 
     // CHECK-LABEL: { i32, i32 } @make_fully_uninhabited_result(i32 %v)
-    // CHECK: %[[ALLOC_V:.+]] = alloca [4 x i8]
-    // CHECK: %[[RET:.+]] = alloca [8 x i8]
-    // CHECK: store i32 %v, ptr %[[ALLOC_V]]
-    // CHECK: %[[TEMP_V:.+]] = load i32, ptr %[[ALLOC_V]]
-    // CHECK: %[[INNER:.+]] = getelementptr inbounds i8, ptr %[[RET]]
-    // CHECK: store i32 %[[TEMP_V]], ptr %[[INNER]]
-    // CHECK: call void @llvm.trap()
-    // CHECK: unreachable
+    // CHECK-NEXT: start:
+    // CHECK-NEXT: call void @llvm.trap()
+    // CHECK-NEXT: call void @llvm.trap()
+    // CHECK-NEXT: call void @llvm.trap()
+    // CHECK-NEXT: unreachable
     Ok((v, n))
 }
 
diff --git a/tests/codegen/simd/aggregate-simd.rs b/tests/codegen/simd/aggregate-simd.rs
new file mode 100644
index 00000000000..065e429a4c7
--- /dev/null
+++ b/tests/codegen/simd/aggregate-simd.rs
@@ -0,0 +1,106 @@
+//@ compile-flags: -C opt-level=3 -C no-prepopulate-passes
+//@ only-64bit
+
+#![feature(core_intrinsics, repr_simd)]
+#![no_std]
+#![crate_type = "lib"]
+
+use core::intrinsics::simd::{simd_add, simd_extract};
+
+#[repr(simd)]
+#[derive(Clone, Copy)]
+pub struct Simd<T, const N: usize>([T; N]);
+
+#[repr(simd, packed)]
+#[derive(Clone, Copy)]
+pub struct PackedSimd<T, const N: usize>([T; N]);
+
+#[repr(transparent)]
+pub struct Transparent<T>(T);
+
+// These tests don't actually care about the add/extract, but it ensures the
+// aggregated temporaries are only used in potentially-SSA ways.
+
+#[no_mangle]
+pub fn simd_aggregate_pot(x: [u32; 4], y: [u32; 4]) -> u32 {
+    // CHECK-LABEL: simd_aggregate_pot
+    // CHECK: %a = load <4 x i32>, ptr %x, align 4
+    // CHECK: %b = load <4 x i32>, ptr %y, align 4
+    // CHECK: add <4 x i32> %a, %b
+
+    unsafe {
+        let a = Simd(x);
+        let b = Simd(y);
+        let c = simd_add(a, b);
+        simd_extract(c, 1)
+    }
+}
+
+#[no_mangle]
+pub fn simd_aggregate_npot(x: [u32; 7], y: [u32; 7]) -> u32 {
+    // CHECK-LABEL: simd_aggregate_npot
+    // CHECK: %a = load <7 x i32>, ptr %x, align 4
+    // CHECK: %b = load <7 x i32>, ptr %y, align 4
+    // CHECK: add <7 x i32> %a, %b
+
+    unsafe {
+        let a = Simd(x);
+        let b = Simd(y);
+        let c = simd_add(a, b);
+        simd_extract(c, 1)
+    }
+}
+
+#[no_mangle]
+pub fn packed_simd_aggregate_pot(x: [u32; 4], y: [u32; 4]) -> u32 {
+    // CHECK-LABEL: packed_simd_aggregate_pot
+    // CHECK: %a = load <4 x i32>, ptr %x, align 4
+    // CHECK: %b = load <4 x i32>, ptr %y, align 4
+    // CHECK: add <4 x i32> %a, %b
+
+    unsafe {
+        let a = PackedSimd(x);
+        let b = PackedSimd(y);
+        let c = simd_add(a, b);
+        simd_extract(c, 1)
+    }
+}
+
+#[no_mangle]
+pub fn packed_simd_aggregate_npot(x: [u32; 7], y: [u32; 7]) -> u32 {
+    // CHECK-LABEL: packed_simd_aggregate_npot
+    // CHECK: %b = alloca [28 x i8], align 4
+    // CHECK: %a = alloca [28 x i8], align 4
+    // CHECK: call void @llvm.memcpy.p0.p0.i64(ptr align 4 %a, ptr align 4 %x, i64 28, i1 false)
+    // CHECK: call void @llvm.memcpy.p0.p0.i64(ptr align 4 %b, ptr align 4 %y, i64 28, i1 false)
+    // CHECK: %[[TEMPA:.+]] = load <7 x i32>, ptr %a, align 4
+    // CHECK: %[[TEMPB:.+]] = load <7 x i32>, ptr %b, align 4
+    // CHECK: add <7 x i32> %[[TEMPA]], %[[TEMPB]]
+
+    unsafe {
+        let a = PackedSimd(x);
+        let b = PackedSimd(y);
+        let c = simd_add(a, b);
+        simd_extract(c, 1)
+    }
+}
+
+#[no_mangle]
+pub fn transparent_simd_aggregate(x: [u32; 4]) -> u32 {
+    // The transparent wrapper can just use the same SSA value as its field.
+    // No extra processing or spilling needed.
+
+    // CHECK-LABEL: transparent_simd_aggregate
+    // CHECK-NOT: alloca
+    // CHECK: %[[RET:.+]] = alloca [4 x i8]
+    // CHECK-NOT: alloca
+    // CHECK: %a = load <4 x i32>, ptr %x, align 4
+    // CHECK: %[[TEMP:.+]] = extractelement <4 x i32> %a, i32 1
+    // CHECK: store i32 %[[TEMP]], ptr %[[RET]]
+
+    unsafe {
+        let a = Simd(x);
+        let b = Transparent(a);
+        simd_extract(b.0, 1)
+    }
+}
diff --git a/tests/codegen/union-aggregate.rs b/tests/codegen/union-aggregate.rs
index 3c6053379fa..aac66c5dcdd 100644
--- a/tests/codegen/union-aggregate.rs
+++ b/tests/codegen/union-aggregate.rs
@@ -4,6 +4,7 @@
 
 #![crate_type = "lib"]
 #![feature(transparent_unions)]
+#![feature(repr_simd)]
 
 #[repr(transparent)]
 union MU<T: Copy> {
@@ -83,3 +84,25 @@ fn make_mu_pair_uninit() -> MU<(u8, u32)> {
     // CHECK-NEXT: ret { i8, i32 } undef
     MU { uninit: () }
 }
+
+#[repr(simd)]
+#[derive(Copy, Clone)]
+struct I32X32([i32; 32]);
+
+#[no_mangle]
+fn make_mu_simd(x: I32X32) -> MU<I32X32> {
+    // CHECK-LABEL: void @make_mu_simd(ptr{{.+}}%_0, ptr{{.+}}%x)
+    // CHECK-NEXT: start:
+    // CHECK-NEXT: %[[TEMP:.+]] = load <32 x i32>, ptr %x,
+    // CHECK-NEXT: store <32 x i32> %[[TEMP]], ptr %_0,
+    // CHECK-NEXT: ret void
+    MU { value: x }
+}
+
+#[no_mangle]
+fn make_mu_simd_uninit() -> MU<I32X32> {
+    // CHECK-LABEL: void @make_mu_simd_uninit(ptr{{.+}}%_0)
+    // CHECK-NEXT: start:
+    // CHECK-NEXT: ret void
+    MU { uninit: () }
+}