Auto merge of #138759 - scottmcm:operand-builder, r=saethlin

Allow `enum` and `union` literals to also create SSA values

Today, `Some(x)` always goes through an `alloca`, even in trivial cases where the niching means the constructor doesn't even change the value.

For example, <https://rust.godbolt.org/z/6KG6PqoYz>
```rust
pub fn demo(r: &i32) -> Option<&i32> {
    Some(r)
}
```
currently emits the IR
```llvm
define align 4 ptr `@demo(ptr` align 4 %r) unnamed_addr {
start:
  %_0 = alloca [8 x i8], align 8
  store ptr %r, ptr %_0, align 8
  %0 = load ptr, ptr %_0, align 8
  ret ptr %0
}
```
but with this PR it becomes just
```llvm
define align 4 ptr `@demo(ptr` align 4 %r) unnamed_addr {
start:
  ret ptr %r
}
```
(Of course the optimizer can clean that up, but it'd be nice if it didn't have to -- especially in debug where it doesn't run.  This is like rust-lang/rust#123886, but that only handled non-simd `struct`s -- this PR generalizes it to all non-simd ADTs.)

Doing this means handing variants other than `FIRST_VARIANT`, handling the active field for unions, refactoring the discriminant code so the Place and Operand parts can share the calculation, etc.

Other PRs that led up to this one:
- https://github.com/rust-lang/rust/pull/142005
- https://github.com/rust-lang/rust/pull/142103
- https://github.com/rust-lang/rust/pull/142324
- https://github.com/rust-lang/rust/pull/142383

---

try-job: aarch64-gnu

6 files changed, 231 insertions, 10 deletions

diff --git a/tests/codegen/align-struct.rs b/tests/codegen/align-struct.rs
index 402a184d4c0..d4cc65e9158 100644
--- a/tests/codegen/align-struct.rs
+++ b/tests/codegen/align-struct.rs
@@ -15,9 +15,11 @@ pub struct Nested64 {
     d: i8,
 }
 
+// This has the extra field in B to ensure it's not ScalarPair,
+// and thus that the test actually emits it via memory, not `insertvalue`.
 pub enum Enum4 {
     A(i32),
-    B(i32),
+    B(i32, i32),
 }
 
 pub enum Enum64 {
@@ -54,7 +56,7 @@ pub fn nested64(a: Align64, b: i32, c: i32, d: i8) -> Nested64 {
 // CHECK-LABEL: @enum4
 #[no_mangle]
 pub fn enum4(a: i32) -> Enum4 {
-    // CHECK: %e4 = alloca [8 x i8], align 4
+    // CHECK: %e4 = alloca [12 x i8], align 4
     let e4 = Enum4::A(a);
     e4
 }
diff --git a/tests/codegen/common_prim_int_ptr.rs b/tests/codegen/common_prim_int_ptr.rs
index a1d7a125f32..53716adccbf 100644
--- a/tests/codegen/common_prim_int_ptr.rs
+++ b/tests/codegen/common_prim_int_ptr.rs
@@ -11,9 +11,9 @@
 #[no_mangle]
 pub fn insert_int(x: usize) -> Result<usize, Box<()>> {
     // CHECK: start:
-    // CHECK-NEXT: inttoptr i{{[0-9]+}} %x to ptr
-    // CHECK-NEXT: insertvalue
-    // CHECK-NEXT: ret { i{{[0-9]+}}, ptr }
+    // CHECK-NEXT: %[[WO_PROV:.+]] = getelementptr i8, ptr null, [[USIZE:i[0-9]+]] %x
+    // CHECK-NEXT: %[[R:.+]] = insertvalue { [[USIZE]], ptr } { [[USIZE]] 0, ptr poison }, ptr %[[WO_PROV]], 1
+    // CHECK-NEXT: ret { [[USIZE]], ptr } %[[R]]
     Ok(x)
 }
 
diff --git a/tests/codegen/enum/enum-aggregate.rs b/tests/codegen/enum/enum-aggregate.rs
new file mode 100644
index 00000000000..b6a9b8dd814
--- /dev/null
+++ b/tests/codegen/enum/enum-aggregate.rs
@@ -0,0 +1,129 @@
+//@ compile-flags: -Copt-level=0 -Cno-prepopulate-passes
+//@ min-llvm-version: 19
+//@ only-64bit
+
+#![crate_type = "lib"]
+
+use std::cmp::Ordering;
+use std::num::NonZero;
+use std::ptr::NonNull;
+
+#[no_mangle]
+fn make_some_bool(x: bool) -> Option<bool> {
+    // CHECK-LABEL: i8 @make_some_bool(i1 zeroext %x)
+    // CHECK-NEXT: start:
+    // CHECK-NEXT: %[[WIDER:.+]] = zext i1 %x to i8
+    // CHECK-NEXT: ret i8 %[[WIDER]]
+    Some(x)
+}
+
+#[no_mangle]
+fn make_none_bool() -> Option<bool> {
+    // CHECK-LABEL: i8 @make_none_bool()
+    // CHECK-NEXT: start:
+    // CHECK-NEXT: ret i8 2
+    None
+}
+
+#[no_mangle]
+fn make_some_ordering(x: Ordering) -> Option<Ordering> {
+    // CHECK-LABEL: i8 @make_some_ordering(i8 %x)
+    // CHECK-NEXT: start:
+    // CHECK-NEXT: ret i8 %x
+    Some(x)
+}
+
+#[no_mangle]
+fn make_some_u16(x: u16) -> Option<u16> {
+    // CHECK-LABEL: { i16, i16 } @make_some_u16(i16 %x)
+    // CHECK-NEXT: start:
+    // CHECK-NEXT: %0 = insertvalue { i16, i16 } { i16 1, i16 poison }, i16 %x, 1
+    // CHECK-NEXT: ret { i16, i16 } %0
+    Some(x)
+}
+
+#[no_mangle]
+fn make_none_u16() -> Option<u16> {
+    // CHECK-LABEL: { i16, i16 } @make_none_u16()
+    // CHECK-NEXT: start:
+    // CHECK-NEXT: ret { i16, i16 } { i16 0, i16 undef }
+    None
+}
+
+#[no_mangle]
+fn make_some_nzu32(x: NonZero<u32>) -> Option<NonZero<u32>> {
+    // CHECK-LABEL: i32 @make_some_nzu32(i32 %x)
+    // CHECK-NEXT: start:
+    // CHECK-NEXT: ret i32 %x
+    Some(x)
+}
+
+#[no_mangle]
+fn make_ok_ptr(x: NonNull<u16>) -> Result<NonNull<u16>, usize> {
+    // CHECK-LABEL: { i64, ptr } @make_ok_ptr(ptr %x)
+    // CHECK-NEXT: start:
+    // CHECK-NEXT: %0 = insertvalue { i64, ptr } { i64 0, ptr poison }, ptr %x, 1
+    // CHECK-NEXT: ret { i64, ptr } %0
+    Ok(x)
+}
+
+#[no_mangle]
+fn make_ok_int(x: usize) -> Result<usize, NonNull<u16>> {
+    // CHECK-LABEL: { i64, ptr } @make_ok_int(i64 %x)
+    // CHECK-NEXT: start:
+    // CHECK-NEXT: %[[NOPROV:.+]] = getelementptr i8, ptr null, i64 %x
+    // CHECK-NEXT: %[[R:.+]] = insertvalue { i64, ptr } { i64 0, ptr poison }, ptr %[[NOPROV]], 1
+    // CHECK-NEXT: ret { i64, ptr } %[[R]]
+    Ok(x)
+}
+
+#[no_mangle]
+fn make_some_ref(x: &u16) -> Option<&u16> {
+    // CHECK-LABEL: ptr @make_some_ref(ptr align 2 %x)
+    // CHECK-NEXT: start:
+    // CHECK-NEXT: ret ptr %x
+    Some(x)
+}
+
+#[no_mangle]
+fn make_none_ref<'a>() -> Option<&'a u16> {
+    // CHECK-LABEL: ptr @make_none_ref()
+    // CHECK-NEXT: start:
+    // CHECK-NEXT: ret ptr null
+    None
+}
+
+#[inline(never)]
+fn make_err_generic<E>(e: E) -> Result<u32, E> {
+    // CHECK-LABEL: define{{.+}}make_err_generic
+    // CHECK-NEXT: start:
+    // CHECK-NEXT: call void @llvm.trap()
+    // CHECK-NEXT: ret i32 poison
+    Err(e)
+}
+
+#[no_mangle]
+fn make_uninhabited_err_indirectly(n: Never) -> Result<u32, Never> {
+    // CHECK-LABEL: i32 @make_uninhabited_err_indirectly()
+    // CHECK-NEXT: start:
+    // CHECK-NEXT: call{{.+}}make_err_generic
+    make_err_generic(n)
+}
+
+#[no_mangle]
+fn make_fully_uninhabited_result(v: u32, n: Never) -> Result<(u32, Never), (Never, u32)> {
+    // We don't try to do this in SSA form since the whole type is uninhabited.
+
+    // CHECK-LABEL: { i32, i32 } @make_fully_uninhabited_result(i32 %v)
+    // CHECK: %[[ALLOC_V:.+]] = alloca [4 x i8]
+    // CHECK: %[[RET:.+]] = alloca [8 x i8]
+    // CHECK: store i32 %v, ptr %[[ALLOC_V]]
+    // CHECK: %[[TEMP_V:.+]] = load i32, ptr %[[ALLOC_V]]
+    // CHECK: %[[INNER:.+]] = getelementptr inbounds i8, ptr %[[RET]]
+    // CHECK: store i32 %[[TEMP_V]], ptr %[[INNER]]
+    // CHECK: call void @llvm.trap()
+    // CHECK: unreachable
+    Ok((v, n))
+}
+
+enum Never {}
diff --git a/tests/codegen/set-discriminant-invalid.rs b/tests/codegen/set-discriminant-invalid.rs
index 0b7cb14880c..dd584ef1c14 100644
--- a/tests/codegen/set-discriminant-invalid.rs
+++ b/tests/codegen/set-discriminant-invalid.rs
@@ -16,10 +16,9 @@ impl IntoError<Error> for Api {
     type Source = ApiError;
     // CHECK-LABEL: @into_error
     // CHECK: llvm.trap()
-    // Also check the next two instructions to make sure we do not match against `trap`
+    // Also check the next instruction to make sure we do not match against `trap`
     // elsewhere in the code.
-    // CHECK-NEXT: load
-    // CHECK-NEXT: ret
+    // CHECK-NEXT: ret i8 poison
     #[no_mangle]
     fn into_error(self, error: Self::Source) -> Error {
         Error::Api { source: error }
diff --git a/tests/codegen/union-aggregate.rs b/tests/codegen/union-aggregate.rs
new file mode 100644
index 00000000000..3c6053379fa
--- /dev/null
+++ b/tests/codegen/union-aggregate.rs
@@ -0,0 +1,85 @@
+//@ compile-flags: -Copt-level=0 -Cno-prepopulate-passes
+//@ min-llvm-version: 19
+//@ only-64bit
+
+#![crate_type = "lib"]
+#![feature(transparent_unions)]
+
+#[repr(transparent)]
+union MU<T: Copy> {
+    uninit: (),
+    value: T,
+}
+
+use std::cmp::Ordering;
+use std::num::NonZero;
+use std::ptr::NonNull;
+
+#[no_mangle]
+fn make_mu_bool(x: bool) -> MU<bool> {
+    // CHECK-LABEL: i8 @make_mu_bool(i1 zeroext %x)
+    // CHECK-NEXT: start:
+    // CHECK-NEXT: %[[WIDER:.+]] = zext i1 %x to i8
+    // CHECK-NEXT: ret i8 %[[WIDER]]
+    MU { value: x }
+}
+
+#[no_mangle]
+fn make_mu_bool_uninit() -> MU<bool> {
+    // CHECK-LABEL: i8 @make_mu_bool_uninit()
+    // CHECK-NEXT: start:
+    // CHECK-NEXT: ret i8 undef
+    MU { uninit: () }
+}
+
+#[no_mangle]
+fn make_mu_ref(x: &u16) -> MU<&u16> {
+    // CHECK-LABEL: ptr @make_mu_ref(ptr align 2 %x)
+    // CHECK-NEXT: start:
+    // CHECK-NEXT: ret ptr %x
+    MU { value: x }
+}
+
+#[no_mangle]
+fn make_mu_ref_uninit<'a>() -> MU<&'a u16> {
+    // CHECK-LABEL: ptr @make_mu_ref_uninit()
+    // CHECK-NEXT: start:
+    // CHECK-NEXT: ret ptr undef
+    MU { uninit: () }
+}
+
+#[no_mangle]
+fn make_mu_str(x: &str) -> MU<&str> {
+    // CHECK-LABEL: { ptr, i64 } @make_mu_str(ptr align 1 %x.0, i64 %x.1)
+    // CHECK-NEXT: start:
+    // CHECK-NEXT: %0 = insertvalue { ptr, i64 } poison, ptr %x.0, 0
+    // CHECK-NEXT: %1 = insertvalue { ptr, i64 } %0, i64 %x.1, 1
+    // CHECK-NEXT: ret { ptr, i64 } %1
+    MU { value: x }
+}
+
+#[no_mangle]
+fn make_mu_str_uninit<'a>() -> MU<&'a str> {
+    // CHECK-LABEL: { ptr, i64 } @make_mu_str_uninit()
+    // CHECK-NEXT: start:
+    // CHECK-NEXT: ret { ptr, i64 } undef
+    MU { uninit: () }
+}
+
+#[no_mangle]
+fn make_mu_pair(x: (u8, u32)) -> MU<(u8, u32)> {
+    // CHECK-LABEL: { i8, i32 } @make_mu_pair(i8 %x.0, i32 %x.1)
+    // CHECK-NEXT: start:
+    // CHECK-NEXT: %0 = insertvalue { i8, i32 } poison, i8 %x.0, 0
+    // CHECK-NEXT: %1 = insertvalue { i8, i32 } %0, i32 %x.1, 1
+    // CHECK-NEXT: ret { i8, i32 } %1
+    MU { value: x }
+}
+
+#[no_mangle]
+fn make_mu_pair_uninit() -> MU<(u8, u32)> {
+    // CHECK-LABEL: { i8, i32 } @make_mu_pair_uninit()
+    // CHECK-NEXT: start:
+    // CHECK-NEXT: ret { i8, i32 } undef
+    MU { uninit: () }
+}
diff --git a/tests/ui/sanitizer/memory-eager.rs b/tests/ui/sanitizer/memory-eager.rs
index 532d7b308f6..709299f87d4 100644
--- a/tests/ui/sanitizer/memory-eager.rs
+++ b/tests/ui/sanitizer/memory-eager.rs
@@ -8,8 +8,14 @@
 //
 //@ run-fail
 //@ error-pattern: MemorySanitizer: use-of-uninitialized-value
-//@ error-pattern: Uninitialized value was created by an allocation
-//@ error-pattern: in the stack frame
+//@ [optimized]error-pattern: Uninitialized value was created by an allocation
+//@ [optimized]error-pattern: in the stack frame
+//
+// FIXME the unoptimized case actually has that text in the output too, per
+// <https://github.com/rust-lang/rust/pull/138759#issuecomment-3037186707>
+// but doesn't seem to be getting picked up for some reason. For now we don't
+// check for that part, since it's still testing that memory sanitizer reported
+// a use of an uninitialized value, which is the critical part.
 //
 // This test case intentionally limits the usage of the std,
 // since it will be linked with an uninstrumented version of it.