Rollup merge of #143410 - scottmcm:redo-transmute-again, r=RalfJung,workingjubilee

Block SIMD in transmute_immediate; delete `OperandValueKind`

Vectors have been causing me problems for years in this code, for example https://github.com/rust-lang/rust/pull/110021#discussion_r1160975086 and https://github.com/rust-lang/rust/pull/143194

See conversation in <https://rust-lang.zulipchat.com/#narrow/channel/131828-t-compiler/topic/Is.20transmuting.20a.20.60T.60.20to.20.60Tx1.60.20.28one-element.20SIMD.20vector.29.20UB.3F/near/526262799>.

By blocking SIMD in `transmute_immediate` it can be simplified to just take the `Scalar`s involved -- the backend types can be gotten from those `Scalar`s, rather than needing to be passed.  And there's an assert added to ICE it if it does get hit.

Accordingly, this changes `rvalue_creates_operand` to not send SIMD transmutes through the operand path, but to always go through memory instead, like they did back before rust-lang/rust#108442.

And thanks to those changes, I could also remove the `OperandValueKind` type that I added back then which `@RalfJung` rightly considers pretty sketchy.

cc `@folkertdev` `@workingjubilee` from the zulip conversation too

5 files changed, 27 insertions, 49 deletions

diff --git a/tests/codegen/intrinsics/transmute-x64.rs b/tests/codegen/intrinsics/transmute-x64.rs
index be45e4db90f..8c9480ab091 100644
--- a/tests/codegen/intrinsics/transmute-x64.rs
+++ b/tests/codegen/intrinsics/transmute-x64.rs
@@ -9,17 +9,20 @@ use std::mem::transmute;
 // CHECK-LABEL: @check_sse_pair_to_avx(
 #[no_mangle]
 pub unsafe fn check_sse_pair_to_avx(x: (__m128i, __m128i)) -> __m256i {
+    // CHECK: start:
     // CHECK-NOT: alloca
-    // CHECK: %0 = load <4 x i64>, ptr %x, align 16
-    // CHECK: store <4 x i64> %0, ptr %_0, align 32
+    // CHECK-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr align 32 %_0, ptr align 16 %x, i64 32, i1 false)
+    // CHECK-NEXT: ret void
     transmute(x)
 }
 
 // CHECK-LABEL: @check_sse_pair_from_avx(
 #[no_mangle]
 pub unsafe fn check_sse_pair_from_avx(x: __m256i) -> (__m128i, __m128i) {
+    // CHECK: start:
     // CHECK-NOT: alloca
-    // CHECK: %0 = load <4 x i64>, ptr %x, align 32
-    // CHECK: store <4 x i64> %0, ptr %_0, align 16
+    // CHECK-NEXT: %[[TEMP:.+]] = load <4 x i64>, ptr %x, align 32
+    // CHECK-NEXT: store <4 x i64> %[[TEMP]], ptr %_0, align 16
+    // CHECK-NEXT: ret void
     transmute(x)
 }
diff --git a/tests/codegen/intrinsics/transmute.rs b/tests/codegen/intrinsics/transmute.rs
index 560ebcccdd0..e375724bc1b 100644
--- a/tests/codegen/intrinsics/transmute.rs
+++ b/tests/codegen/intrinsics/transmute.rs
@@ -29,28 +29,28 @@ pub struct Aggregate8(u8);
 // CHECK-LABEL: @check_bigger_size(
 #[no_mangle]
 pub unsafe fn check_bigger_size(x: u16) -> u32 {
-    // CHECK: call void @llvm.trap
+    // CHECK: call void @llvm.assume(i1 false)
     transmute_unchecked(x)
 }
 
 // CHECK-LABEL: @check_smaller_size(
 #[no_mangle]
 pub unsafe fn check_smaller_size(x: u32) -> u16 {
-    // CHECK: call void @llvm.trap
+    // CHECK: call void @llvm.assume(i1 false)
     transmute_unchecked(x)
 }
 
 // CHECK-LABEL: @check_smaller_array(
 #[no_mangle]
 pub unsafe fn check_smaller_array(x: [u32; 7]) -> [u32; 3] {
-    // CHECK: call void @llvm.trap
+    // CHECK: call void @llvm.assume(i1 false)
     transmute_unchecked(x)
 }
 
 // CHECK-LABEL: @check_bigger_array(
 #[no_mangle]
 pub unsafe fn check_bigger_array(x: [u32; 3]) -> [u32; 7] {
-    // CHECK: call void @llvm.trap
+    // CHECK: call void @llvm.assume(i1 false)
     transmute_unchecked(x)
 }
 
@@ -73,9 +73,9 @@ pub unsafe fn check_to_empty_array(x: [u32; 5]) -> [u32; 0] {
 #[no_mangle]
 #[custom_mir(dialect = "runtime", phase = "optimized")]
 pub unsafe fn check_from_empty_array(x: [u32; 0]) -> [u32; 5] {
-    // CHECK-NOT: trap
-    // CHECK: call void @llvm.trap
-    // CHECK-NOT: trap
+    // CHECK-NOT: call
+    // CHECK: call void @llvm.assume(i1 false)
+    // CHECK-NOT: call
     mir! {
         {
             RET = CastTransmute(x);
diff --git a/tests/codegen/simd-intrinsic/simd-intrinsic-transmute-array.rs b/tests/codegen/simd-intrinsic/simd-intrinsic-transmute-array.rs
index 977bf3379b7..301f06c2d74 100644
--- a/tests/codegen/simd-intrinsic/simd-intrinsic-transmute-array.rs
+++ b/tests/codegen/simd-intrinsic/simd-intrinsic-transmute-array.rs
@@ -40,8 +40,7 @@ pub fn build_array_s(x: [f32; 4]) -> S<4> {
 // CHECK-LABEL: @build_array_transmute_s
 #[no_mangle]
 pub fn build_array_transmute_s(x: [f32; 4]) -> S<4> {
-    // CHECK: %[[VAL:.+]] = load <4 x float>, ptr %x, align [[ARRAY_ALIGN]]
-    // CHECK: store <4 x float> %[[VAL:.+]], ptr %_0, align [[VECTOR_ALIGN]]
+    // CHECK: call void @llvm.memcpy.{{.+}}({{.*}} align [[VECTOR_ALIGN]] {{.*}} align [[ARRAY_ALIGN]] {{.*}}, [[USIZE]] 16, i1 false)
     unsafe { std::mem::transmute(x) }
 }
 
@@ -55,7 +54,6 @@ pub fn build_array_t(x: [f32; 4]) -> T {
 // CHECK-LABEL: @build_array_transmute_t
 #[no_mangle]
 pub fn build_array_transmute_t(x: [f32; 4]) -> T {
-    // CHECK: %[[VAL:.+]] = load <4 x float>, ptr %x, align [[ARRAY_ALIGN]]
-    // CHECK: store <4 x float> %[[VAL:.+]], ptr %_0, align [[VECTOR_ALIGN]]
+    // CHECK: call void @llvm.memcpy.{{.+}}({{.*}} align [[VECTOR_ALIGN]] {{.*}} align [[ARRAY_ALIGN]] {{.*}}, [[USIZE]] 16, i1 false)
     unsafe { std::mem::transmute(x) }
 }
diff --git a/tests/codegen/transmute-scalar.rs b/tests/codegen/transmute-scalar.rs
index 3ac6ba3beb1..ce1b0558b2e 100644
--- a/tests/codegen/transmute-scalar.rs
+++ b/tests/codegen/transmute-scalar.rs
@@ -111,8 +111,11 @@ pub fn fake_bool_unsigned_to_bool(b: FakeBoolUnsigned) -> bool {
 struct S([i64; 1]);
 
 // CHECK-LABEL: define{{.*}}i64 @single_element_simd_to_scalar(<1 x i64> %b)
-// CHECK: bitcast <1 x i64> %b to i64
-// CHECK: ret i64
+// CHECK-NEXT: start:
+// CHECK-NEXT: %[[RET:.+]] = alloca [8 x i8]
+// CHECK-NEXT: store <1 x i64> %b, ptr %[[RET]]
+// CHECK-NEXT: %[[TEMP:.+]] = load i64, ptr %[[RET]]
+// CHECK-NEXT: ret i64 %[[TEMP]]
 #[no_mangle]
 #[cfg_attr(target_family = "wasm", target_feature(enable = "simd128"))]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "neon"))]
@@ -124,8 +127,11 @@ pub extern "C" fn single_element_simd_to_scalar(b: S) -> i64 {
 }
 
 // CHECK-LABEL: define{{.*}}<1 x i64> @scalar_to_single_element_simd(i64 %b)
-// CHECK: bitcast i64 %b to <1 x i64>
-// CHECK: ret <1 x i64>
+// CHECK-NEXT: start:
+// CHECK-NEXT: %[[RET:.+]] = alloca [8 x i8]
+// CHECK-NEXT: store i64 %b, ptr %[[RET]]
+// CHECK-NEXT: %[[TEMP:.+]] = load <1 x i64>, ptr %[[RET]]
+// CHECK-NEXT: ret <1 x i64> %[[TEMP]]
 #[no_mangle]
 #[cfg_attr(target_family = "wasm", target_feature(enable = "simd128"))]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "neon"))]
diff --git a/tests/codegen/vec-in-place.rs b/tests/codegen/vec-in-place.rs
index 1f6836f6dfa..a5ef8653b99 100644
--- a/tests/codegen/vec-in-place.rs
+++ b/tests/codegen/vec-in-place.rs
@@ -41,9 +41,6 @@ pub fn vec_iterator_cast_primitive(vec: Vec<i8>) -> Vec<u8> {
     // CHECK: call{{.+}}void @llvm.assume(i1 %{{.+}})
     // CHECK-NOT: loop
     // CHECK-NOT: call
-    // CHECK: call{{.+}}void @llvm.assume(i1 %{{.+}})
-    // CHECK-NOT: loop
-    // CHECK-NOT: call
     vec.into_iter().map(|e| e as u8).collect()
 }
 
@@ -55,9 +52,6 @@ pub fn vec_iterator_cast_wrapper(vec: Vec<u8>) -> Vec<Wrapper<u8>> {
     // CHECK: call{{.+}}void @llvm.assume(i1 %{{.+}})
     // CHECK-NOT: loop
     // CHECK-NOT: call
-    // CHECK: call{{.+}}void @llvm.assume(i1 %{{.+}})
-    // CHECK-NOT: loop
-    // CHECK-NOT: call
     vec.into_iter().map(|e| Wrapper(e)).collect()
 }
 
@@ -86,9 +80,6 @@ pub fn vec_iterator_cast_unwrap(vec: Vec<Wrapper<u8>>) -> Vec<u8> {
     // CHECK: call{{.+}}void @llvm.assume(i1 %{{.+}})
     // CHECK-NOT: loop
     // CHECK-NOT: call
-    // CHECK: call{{.+}}void @llvm.assume(i1 %{{.+}})
-    // CHECK-NOT: loop
-    // CHECK-NOT: call
     vec.into_iter().map(|e| e.0).collect()
 }
 
@@ -100,9 +91,6 @@ pub fn vec_iterator_cast_aggregate(vec: Vec<[u64; 4]>) -> Vec<Foo> {
     // CHECK: call{{.+}}void @llvm.assume(i1 %{{.+}})
     // CHECK-NOT: loop
     // CHECK-NOT: call
-    // CHECK: call{{.+}}void @llvm.assume(i1 %{{.+}})
-    // CHECK-NOT: loop
-    // CHECK-NOT: call
     vec.into_iter().map(|e| unsafe { std::mem::transmute(e) }).collect()
 }
 
@@ -114,9 +102,6 @@ pub fn vec_iterator_cast_deaggregate_tra(vec: Vec<Bar>) -> Vec<[u64; 4]> {
     // CHECK: call{{.+}}void @llvm.assume(i1 %{{.+}})
     // CHECK-NOT: loop
     // CHECK-NOT: call
-    // CHECK: call{{.+}}void @llvm.assume(i1 %{{.+}})
-    // CHECK-NOT: loop
-    // CHECK-NOT: call
 
     // Safety: For the purpose of this test we assume that Bar layout matches [u64; 4].
     // This currently is not guaranteed for repr(Rust) types, but it happens to work here and
@@ -133,9 +118,6 @@ pub fn vec_iterator_cast_deaggregate_fold(vec: Vec<Baz>) -> Vec<[u64; 4]> {
     // CHECK: call{{.+}}void @llvm.assume(i1 %{{.+}})
     // CHECK-NOT: loop
     // CHECK-NOT: call
-    // CHECK: call{{.+}}void @llvm.assume(i1 %{{.+}})
-    // CHECK-NOT: loop
-    // CHECK-NOT: call
 
     // Safety: For the purpose of this test we assume that Bar layout matches [u64; 4].
     // This currently is not guaranteed for repr(Rust) types, but it happens to work here and
@@ -156,12 +138,7 @@ pub fn vec_iterator_cast_unwrap_drop(vec: Vec<Wrapper<String>>) -> Vec<String> {
     // CHECK-NOT: call
     // CHECK-NOT: %{{.*}} = mul
     // CHECK-NOT: %{{.*}} = udiv
-    // CHECK: call
-    // CHECK-SAME: void @llvm.assume(i1 %{{.+}})
-    // CHECK-NOT: br i1 %{{.*}}, label %{{.*}}, label %{{.*}}
-    // CHECK-NOT: call
-    // CHECK-NOT: %{{.*}} = mul
-    // CHECK-NOT: %{{.*}} = udiv
+    // CHECK: ret void
 
     vec.into_iter().map(|Wrapper(e)| e).collect()
 }
@@ -178,12 +155,6 @@ pub fn vec_iterator_cast_wrap_drop(vec: Vec<String>) -> Vec<Wrapper<String>> {
     // CHECK-NOT: call
     // CHECK-NOT: %{{.*}} = mul
     // CHECK-NOT: %{{.*}} = udiv
-    // CHECK: call
-    // CHECK-SAME: void @llvm.assume(i1 %{{.+}})
-    // CHECK-NOT: br i1 %{{.*}}, label %{{.*}}, label %{{.*}}
-    // CHECK-NOT: call
-    // CHECK-NOT: %{{.*}} = mul
-    // CHECK-NOT: %{{.*}} = udiv
     // CHECK: ret void
 
     vec.into_iter().map(Wrapper).collect()