Auto merge of #111010 - scottmcm:mem-replace-simpler, r=WaffleLapkin

Make `mem::replace` simpler in codegen

Since they'd mentioned more intrinsics for simplifying stuff recently,
r? `@WaffleLapkin`

This is a continuation of me looking at foundational stuff that ends up with more instructions than it really needs.  Specifically I noticed this one because `Range::next` isn't MIR-inlining, and one of the largest parts of it is a `replace::<usize>` that's a good dozen instructions instead of the two it could be.

So this means that `ptr::write` with a `Copy` type no longer generates worse IR than manually dereferencing (well, at least in LLVM -- MIR still has bonus pointer casts), and in doing so means that we're finally down to just the two essential `memcpy`s when emitting `mem::replace` for a large type, rather than the bonus-`alloca` and three `memcpy`s we emitted before this ([or the 6 we currently emit in 1.69 stable](https://rust.godbolt.org/z/67W8on6nP)).  That said, LLVM does _usually_ manage to optimize the extra code away.  But it's still nice for it not to have to do as much, thanks to (for example) not going through an `alloca` when `replace`ing a primitive like a `usize`.

(This is a new intrinsic, but one that's immediately lowered to existing MIR constructs, so not anything that MIRI or the codegen backends or MIR semantics needs to do work to handle.)

3 files changed, 40 insertions, 41 deletions

diff --git a/tests/codegen/mem-replace-big-type.rs b/tests/codegen/mem-replace-big-type.rs
index f6898e2f758..81e56b5490d 100644
--- a/tests/codegen/mem-replace-big-type.rs
+++ b/tests/codegen/mem-replace-big-type.rs
@@ -11,7 +11,9 @@
 #[repr(C, align(8))]
 pub struct Big([u64; 7]);
 pub fn replace_big(dst: &mut Big, src: Big) -> Big {
-    // Before the `read_via_copy` intrinsic, this emitted six `memcpy`s.
+    // Back in 1.68, this emitted six `memcpy`s.
+    // `read_via_copy` in 1.69 got that down to three.
+    // `write_via_move` has it down to just the two essential ones.
     std::mem::replace(dst, src)
 }
 
@@ -20,17 +22,13 @@ pub fn replace_big(dst: &mut Big, src: Big) -> Big {
 
 // CHECK-NOT: call void @llvm.memcpy
 
-// For a large type, we expect exactly three `memcpy`s
+// For a large type, we expect exactly two `memcpy`s
 // CHECK-LABEL: define internal void @{{.+}}mem{{.+}}replace{{.+}}sret(%Big)
     // CHECK-NOT: alloca
-    // CHECK: alloca %Big
-    // CHECK-NOT: alloca
-    // CHECK-NOT: call void @llvm.memcpy
-    // CHECK: call void @llvm.memcpy.{{.+}}({{i8\*|ptr}} align 8 %{{.*}}, {{i8\*|ptr}} align 8 %{{.*}}, i{{.*}} 56, i1 false)
     // CHECK-NOT: call void @llvm.memcpy
-    // CHECK: call void @llvm.memcpy.{{.+}}({{i8\*|ptr}} align 8 %{{.*}}, {{i8\*|ptr}} align 8 %{{.*}}, i{{.*}} 56, i1 false)
+    // CHECK: call void @llvm.memcpy.{{.+}}({{i8\*|ptr}} align 8 %0, {{i8\*|ptr}} align 8 %dest, i{{.*}} 56, i1 false)
     // CHECK-NOT: call void @llvm.memcpy
-    // CHECK: call void @llvm.memcpy.{{.+}}({{i8\*|ptr}} align 8 %{{.*}}, {{i8\*|ptr}} align 8 %{{.*}}, i{{.*}} 56, i1 false)
+    // CHECK: call void @llvm.memcpy.{{.+}}({{i8\*|ptr}} align 8 %dest, {{i8\*|ptr}} align 8 %src, i{{.*}} 56, i1 false)
     // CHECK-NOT: call void @llvm.memcpy
 
 // CHECK-NOT: call void @llvm.memcpy
diff --git a/tests/codegen/mem-replace-direct-memcpy.rs b/tests/codegen/mem-replace-direct-memcpy.rs
deleted file mode 100644
index 83babab4f84..00000000000
--- a/tests/codegen/mem-replace-direct-memcpy.rs
+++ /dev/null
@@ -1,33 +0,0 @@
-// This test ensures that `mem::replace::<T>` only ever calls `@llvm.memcpy`
-// with `size_of::<T>()` as the size, and never goes through any wrapper that
-// may e.g. multiply `size_of::<T>()` with a variable "count" (which is only
-// known to be `1` after inlining).
-
-// compile-flags: -C no-prepopulate-passes -Zinline-mir=no
-// ignore-debug: the debug assertions get in the way
-
-#![crate_type = "lib"]
-
-pub fn replace_byte(dst: &mut u8, src: u8) -> u8 {
-    std::mem::replace(dst, src)
-}
-
-// NOTE(eddyb) the `CHECK-NOT`s ensure that the only calls of `@llvm.memcpy` in
-// the entire output, are the direct calls we want, from `ptr::replace`.
-
-// CHECK-NOT: call void @llvm.memcpy
-
-// For a small type, we expect one each of `load`/`store`/`memcpy` instead
-// CHECK-LABEL: define internal noundef i8 @{{.+}}mem{{.+}}replace
-    // CHECK-NOT: alloca
-    // CHECK: alloca i8
-    // CHECK-NOT: alloca
-    // CHECK-NOT: call void @llvm.memcpy
-    // CHECK: load i8
-    // CHECK-NOT: call void @llvm.memcpy
-    // CHECK: store i8
-    // CHECK-NOT: call void @llvm.memcpy
-    // CHECK: call void @llvm.memcpy.{{.+}}({{i8\*|ptr}} align 1 %{{.*}}, {{i8\*|ptr}} align 1 %{{.*}}, i{{.*}} 1, i1 false)
-    // CHECK-NOT: call void @llvm.memcpy
-
-// CHECK-NOT: call void @llvm.memcpy
diff --git a/tests/codegen/mem-replace-simple-type.rs b/tests/codegen/mem-replace-simple-type.rs
new file mode 100644
index 00000000000..4253ef13666
--- /dev/null
+++ b/tests/codegen/mem-replace-simple-type.rs
@@ -0,0 +1,34 @@
+// compile-flags: -O -C no-prepopulate-passes
+// min-llvm-version: 15.0 (for opaque pointers)
+// only-x86_64 (to not worry about usize differing)
+// ignore-debug (the debug assertions get in the way)
+
+#![crate_type = "lib"]
+
+#[no_mangle]
+// CHECK-LABEL: @replace_usize(
+pub fn replace_usize(r: &mut usize, v: usize) -> usize {
+    // CHECK-NOT: alloca
+    // CHECK: %[[R:.+]] = load i64, ptr %r
+    // CHECK: store i64 %v, ptr %r
+    // CHECK: ret i64 %[[R]]
+    std::mem::replace(r, v)
+}
+
+#[no_mangle]
+// CHECK-LABEL: @replace_ref_str(
+pub fn replace_ref_str<'a>(r: &mut &'a str, v: &'a str) -> &'a str {
+    // CHECK-NOT: alloca
+    // CHECK: %[[A:.+]] = load ptr
+    // CHECK: %[[B:.+]] = load i64
+    // CHECK-NOT: store
+    // CHECK-NOT: load
+    // CHECK: store ptr
+    // CHECK: store i64
+    // CHECK-NOT: load
+    // CHECK-NOT: store
+    // CHECK: %[[P1:.+]] = insertvalue { ptr, i64 } poison, ptr %[[A]], 0
+    // CHECK: %[[P2:.+]] = insertvalue { ptr, i64 } %[[P1]], i64 %[[B]], 1
+    // CHECK: ret { ptr, i64 } %[[P2]]
+    std::mem::replace(r, v)
+}