Merge from rustc

36 files changed, 1043 insertions, 127 deletions

diff --git a/tests/codegen/align-fn.rs b/tests/codegen/align-fn.rs
index 97f23cc0423..660d8cd2bbf 100644
--- a/tests/codegen/align-fn.rs
+++ b/tests/codegen/align-fn.rs
@@ -47,3 +47,22 @@ impl T for () {}
 pub fn foo() {
     ().trait_method();
 }
+
+// CHECK-LABEL: align_specified_twice_1
+// CHECK-SAME: align 64
+#[no_mangle]
+#[repr(align(32), align(64))]
+pub fn align_specified_twice_1() {}
+
+// CHECK-LABEL: align_specified_twice_2
+// CHECK-SAME: align 128
+#[no_mangle]
+#[repr(align(128), align(32))]
+pub fn align_specified_twice_2() {}
+
+// CHECK-LABEL: align_specified_twice_3
+// CHECK-SAME: align 256
+#[no_mangle]
+#[repr(align(32))]
+#[repr(align(256))]
+pub fn align_specified_twice_3() {}
diff --git a/tests/codegen/array-cmp.rs b/tests/codegen/array-cmp.rs
index 2565a385b61..0d337655401 100644
--- a/tests/codegen/array-cmp.rs
+++ b/tests/codegen/array-cmp.rs
@@ -1,6 +1,7 @@
 // Ensure the asm for array comparisons is properly optimized.
 
 //@ compile-flags: -C opt-level=2
+//@ needs-deterministic-layouts (checks depend on tuple layout)
 
 #![crate_type = "lib"]
 
@@ -17,3 +18,57 @@ pub fn compare() -> bool {
             [0x00, 0x00, 0x48, 0x41]
         }
 }
+
+// CHECK-LABEL: @array_of_tuple_le
+#[no_mangle]
+pub fn array_of_tuple_le(a: &[(i16, u16); 2], b: &[(i16, u16); 2]) -> bool {
+    // Ensure that, after all the optimizations have run, the happy path just checks
+    // `eq` on each corresponding pair and moves onto the next one if it is.
+    // Then there's a dedup'd comparison for the place that's different.
+    // (As opposed to, say, running a full `[su]cmp` as part of checking equality.)
+
+    // This is written quite specifically because different library code was triggering
+    // <https://github.com/llvm/llvm-project/issues/132678> along the way, so this
+    // has enough checks to make sure that's not happening. It doesn't need to be
+    // *exactly* this IR, but be careful if you ever need to update these checks.
+
+    // CHECK: start:
+    // CHECK: %[[A00:.+]] = load i16, ptr %a
+    // CHECK: %[[B00:.+]] = load i16, ptr %b
+    // CHECK-NOT: cmp
+    // CHECK: %[[EQ00:.+]] = icmp eq i16 %[[A00]], %[[B00]]
+    // CHECK-NEXT: br i1 %[[EQ00]], label %[[L01:.+]], label %[[EXIT_S:.+]]
+
+    // CHECK: [[L01]]:
+    // CHECK: %[[PA01:.+]] = getelementptr{{.+}}i8, ptr %a, {{i32|i64}} 2
+    // CHECK: %[[PB01:.+]] = getelementptr{{.+}}i8, ptr %b, {{i32|i64}} 2
+    // CHECK: %[[A01:.+]] = load i16, ptr %[[PA01]]
+    // CHECK: %[[B01:.+]] = load i16, ptr %[[PB01]]
+    // CHECK-NOT: cmp
+    // CHECK: %[[EQ01:.+]] = icmp eq i16 %[[A01]], %[[B01]]
+    // CHECK-NEXT: br i1 %[[EQ01]], label %[[L10:.+]], label %[[EXIT_U:.+]]
+
+    // CHECK: [[L10]]:
+    // CHECK: %[[PA10:.+]] = getelementptr{{.+}}i8, ptr %a, {{i32|i64}} 4
+    // CHECK: %[[PB10:.+]] = getelementptr{{.+}}i8, ptr %b, {{i32|i64}} 4
+    // CHECK: %[[A10:.+]] = load i16, ptr %[[PA10]]
+    // CHECK: %[[B10:.+]] = load i16, ptr %[[PB10]]
+    // CHECK-NOT: cmp
+    // CHECK: %[[EQ10:.+]] = icmp eq i16 %[[A10]], %[[B10]]
+    // CHECK-NEXT: br i1 %[[EQ10]], label %[[L11:.+]], label %[[EXIT_S]]
+
+    // CHECK: [[L11]]:
+    // CHECK: %[[PA11:.+]] = getelementptr{{.+}}i8, ptr %a, {{i32|i64}} 6
+    // CHECK: %[[PB11:.+]] = getelementptr{{.+}}i8, ptr %b, {{i32|i64}} 6
+    // CHECK: %[[A11:.+]] = load i16, ptr %[[PA11]]
+    // CHECK: %[[B11:.+]] = load i16, ptr %[[PB11]]
+    // CHECK-NOT: cmp
+    // CHECK: %[[EQ11:.+]] = icmp eq i16 %[[A11]], %[[B11]]
+    // CHECK-NEXT: br i1 %[[EQ11]], label %[[DONE:.+]], label %[[EXIT_U]]
+
+    // CHECK: [[DONE]]:
+    // CHECK: %[[RET:.+]] = phi i1 [ %{{.+}}, %[[EXIT_S]] ], [ %{{.+}}, %[[EXIT_U]] ], [ true, %[[L11]] ]
+    // CHECK: ret i1 %[[RET]]
+
+    a <= b
+}
diff --git a/tests/codegen/async-closure-debug.rs b/tests/codegen/async-closure-debug.rs
index 2d67e02eb9c..b5b369e6e54 100644
--- a/tests/codegen/async-closure-debug.rs
+++ b/tests/codegen/async-closure-debug.rs
@@ -1,6 +1,7 @@
 // Just make sure that async closures don't ICE.
 //
-//@ compile-flags: -C debuginfo=2 --edition=2018
+//@ compile-flags: -C debuginfo=2
+//@ edition: 2018
 //@ ignore-msvc
 
 // CHECK-DAG:  [[GEN_FN:!.*]] = !DINamespace(name: "async_closure_test"
diff --git a/tests/codegen/async-fn-debug-awaitee-field.rs b/tests/codegen/async-fn-debug-awaitee-field.rs
index ab13d4509e2..50860c90662 100644
--- a/tests/codegen/async-fn-debug-awaitee-field.rs
+++ b/tests/codegen/async-fn-debug-awaitee-field.rs
@@ -7,7 +7,8 @@
 //@[MSVC] only-msvc
 //@[NONMSVC] ignore-msvc
 
-//@ compile-flags: -C debuginfo=2 --edition=2018 -Copt-level=0
+//@ compile-flags: -C debuginfo=2 -Copt-level=0
+//@ edition: 2018
 
 #![crate_type = "lib"]
 
diff --git a/tests/codegen/async-fn-debug-msvc.rs b/tests/codegen/async-fn-debug-msvc.rs
index 7c695042b42..e0c601146f8 100644
--- a/tests/codegen/async-fn-debug-msvc.rs
+++ b/tests/codegen/async-fn-debug-msvc.rs
@@ -4,7 +4,8 @@
 //  - Other fields are not marked artificial
 //
 //
-//@ compile-flags: -C debuginfo=2 --edition=2018
+//@ compile-flags: -C debuginfo=2
+//@ edition: 2018
 //@ only-msvc
 
 async fn foo() {}
@@ -19,23 +20,23 @@ async fn async_fn_test() {
 // CHECK-DAG:  [[GEN:!.*]] = !DICompositeType(tag: DW_TAG_union_type, name: "enum2$<async_fn_debug_msvc::async_fn_test::async_fn_env$0>",
 // CHECK:      {{!.*}} = !DIDerivedType(tag: DW_TAG_member, name: "variant0", scope: [[GEN]],
 // For brevity, we only check the struct name and members of the last variant.
-// CHECK-SAME: file: [[FILE:![0-9]*]], line: 11,
+// CHECK-SAME: file: [[FILE:![0-9]*]], line: 12,
 // CHECK-NOT:  flags: DIFlagArtificial
 // CHECK-SAME: )
 // CHECK:      {{!.*}} = !DIDerivedType(tag: DW_TAG_member, name: "variant1", scope: [[GEN]],
-// CHECK-SAME: file: [[FILE]], line: 15,
+// CHECK-SAME: file: [[FILE]], line: 16,
 // CHECK-NOT:  flags: DIFlagArtificial
 // CHECK-SAME: )
 // CHECK:      {{!.*}} = !DIDerivedType(tag: DW_TAG_member, name: "variant2", scope: [[GEN]],
-// CHECK-SAME: file: [[FILE]], line: 15,
+// CHECK-SAME: file: [[FILE]], line: 16,
 // CHECK-NOT:  flags: DIFlagArtificial
 // CHECK-SAME: )
 // CHECK:      {{!.*}} = !DIDerivedType(tag: DW_TAG_member, name: "variant3", scope: [[GEN]],
-// CHECK-SAME: file: [[FILE]], line: 12,
+// CHECK-SAME: file: [[FILE]], line: 13,
 // CHECK-NOT:  flags: DIFlagArtificial
 // CHECK-SAME: )
 // CHECK:      {{!.*}} = !DIDerivedType(tag: DW_TAG_member, name: "variant4", scope: [[GEN]],
-// CHECK-SAME: file: [[FILE]], line: 14,
+// CHECK-SAME: file: [[FILE]], line: 15,
 // CHECK-SAME: baseType: [[VARIANT_WRAPPER:![0-9]*]]
 // CHECK-NOT:  flags: DIFlagArtificial
 // CHECK-SAME: )
diff --git a/tests/codegen/async-fn-debug.rs b/tests/codegen/async-fn-debug.rs
index 7be4ad45665..ed704c7cc8b 100644
--- a/tests/codegen/async-fn-debug.rs
+++ b/tests/codegen/async-fn-debug.rs
@@ -4,7 +4,8 @@
 //  - Other fields are not marked artificial
 //
 //
-//@ compile-flags: -C debuginfo=2 --edition=2018
+//@ compile-flags: -C debuginfo=2
+//@ edition: 2018
 //@ ignore-msvc
 
 async fn foo() {}
@@ -22,26 +23,26 @@ async fn async_fn_test() {
 // CHECK-NOT:  flags: DIFlagArtificial
 // CHECK-SAME: discriminator: [[DISC:![0-9]*]]
 // CHECK:      {{!.*}} = !DIDerivedType(tag: DW_TAG_member, name: "0", scope: [[VARIANT]],
-// CHECK-SAME: file: [[FILE:![0-9]*]], line: 11,
+// CHECK-SAME: file: [[FILE:![0-9]*]], line: 12,
 // CHECK-NOT:  flags: DIFlagArtificial
 // CHECK-SAME: )
 // CHECK:      {{!.*}} = !DICompositeType(tag: DW_TAG_structure_type, name: "Unresumed", scope: [[GEN]],
 // CHECK-NOT:  flags: DIFlagArtificial
 // CHECK-SAME: )
 // CHECK:      {{!.*}} = !DIDerivedType(tag: DW_TAG_member, name: "1", scope: [[VARIANT]],
-// CHECK-SAME: file: [[FILE]], line: 15,
+// CHECK-SAME: file: [[FILE]], line: 16,
 // CHECK-NOT:  flags: DIFlagArtificial
 // CHECK-SAME: )
 // CHECK:      {{!.*}} = !DIDerivedType(tag: DW_TAG_member, name: "2", scope: [[VARIANT]],
-// CHECK-SAME: file: [[FILE]], line: 15,
+// CHECK-SAME: file: [[FILE]], line: 16,
 // CHECK-NOT:  flags: DIFlagArtificial
 // CHECK-SAME: )
 // CHECK:      {{!.*}} = !DIDerivedType(tag: DW_TAG_member, name: "3", scope: [[VARIANT]],
-// CHECK-SAME: file: [[FILE]], line: 12,
+// CHECK-SAME: file: [[FILE]], line: 13,
 // CHECK-NOT:  flags: DIFlagArtificial
 // CHECK-SAME: )
 // CHECK:      {{!.*}} = !DIDerivedType(tag: DW_TAG_member, name: "4", scope: [[VARIANT]],
-// CHECK-SAME: file: [[FILE]], line: 14,
+// CHECK-SAME: file: [[FILE]], line: 15,
 // CHECK-NOT:  flags: DIFlagArtificial
 // CHECK-SAME: )
 // CHECK:      [[S1:!.*]] = !DICompositeType(tag: DW_TAG_structure_type, name: "Suspend1", scope: [[GEN]],
diff --git a/tests/codegen/autodiffv.rs b/tests/codegen/autodiff/batched.rs
index e0047116405..e0047116405 100644
--- a/tests/codegen/autodiffv.rs
+++ b/tests/codegen/autodiff/batched.rs
diff --git a/tests/codegen/autodiff.rs b/tests/codegen/autodiff/scalar.rs
index 85358f5fcb6..85358f5fcb6 100644
--- a/tests/codegen/autodiff.rs
+++ b/tests/codegen/autodiff/scalar.rs
diff --git a/tests/codegen/autodiff/sret.rs b/tests/codegen/autodiff/sret.rs
new file mode 100644
index 00000000000..5ead90041ed
--- /dev/null
+++ b/tests/codegen/autodiff/sret.rs
@@ -0,0 +1,45 @@
+//@ compile-flags: -Zautodiff=Enable -C opt-level=3  -Clto=fat
+//@ no-prefer-dynamic
+//@ needs-enzyme
+
+// This test is almost identical to the scalar.rs one,
+// but we intentionally add a few more floats.
+// `df` would ret `{ f64, f32, f32 }`, but is lowered as an sret.
+// We therefore use this test to verify some of our sret handling.
+
+#![feature(autodiff)]
+
+use std::autodiff::autodiff;
+
+#[no_mangle]
+#[autodiff(df, Reverse, Active, Active, Active)]
+fn primal(x: f32, y: f32) -> f64 {
+    (x * x * y) as f64
+}
+
+// CHECK:define internal fastcc void @_ZN4sret2df17h93be4316dd8ea006E(ptr dead_on_unwind noalias nocapture noundef nonnull writable writeonly align 8 dereferenceable(16) initializes((0, 16)) %_0, float noundef %x, float noundef %y)
+// CHECK-NEXT:start:
+// CHECK-NEXT:  %0 = tail call fastcc { double, float, float } @diffeprimal(float %x, float %y)
+// CHECK-NEXT:  %.elt = extractvalue { double, float, float } %0, 0
+// CHECK-NEXT:  store double %.elt, ptr %_0, align 8
+// CHECK-NEXT:  %_0.repack1 = getelementptr inbounds nuw i8, ptr %_0, i64 8
+// CHECK-NEXT:  %.elt2 = extractvalue { double, float, float } %0, 1
+// CHECK-NEXT:  store float %.elt2, ptr %_0.repack1, align 8
+// CHECK-NEXT:  %_0.repack3 = getelementptr inbounds nuw i8, ptr %_0, i64 12
+// CHECK-NEXT:  %.elt4 = extractvalue { double, float, float } %0, 2
+// CHECK-NEXT:  store float %.elt4, ptr %_0.repack3, align 4
+// CHECK-NEXT:  ret void
+// CHECK-NEXT:}
+
+fn main() {
+    let x = std::hint::black_box(3.0);
+    let y = std::hint::black_box(2.5);
+    let scalar = std::hint::black_box(1.0);
+    let (r1, r2, r3) = df(x, y, scalar);
+    // 3*3*1.5 = 22.5
+    assert_eq!(r1, 22.5);
+    // 2*x*y = 2*3*2.5 = 15.0
+    assert_eq!(r2, 15.0);
+    // x*x*1 = 3*3 = 9
+    assert_eq!(r3, 9.0);
+}
diff --git a/tests/codegen/autodiffv2.rs b/tests/codegen/autodiffv2.rs
new file mode 100644
index 00000000000..a40d19d3be3
--- /dev/null
+++ b/tests/codegen/autodiffv2.rs
@@ -0,0 +1,113 @@
+//@ compile-flags: -Zautodiff=Enable -C opt-level=3  -Clto=fat
+//@ no-prefer-dynamic
+//@ needs-enzyme
+//
+// In Enzyme, we test against a large range of LLVM versions (5+) and don't have overly many
+// breakages. One benefit is that we match the IR generated by Enzyme only after running it
+// through LLVM's O3 pipeline, which will remove most of the noise.
+// However, our integration test could also be affected by changes in how rustc lowers MIR into
+// LLVM-IR, which could cause additional noise and thus breakages. If that's the case, we should
+// reduce this test to only match the first lines and the ret instructions.
+//
+// The function tested here has 4 inputs and 5 outputs, so we could either call forward-mode
+// autodiff 4 times, or reverse mode 5 times. Since a forward-mode call is usually faster than
+// reverse mode, we prefer it here. This file also tests a new optimization (batch mode), which
+// allows us to call forward-mode autodiff only once, and get all 5 outputs in a single call.
+//
+// We support 2 different batch modes. `d_square2` has the same interface as scalar forward-mode,
+// but each shadow argument is `width` times larger (thus 16 and 20 elements here).
+// `d_square3` instead takes `width` (4) shadow arguments, which are all the same size as the
+// original function arguments.
+//
+// FIXME(autodiff): We currently can't test `d_square1` and `d_square3` in the same file, since they
+// generate the same dummy functions which get merged by LLVM, breaking pieces of our pipeline which
+// try to rewrite the dummy functions later. We should consider to change to pure declarations both
+// in our frontend and in the llvm backend to avoid these issues.
+
+#![feature(autodiff)]
+
+use std::autodiff::autodiff;
+
+#[no_mangle]
+//#[autodiff(d_square1, Forward, Dual, Dual)]
+#[autodiff(d_square2, Forward, 4, Dualv, Dualv)]
+#[autodiff(d_square3, Forward, 4, Dual, Dual)]
+fn square(x: &[f32], y: &mut [f32]) {
+    assert!(x.len() >= 4);
+    assert!(y.len() >= 5);
+    y[0] = 4.3 * x[0] + 1.2 * x[1] + 3.4 * x[2] + 2.1 * x[3];
+    y[1] = 2.3 * x[0] + 4.5 * x[1] + 1.7 * x[2] + 6.4 * x[3];
+    y[2] = 1.1 * x[0] + 3.3 * x[1] + 2.5 * x[2] + 4.7 * x[3];
+    y[3] = 5.2 * x[0] + 1.4 * x[1] + 2.6 * x[2] + 3.8 * x[3];
+    y[4] = 1.0 * x[0] + 2.0 * x[1] + 3.0 * x[2] + 4.0 * x[3];
+}
+
+fn main() {
+    let x1 = std::hint::black_box(vec![0.0, 1.0, 2.0, 3.0]);
+
+    let dx1 = std::hint::black_box(vec![1.0; 12]);
+
+    let z1 = std::hint::black_box(vec![1.0, 0.0, 0.0, 0.0]);
+    let z2 = std::hint::black_box(vec![0.0, 1.0, 0.0, 0.0]);
+    let z3 = std::hint::black_box(vec![0.0, 0.0, 1.0, 0.0]);
+    let z4 = std::hint::black_box(vec![0.0, 0.0, 0.0, 1.0]);
+
+    let z5 = std::hint::black_box(vec![
+        1.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 1.0,
+    ]);
+
+    let mut y1 = std::hint::black_box(vec![0.0; 5]);
+    let mut y2 = std::hint::black_box(vec![0.0; 5]);
+    let mut y3 = std::hint::black_box(vec![0.0; 5]);
+    let mut y4 = std::hint::black_box(vec![0.0; 5]);
+
+    let mut y5 = std::hint::black_box(vec![0.0; 5]);
+
+    let mut y6 = std::hint::black_box(vec![0.0; 5]);
+
+    let mut dy1_1 = std::hint::black_box(vec![0.0; 5]);
+    let mut dy1_2 = std::hint::black_box(vec![0.0; 5]);
+    let mut dy1_3 = std::hint::black_box(vec![0.0; 5]);
+    let mut dy1_4 = std::hint::black_box(vec![0.0; 5]);
+
+    let mut dy2 = std::hint::black_box(vec![0.0; 20]);
+
+    let mut dy3_1 = std::hint::black_box(vec![0.0; 5]);
+    let mut dy3_2 = std::hint::black_box(vec![0.0; 5]);
+    let mut dy3_3 = std::hint::black_box(vec![0.0; 5]);
+    let mut dy3_4 = std::hint::black_box(vec![0.0; 5]);
+
+    // scalar.
+    //d_square1(&x1, &z1, &mut y1, &mut dy1_1);
+    //d_square1(&x1, &z2, &mut y2, &mut dy1_2);
+    //d_square1(&x1, &z3, &mut y3, &mut dy1_3);
+    //d_square1(&x1, &z4, &mut y4, &mut dy1_4);
+
+    // assert y1 == y2 == y3 == y4
+    //for i in 0..5 {
+    //    assert_eq!(y1[i], y2[i]);
+    //    assert_eq!(y1[i], y3[i]);
+    //    assert_eq!(y1[i], y4[i]);
+    //}
+
+    // batch mode A)
+    d_square2(&x1, &z5, &mut y5, &mut dy2);
+
+    // assert y1 == y2 == y3 == y4 == y5
+    //for i in 0..5 {
+    //    assert_eq!(y1[i], y5[i]);
+    //}
+
+    // batch mode B)
+    d_square3(&x1, &z1, &z2, &z3, &z4, &mut y6, &mut dy3_1, &mut dy3_2, &mut dy3_3, &mut dy3_4);
+    for i in 0..5 {
+        assert_eq!(y5[i], y6[i]);
+    }
+
+    for i in 0..5 {
+        assert_eq!(dy2[0..5][i], dy3_1[i]);
+        assert_eq!(dy2[5..10][i], dy3_2[i]);
+        assert_eq!(dy2[10..15][i], dy3_3[i]);
+        assert_eq!(dy2[15..20][i], dy3_4[i]);
+    }
+}
diff --git a/tests/codegen/avr/avr-func-addrspace.rs b/tests/codegen/avr/avr-func-addrspace.rs
index 2ae2f40d7b3..e0192f8b45a 100644
--- a/tests/codegen/avr/avr-func-addrspace.rs
+++ b/tests/codegen/avr/avr-func-addrspace.rs
@@ -17,9 +17,8 @@
 extern crate minicore;
 use minicore::*;
 
-extern "rust-intrinsic" {
-    pub fn transmute<Src, Dst>(src: Src) -> Dst;
-}
+#[rustc_intrinsic]
+pub unsafe fn transmute<Src, Dst>(src: Src) -> Dst;
 
 pub static mut STORAGE_FOO: fn(&usize, &mut u32) -> Result<(), ()> = arbitrary_black_box;
 pub static mut STORAGE_BAR: u32 = 12;
diff --git a/tests/codegen/cffi/ffi-const.rs b/tests/codegen/cffi/ffi-const.rs
index 6c90902e89f..3ea9d517ec2 100644
--- a/tests/codegen/cffi/ffi-const.rs
+++ b/tests/codegen/cffi/ffi-const.rs
@@ -10,6 +10,6 @@ extern "C" {
     // CHECK-LABEL: declare{{.*}}void @foo()
     // CHECK-SAME: [[ATTRS:#[0-9]+]]
     // CHECK-DAG: attributes [[ATTRS]] = { {{.*}}memory(none){{.*}} }
-    #[ffi_const]
+    #[unsafe(ffi_const)]
     pub fn foo();
 }
diff --git a/tests/codegen/cffi/ffi-pure.rs b/tests/codegen/cffi/ffi-pure.rs
index 2c5d5f5b4b1..a61e80ecf65 100644
--- a/tests/codegen/cffi/ffi-pure.rs
+++ b/tests/codegen/cffi/ffi-pure.rs
@@ -10,6 +10,6 @@ extern "C" {
     // CHECK-LABEL: declare{{.*}}void @foo()
     // CHECK-SAME: [[ATTRS:#[0-9]+]]
     // CHECK-DAG: attributes [[ATTRS]] = { {{.*}}memory(read){{.*}} }
-    #[ffi_pure]
+    #[unsafe(ffi_pure)]
     pub fn foo();
 }
diff --git a/tests/codegen/coroutine-debug.rs b/tests/codegen/coroutine-debug.rs
index d00667a37d5..ff62e9709b4 100644
--- a/tests/codegen/coroutine-debug.rs
+++ b/tests/codegen/coroutine-debug.rs
@@ -4,7 +4,8 @@
 //  - Other fields are not marked artificial
 //
 //
-//@ compile-flags: -C debuginfo=2 --edition=2018
+//@ compile-flags: -C debuginfo=2
+//@ edition: 2018
 //@ ignore-msvc
 
 #![feature(coroutines, coroutine_trait)]
@@ -27,26 +28,26 @@ fn coroutine_test() -> impl Coroutine<Yield = i32, Return = ()> {
 // CHECK-NOT:  flags: DIFlagArtificial
 // CHECK-SAME: discriminator: [[DISC:![0-9]*]]
 // CHECK:      {{!.*}} = !DIDerivedType(tag: DW_TAG_member, name: "0", scope: [[VARIANT]],
-// CHECK-SAME: file: [[FILE:![0-9]*]], line: 15,
+// CHECK-SAME: file: [[FILE:![0-9]*]], line: 16,
 // CHECK-NOT:  flags: DIFlagArtificial
 // CHECK-SAME: )
 // CHECK:      {{!.*}} = !DICompositeType(tag: DW_TAG_structure_type, name: "Unresumed", scope: [[GEN]],
 // CHECK-NOT:  flags: DIFlagArtificial
 // CHECK-SAME: )
 // CHECK:      {{!.*}} = !DIDerivedType(tag: DW_TAG_member, name: "1", scope: [[VARIANT]],
-// CHECK-SAME: file: [[FILE]], line: 19,
+// CHECK-SAME: file: [[FILE]], line: 20,
 // CHECK-NOT:  flags: DIFlagArtificial
 // CHECK-SAME: )
 // CHECK:      {{!.*}} = !DIDerivedType(tag: DW_TAG_member, name: "2", scope: [[VARIANT]],
-// CHECK-SAME: file: [[FILE]], line: 19,
+// CHECK-SAME: file: [[FILE]], line: 20,
 // CHECK-NOT:  flags: DIFlagArtificial
 // CHECK-SAME: )
 // CHECK:      {{!.*}} = !DIDerivedType(tag: DW_TAG_member, name: "3", scope: [[VARIANT]],
-// CHECK-SAME: file: [[FILE]], line: 16,
+// CHECK-SAME: file: [[FILE]], line: 17,
 // CHECK-NOT:  flags: DIFlagArtificial
 // CHECK-SAME: )
 // CHECK:      {{!.*}} = !DIDerivedType(tag: DW_TAG_member, name: "4", scope: [[VARIANT]],
-// CHECK-SAME: file: [[FILE]], line: 18,
+// CHECK-SAME: file: [[FILE]], line: 19,
 // CHECK-NOT:  flags: DIFlagArtificial
 // CHECK-SAME: )
 // CHECK:      [[S1:!.*]] = !DICompositeType(tag: DW_TAG_structure_type, name: "Suspend1", scope: [[GEN]],
diff --git a/tests/codegen/debuginfo-generic-closure-env-names.rs b/tests/codegen/debuginfo-generic-closure-env-names.rs
index 6b314c9abae..64bc58e1df7 100644
--- a/tests/codegen/debuginfo-generic-closure-env-names.rs
+++ b/tests/codegen/debuginfo-generic-closure-env-names.rs
@@ -18,7 +18,8 @@
 // legacy mangling scheme rustc version and generic parameters are both hashed into a single part
 // of the name, thus randomizing item order with respect to rustc version.
 
-//@ compile-flags: -Cdebuginfo=2 --edition 2021 -Copt-level=0 -Csymbol-mangling-version=v0
+//@ compile-flags: -Cdebuginfo=2 -Copt-level=0 -Csymbol-mangling-version=v0
+//@ edition: 2021
 
 // non_generic_closure()
 // NONMSVC: !DICompositeType(tag: DW_TAG_structure_type, name: "{closure_env#0}", scope: ![[non_generic_closure_NAMESPACE:[0-9]+]],
diff --git a/tests/codegen/dont-shuffle-bswaps.rs b/tests/codegen/dont-shuffle-bswaps.rs
index e100474f606..c1dab2bc295 100644
--- a/tests/codegen/dont-shuffle-bswaps.rs
+++ b/tests/codegen/dont-shuffle-bswaps.rs
@@ -1,8 +1,11 @@
-//@ revisions: OPT2 OPT3
+//@ revisions: OPT2 OPT3 OPT3_S390X
 //@[OPT2] compile-flags: -Copt-level=2
 //@[OPT3] compile-flags: -C opt-level=3
 // some targets don't do the opt we are looking for
 //@[OPT3] only-64bit
+//@[OPT3] ignore-s390x
+//@[OPT3_S390X] compile-flags: -C opt-level=3 -C target-cpu=z13
+//@[OPT3_S390X] only-s390x
 
 #![crate_type = "lib"]
 #![no_std]
@@ -17,6 +20,10 @@
 // OPT3-NEXT: call <8 x i16> @llvm.bswap
 // OPT3-NEXT: store <8 x i16>
 // OPT3-NEXT: ret void
+// OPT3_S390X: load <8 x i16>
+// OPT3_S390X-NEXT: call <8 x i16> @llvm.bswap
+// OPT3_S390X-NEXT: store <8 x i16>
+// OPT3_S390X-NEXT: ret void
 #[no_mangle]
 pub fn convert(value: [u16; 8]) -> [u8; 16] {
     #[cfg(target_endian = "little")]
diff --git a/tests/codegen/emscripten-catch-unwind-js-eh.rs b/tests/codegen/emscripten-catch-unwind-js-eh.rs
index 018ad5454fc..3ab4b5c9c63 100644
--- a/tests/codegen/emscripten-catch-unwind-js-eh.rs
+++ b/tests/codegen/emscripten-catch-unwind-js-eh.rs
@@ -23,13 +23,12 @@ fn size_of<T>() -> usize {
     loop {}
 }
 
-extern "rust-intrinsic" {
-    fn catch_unwind(
-        try_fn: fn(_: *mut u8),
-        data: *mut u8,
-        catch_fn: fn(_: *mut u8, _: *mut u8),
-    ) -> i32;
-}
+#[rustc_intrinsic]
+unsafe fn catch_unwind(
+    try_fn: fn(_: *mut u8),
+    data: *mut u8,
+    catch_fn: fn(_: *mut u8, _: *mut u8),
+) -> i32;
 
 // CHECK-LABEL: @ptr_size
 #[no_mangle]
diff --git a/tests/codegen/emscripten-catch-unwind-wasm-eh.rs b/tests/codegen/emscripten-catch-unwind-wasm-eh.rs
index 0fc9ae96720..d0571e4df08 100644
--- a/tests/codegen/emscripten-catch-unwind-wasm-eh.rs
+++ b/tests/codegen/emscripten-catch-unwind-wasm-eh.rs
@@ -21,14 +21,12 @@ impl<T> Copy for *mut T {}
 fn size_of<T>() -> usize {
     loop {}
 }
-
-extern "rust-intrinsic" {
-    fn catch_unwind(
-        try_fn: fn(_: *mut u8),
-        data: *mut u8,
-        catch_fn: fn(_: *mut u8, _: *mut u8),
-    ) -> i32;
-}
+#[rustc_intrinsic]
+unsafe fn catch_unwind(
+    try_fn: fn(_: *mut u8),
+    data: *mut u8,
+    catch_fn: fn(_: *mut u8, _: *mut u8),
+) -> i32;
 
 // CHECK-LABEL: @ptr_size
 #[no_mangle]
diff --git a/tests/codegen/enum/enum-match.rs b/tests/codegen/enum/enum-match.rs
index a24b98050d2..6da6ad1f078 100644
--- a/tests/codegen/enum/enum-match.rs
+++ b/tests/codegen/enum/enum-match.rs
@@ -1,21 +1,26 @@
 //@ compile-flags: -Copt-level=1
-//@ only-x86_64
+//@ only-64bit
 
 #![crate_type = "lib"]
+#![feature(core_intrinsics)]
 
 // Check each of the 3 cases for `codegen_get_discr`.
 
+// FIXME: once our min-bar LLVM has `range` attributes, update the various
+// tests here to no longer have the `range`s and `nsw`s as optional.
+
 // Case 0: One tagged variant.
 pub enum Enum0 {
     A(bool),
     B,
 }
 
-// CHECK: define noundef{{( range\(i8 [0-9]+, [0-9]+\))?}} i8 @match0{{.*}}
+// CHECK-LABEL: define{{( dso_local)?}} noundef{{( range\(i8 [0-9]+, [0-9]+\))?}} i8 @match0(i8{{.+}}%0)
 // CHECK-NEXT: start:
-// CHECK-NEXT: %1 = icmp eq i8 %0, 2
-// CHECK-NEXT: %2 = and i8 %0, 1
-// CHECK-NEXT: %{{.+}} = select i1 %1, i8 13, i8 %2
+// CHECK-NEXT: %[[IS_B:.+]] = icmp eq i8 %0, 2
+// CHECK-NEXT: %[[TRUNC:.+]] = and i8 %0, 1
+// CHECK-NEXT: %[[R:.+]] = select i1 %[[IS_B]], i8 13, i8 %[[TRUNC]]
+// CHECK-NEXT: ret i8 %[[R]]
 #[no_mangle]
 pub fn match0(e: Enum0) -> u8 {
     use Enum0::*;
@@ -32,13 +37,14 @@ pub enum Enum1 {
     C,
 }
 
-// CHECK: define noundef{{( range\(i8 [0-9]+, [0-9]+\))?}} i8 @match1{{.*}}
+// CHECK-LABEL: define{{( dso_local)?}} noundef{{( range\(i8 [0-9]+, [0-9]+\))?}} i8 @match1(i8{{.+}}%0)
 // CHECK-NEXT: start:
-// CHECK-NEXT: %1 = add{{( nsw)?}} i8 %0, -2
-// CHECK-NEXT: %2 = zext i8 %1 to i64
-// CHECK-NEXT: %3 = icmp ult i8 %1, 2
-// CHECK-NEXT: %4 = add nuw nsw i64 %2, 1
-// CHECK-NEXT: %_2 = select i1 %3, i64 %4, i64 0
+// CHECK-NEXT: %[[REL_VAR:.+]] = add{{( nsw)?}} i8 %0, -2
+// CHECK-NEXT: %[[REL_VAR_WIDE:.+]] = zext i8 %[[REL_VAR]] to i64
+// CHECK-NEXT: %[[IS_NICHE:.+]] = icmp ult i8 %[[REL_VAR]], 2
+// CHECK-NEXT: %[[NICHE_DISCR:.+]] = add nuw nsw i64 %[[REL_VAR_WIDE]], 1
+// CHECK-NEXT: %[[DISCR:.+]] = select i1 %[[IS_NICHE]], i64 %[[NICHE_DISCR]], i64 0
+// CHECK-NEXT: switch i64 %[[DISCR]]
 #[no_mangle]
 pub fn match1(e: Enum1) -> u8 {
     use Enum1::*;
@@ -92,14 +98,14 @@ pub enum Enum2 {
     E,
 }
 
-// CHECK: define noundef{{( range\(i8 [0-9]+, [0-9]+\))?}} i8 @match2{{.*}}
+// CHECK-LABEL: define{{( dso_local)?}} noundef{{( range\(i8 [0-9]+, [0-9]+\))?}} i8 @match2(i8{{.+}}%0)
 // CHECK-NEXT: start:
-// CHECK-NEXT: %1 = add i8 %0, 2
-// CHECK-NEXT: %2 = zext i8 %1 to i64
-// CHECK-NEXT: %3 = icmp ult i8 %1, 4
-// CHECK-NEXT: %4 = add nuw nsw i64 %2, 1
-// CHECK-NEXT: %_2 = select i1 %3, i64 %4, i64 0
-// CHECK-NEXT: switch i64 %_2, label {{.*}} [
+// CHECK-NEXT: %[[REL_VAR:.+]] = add i8 %0, 2
+// CHECK-NEXT: %[[REL_VAR_WIDE:.+]] = zext i8 %[[REL_VAR]] to i64
+// CHECK-NEXT: %[[IS_NICHE:.+]] = icmp ult i8 %[[REL_VAR]], 4
+// CHECK-NEXT: %[[NICHE_DISCR:.+]] = add nuw nsw i64 %[[REL_VAR_WIDE]], 1
+// CHECK-NEXT: %[[DISCR:.+]] = select i1 %[[IS_NICHE]], i64 %[[NICHE_DISCR]], i64 0
+// CHECK-NEXT: switch i64 %[[DISCR]]
 #[no_mangle]
 pub fn match2(e: Enum2) -> u8 {
     use Enum2::*;
@@ -111,3 +117,357 @@ pub fn match2(e: Enum2) -> u8 {
         E => 250,
     }
 }
+
+// And make sure it works even if the niched scalar is a pointer.
+// (For example, that we don't try to `sub` on pointers.)
+
+// CHECK-LABEL: define{{( dso_local)?}} noundef{{( range\(i16 -?[0-9]+, -?[0-9]+\))?}} i16 @match3(ptr{{.+}}%0)
+// CHECK-NEXT: start:
+// CHECK-NEXT: %[[IS_NULL:.+]] = icmp eq ptr %0, null
+// CHECK-NEXT: br i1 %[[IS_NULL]]
+#[no_mangle]
+pub fn match3(e: Option<&u8>) -> i16 {
+    match e {
+        Some(r) => *r as _,
+        None => -1,
+    }
+}
+
+// If the untagged variant is in the middle, there's an impossible value that's
+// not reflected in the `range` parameter attribute, so we assume it away.
+
+#[derive(PartialEq)]
+pub enum MiddleNiche {
+    A,
+    B,
+    C(bool),
+    D,
+    E,
+}
+
+// CHECK-LABEL: define{{( dso_local)?}} noundef{{( range\(i8 -?[0-9]+, -?[0-9]+\))?}} i8 @match4(i8{{.+}}%0)
+// CHECK-NEXT: start:
+// CHECK-NEXT: %[[REL_VAR:.+]] = add{{( nsw)?}} i8 %0, -2
+// CHECK-NEXT: %[[IS_NICHE:.+]] = icmp ult i8 %[[REL_VAR]], 5
+// CHECK-NEXT: %[[NOT_IMPOSSIBLE:.+]] = icmp ne i8 %[[REL_VAR]], 2
+// CHECK-NEXT: call void @llvm.assume(i1 %[[NOT_IMPOSSIBLE]])
+// CHECK-NEXT: %[[DISCR:.+]] = select i1 %[[IS_NICHE]], i8 %[[REL_VAR]], i8 2
+// CHECK-NEXT: switch i8 %[[DISCR]]
+#[no_mangle]
+pub fn match4(e: MiddleNiche) -> u8 {
+    use MiddleNiche::*;
+    match e {
+        A => 13,
+        B => 100,
+        C(b) => b as u8,
+        D => 200,
+        E => 250,
+    }
+}
+
+// CHECK-LABEL: define{{.+}}i1 @match4_is_c(i8{{.+}}%e)
+// CHECK-NEXT: start
+// CHECK-NEXT: %[[REL_VAR:.+]] = add{{( nsw)?}} i8 %e, -2
+// CHECK-NEXT: %[[NOT_NICHE:.+]] = icmp ugt i8 %[[REL_VAR]], 4
+// CHECK-NEXT: %[[NOT_IMPOSSIBLE:.+]] = icmp ne i8 %[[REL_VAR]], 2
+// CHECK-NEXT: call void @llvm.assume(i1 %[[NOT_IMPOSSIBLE]])
+// CHECK-NEXT: ret i1 %[[NOT_NICHE]]
+#[no_mangle]
+pub fn match4_is_c(e: MiddleNiche) -> bool {
+    // Before #139098, this couldn't optimize out the `select` because it looked
+    // like it was possible for a `2` to be produced on both sides.
+
+    std::intrinsics::discriminant_value(&e) == 2
+}
+
+// You have to do something pretty obnoxious to get a variant index that doesn't
+// fit in the tag size, but it's possible
+
+pub enum Never {}
+
+pub enum HugeVariantIndex {
+    V000(Never),
+    V001(Never),
+    V002(Never),
+    V003(Never),
+    V004(Never),
+    V005(Never),
+    V006(Never),
+    V007(Never),
+    V008(Never),
+    V009(Never),
+    V010(Never),
+    V011(Never),
+    V012(Never),
+    V013(Never),
+    V014(Never),
+    V015(Never),
+    V016(Never),
+    V017(Never),
+    V018(Never),
+    V019(Never),
+    V020(Never),
+    V021(Never),
+    V022(Never),
+    V023(Never),
+    V024(Never),
+    V025(Never),
+    V026(Never),
+    V027(Never),
+    V028(Never),
+    V029(Never),
+    V030(Never),
+    V031(Never),
+    V032(Never),
+    V033(Never),
+    V034(Never),
+    V035(Never),
+    V036(Never),
+    V037(Never),
+    V038(Never),
+    V039(Never),
+    V040(Never),
+    V041(Never),
+    V042(Never),
+    V043(Never),
+    V044(Never),
+    V045(Never),
+    V046(Never),
+    V047(Never),
+    V048(Never),
+    V049(Never),
+    V050(Never),
+    V051(Never),
+    V052(Never),
+    V053(Never),
+    V054(Never),
+    V055(Never),
+    V056(Never),
+    V057(Never),
+    V058(Never),
+    V059(Never),
+    V060(Never),
+    V061(Never),
+    V062(Never),
+    V063(Never),
+    V064(Never),
+    V065(Never),
+    V066(Never),
+    V067(Never),
+    V068(Never),
+    V069(Never),
+    V070(Never),
+    V071(Never),
+    V072(Never),
+    V073(Never),
+    V074(Never),
+    V075(Never),
+    V076(Never),
+    V077(Never),
+    V078(Never),
+    V079(Never),
+    V080(Never),
+    V081(Never),
+    V082(Never),
+    V083(Never),
+    V084(Never),
+    V085(Never),
+    V086(Never),
+    V087(Never),
+    V088(Never),
+    V089(Never),
+    V090(Never),
+    V091(Never),
+    V092(Never),
+    V093(Never),
+    V094(Never),
+    V095(Never),
+    V096(Never),
+    V097(Never),
+    V098(Never),
+    V099(Never),
+    V100(Never),
+    V101(Never),
+    V102(Never),
+    V103(Never),
+    V104(Never),
+    V105(Never),
+    V106(Never),
+    V107(Never),
+    V108(Never),
+    V109(Never),
+    V110(Never),
+    V111(Never),
+    V112(Never),
+    V113(Never),
+    V114(Never),
+    V115(Never),
+    V116(Never),
+    V117(Never),
+    V118(Never),
+    V119(Never),
+    V120(Never),
+    V121(Never),
+    V122(Never),
+    V123(Never),
+    V124(Never),
+    V125(Never),
+    V126(Never),
+    V127(Never),
+    V128(Never),
+    V129(Never),
+    V130(Never),
+    V131(Never),
+    V132(Never),
+    V133(Never),
+    V134(Never),
+    V135(Never),
+    V136(Never),
+    V137(Never),
+    V138(Never),
+    V139(Never),
+    V140(Never),
+    V141(Never),
+    V142(Never),
+    V143(Never),
+    V144(Never),
+    V145(Never),
+    V146(Never),
+    V147(Never),
+    V148(Never),
+    V149(Never),
+    V150(Never),
+    V151(Never),
+    V152(Never),
+    V153(Never),
+    V154(Never),
+    V155(Never),
+    V156(Never),
+    V157(Never),
+    V158(Never),
+    V159(Never),
+    V160(Never),
+    V161(Never),
+    V162(Never),
+    V163(Never),
+    V164(Never),
+    V165(Never),
+    V166(Never),
+    V167(Never),
+    V168(Never),
+    V169(Never),
+    V170(Never),
+    V171(Never),
+    V172(Never),
+    V173(Never),
+    V174(Never),
+    V175(Never),
+    V176(Never),
+    V177(Never),
+    V178(Never),
+    V179(Never),
+    V180(Never),
+    V181(Never),
+    V182(Never),
+    V183(Never),
+    V184(Never),
+    V185(Never),
+    V186(Never),
+    V187(Never),
+    V188(Never),
+    V189(Never),
+    V190(Never),
+    V191(Never),
+    V192(Never),
+    V193(Never),
+    V194(Never),
+    V195(Never),
+    V196(Never),
+    V197(Never),
+    V198(Never),
+    V199(Never),
+    V200(Never),
+    V201(Never),
+    V202(Never),
+    V203(Never),
+    V204(Never),
+    V205(Never),
+    V206(Never),
+    V207(Never),
+    V208(Never),
+    V209(Never),
+    V210(Never),
+    V211(Never),
+    V212(Never),
+    V213(Never),
+    V214(Never),
+    V215(Never),
+    V216(Never),
+    V217(Never),
+    V218(Never),
+    V219(Never),
+    V220(Never),
+    V221(Never),
+    V222(Never),
+    V223(Never),
+    V224(Never),
+    V225(Never),
+    V226(Never),
+    V227(Never),
+    V228(Never),
+    V229(Never),
+    V230(Never),
+    V231(Never),
+    V232(Never),
+    V233(Never),
+    V234(Never),
+    V235(Never),
+    V236(Never),
+    V237(Never),
+    V238(Never),
+    V239(Never),
+    V240(Never),
+    V241(Never),
+    V242(Never),
+    V243(Never),
+    V244(Never),
+    V245(Never),
+    V246(Never),
+    V247(Never),
+    V248(Never),
+    V249(Never),
+    V250(Never),
+    V251(Never),
+    V252(Never),
+    V253(Never),
+    V254(Never),
+    V255(Never),
+    V256(Never),
+
+    Possible257,
+    Bool258(bool),
+    Possible259,
+}
+
+// CHECK-LABEL: define{{( dso_local)?}} noundef{{( range\(i8 [0-9]+, [0-9]+\))?}} i8 @match5(i8{{.+}}%0)
+// CHECK-NEXT: start:
+// CHECK-NEXT: %[[REL_VAR:.+]] = add{{( nsw)?}} i8 %0, -2
+// CHECK-NEXT: %[[REL_VAR_WIDE:.+]] = zext i8 %[[REL_VAR]] to i64
+// CHECK-NEXT: %[[IS_NICHE:.+]] = icmp ult i8 %[[REL_VAR]], 3
+// CHECK-NEXT: %[[NOT_IMPOSSIBLE:.+]] = icmp ne i8 %[[REL_VAR]], 1
+// CHECK-NEXT: call void @llvm.assume(i1 %[[NOT_IMPOSSIBLE]])
+// CHECK-NEXT: %[[NICHE_DISCR:.+]] = add nuw nsw i64 %[[REL_VAR_WIDE]], 257
+// CHECK-NEXT: %[[DISCR:.+]] = select i1 %[[IS_NICHE]], i64 %[[NICHE_DISCR]], i64 258
+// CHECK-NEXT: switch i64 %[[DISCR]],
+// CHECK-NEXT:   i64 257,
+// CHECK-NEXT:   i64 258,
+// CHECK-NEXT:   i64 259,
+#[no_mangle]
+pub fn match5(e: HugeVariantIndex) -> u8 {
+    use HugeVariantIndex::*;
+    match e {
+        Possible257 => 13,
+        Bool258(b) => b as u8,
+        Possible259 => 100,
+    }
+}
diff --git a/tests/codegen/enum/enum-two-variants-match.rs b/tests/codegen/enum/enum-two-variants-match.rs
index 21ae1f96bca..12d9edc4d62 100644
--- a/tests/codegen/enum/enum-two-variants-match.rs
+++ b/tests/codegen/enum/enum-two-variants-match.rs
@@ -1,8 +1,12 @@
 //@ compile-flags: -Copt-level=3 -C no-prepopulate-passes
-//@ only-x86_64 (because these discriminants are isize)
+//@ only-64bit (because these discriminants are isize)
 
 #![crate_type = "lib"]
 
+// This directly tests what we emit for these matches, rather than what happens
+// after optimization, so it doesn't need to worry about extra flags on the
+// instructions and is less susceptible to being broken on LLVM updates.
+
 // CHECK-LABEL: @option_match
 #[no_mangle]
 pub fn option_match(x: Option<i32>) -> u16 {
@@ -51,3 +55,76 @@ pub fn result_match(x: Result<u64, i64>) -> u16 {
         Ok(_) => 42,
     }
 }
+
+// CHECK-LABEL: @option_bool_match(
+#[no_mangle]
+pub fn option_bool_match(x: Option<bool>) -> char {
+    // CHECK: %[[RAW:.+]] = load i8, ptr %x
+    // CHECK: %[[IS_NONE:.+]] = icmp eq i8 %[[RAW]], 2
+    // CHECK: %[[OPT_DISCR:.+]] = select i1 %[[IS_NONE]], i64 0, i64 1
+    // CHECK: %[[OPT_DISCR_T:.+]] = trunc nuw i64 %[[OPT_DISCR]] to i1
+    // CHECK: br i1 %[[OPT_DISCR_T]], label %[[BB_SOME:.+]], label %[[BB_NONE:.+]]
+
+    // CHECK: [[BB_SOME]]:
+    // CHECK: %[[FIELD:.+]] = load i8, ptr %x
+    // CHECK: %[[FIELD_T:.+]] = trunc nuw i8 %[[FIELD]] to i1
+    // CHECK: br i1 %[[FIELD_T]]
+    match x {
+        None => 'n',
+        Some(false) => 'f',
+        Some(true) => 't',
+    }
+}
+
+use std::cmp::Ordering::{self, *};
+// CHECK-LABEL: @option_ordering_match(
+#[no_mangle]
+pub fn option_ordering_match(x: Option<Ordering>) -> char {
+    // CHECK: %[[RAW:.+]] = load i8, ptr %x
+    // CHECK: %[[IS_NONE:.+]] = icmp eq i8 %[[RAW]], 2
+    // CHECK: %[[OPT_DISCR:.+]] = select i1 %[[IS_NONE]], i64 0, i64 1
+    // CHECK: %[[OPT_DISCR_T:.+]] = trunc nuw i64 %[[OPT_DISCR]] to i1
+    // CHECK: br i1 %[[OPT_DISCR_T]], label %[[BB_SOME:.+]], label %[[BB_NONE:.+]]
+
+    // CHECK: [[BB_SOME]]:
+    // CHECK: %[[FIELD:.+]] = load i8, ptr %x
+    // CHECK: switch i8 %[[FIELD]], label %[[UNREACHABLE:.+]] [
+    // CHECK-NEXT: i8 -1, label
+    // CHECK-NEXT: i8 0, label
+    // CHECK-NEXT: i8 1, label
+    // CHECK-NEXT: ]
+
+    // CHECK: [[UNREACHABLE]]:
+    // CHECK-NEXT: unreachable
+    match x {
+        None => '?',
+        Some(Less) => '<',
+        Some(Equal) => '=',
+        Some(Greater) => '>',
+    }
+}
+
+// CHECK-LABEL: @option_nonzero_match(
+#[no_mangle]
+pub fn option_nonzero_match(x: Option<std::num::NonZero<u16>>) -> u16 {
+    // CHECK: %[[OUT:.+]] = alloca [2 x i8]
+
+    // CHECK: %[[IS_NONE:.+]] = icmp eq i16 %x, 0
+    // CHECK: %[[OPT_DISCR:.+]] = select i1 %[[IS_NONE]], i64 0, i64 1
+    // CHECK: %[[OPT_DISCR_T:.+]] = trunc nuw i64 %[[OPT_DISCR]] to i1
+    // CHECK: br i1 %[[OPT_DISCR_T]], label %[[BB_SOME:.+]], label %[[BB_NONE:.+]]
+
+    // CHECK: [[BB_SOME]]:
+    // CHECK: store i16 987, ptr %[[OUT]]
+
+    // CHECK: [[BB_NONE]]:
+    // CHECK: store i16 123, ptr %[[OUT]]
+
+    // CHECK: %[[RET:.+]] = load i16, ptr %[[OUT]]
+    // CHECK: ret i16 %[[RET]]
+
+    match x {
+        None => 123,
+        Some(_) => 987,
+    }
+}
diff --git a/tests/codegen/ergonomic-clones/closure.rs b/tests/codegen/ergonomic-clones/closure.rs
new file mode 100644
index 00000000000..b6fc8172641
--- /dev/null
+++ b/tests/codegen/ergonomic-clones/closure.rs
@@ -0,0 +1,55 @@
+//@ compile-flags: -C no-prepopulate-passes -Copt-level=0 -Zmir-opt-level=0
+
+#![crate_type = "lib"]
+
+#![feature(ergonomic_clones)]
+#![allow(incomplete_features)]
+
+use std::clone::UseCloned;
+
+pub fn ergonomic_clone_closure_move() -> String {
+    let s = String::from("hi");
+
+    // CHECK-NOT: ; call core::clone::impls::<impl core::clone::Clone for String>::clone
+    let cl = use || s;
+    cl()
+}
+
+#[derive(Clone)]
+struct Foo;
+
+impl UseCloned for Foo {}
+
+pub fn ergonomic_clone_closure_use_cloned() -> Foo {
+    let f = Foo;
+
+    // CHECK: ; call <closure::Foo as core::clone::Clone>::clone
+    let f1 = use || f;
+
+    // CHECK: ; call <closure::Foo as core::clone::Clone>::clone
+    let f2 = use || f;
+
+    f
+}
+
+pub fn ergonomic_clone_closure_copy() -> i32 {
+    let i = 1;
+
+    // CHECK-NOT: ; call core::clone::impls::<impl core::clone::Clone for i32>::clone
+    let i1 = use || i;
+
+    // CHECK-NOT: ; call core::clone::impls::<impl core::clone::Clone for i32>::clone
+    let i2 = use || i;
+
+    i
+}
+
+pub fn ergonomic_clone_closure_use_cloned_generics<T: UseCloned>(f: T) -> T {
+    // CHECK-NOT: ; call core::clone::impls::<impl core::clone::Clone for i32>::clone
+    let f1 = use || f;
+
+    // CHECK-NOT: ; call core::clone::impls::<impl core::clone::Clone for i32>::clone
+    let f2 = use || f;
+
+    f
+}
diff --git a/tests/codegen/infallible-unwrap-in-opt-z.rs b/tests/codegen/infallible-unwrap-in-opt-z.rs
index 3756fafe384..c2297c58e77 100644
--- a/tests/codegen/infallible-unwrap-in-opt-z.rs
+++ b/tests/codegen/infallible-unwrap-in-opt-z.rs
@@ -1,4 +1,5 @@
-//@ compile-flags: -C opt-level=z --edition=2021
+//@ compile-flags: -C opt-level=z
+//@ edition: 2021
 
 #![crate_type = "lib"]
 
diff --git a/tests/codegen/inline-function-args-debug-info.rs b/tests/codegen/inline-function-args-debug-info.rs
index 53a179160dc..c31419cb914 100644
--- a/tests/codegen/inline-function-args-debug-info.rs
+++ b/tests/codegen/inline-function-args-debug-info.rs
@@ -2,7 +2,8 @@
 // gets inlined by MIR inlining. Without function argument indexes, `info args` in gdb won't show
 // arguments and their values for the current function.
 
-//@ compile-flags: -Zinline-mir=yes -Cdebuginfo=2 --edition=2021
+//@ compile-flags: -Zinline-mir=yes -Cdebuginfo=2
+//@ edition: 2021
 
 #![crate_type = "lib"]
 
@@ -14,9 +15,9 @@ pub fn outer_function(x: usize, y: usize) -> usize {
 #[inline]
 fn inner_function(aaaa: usize, bbbb: usize) -> usize {
     // CHECK: !DILocalVariable(name: "aaaa", arg: 1
-    // CHECK-SAME: line: 15
+    // CHECK-SAME: line: 16
     // CHECK-NOT: !DILexicalBlock(
     // CHECK: !DILocalVariable(name: "bbbb", arg: 2
-    // CHECK-SAME: line: 15
+    // CHECK-SAME: line: 16
     aaaa + bbbb
 }
diff --git a/tests/codegen/intrinsic-no-unnamed-attr.rs b/tests/codegen/intrinsic-no-unnamed-attr.rs
index fce0de80d7b..4bec579831d 100644
--- a/tests/codegen/intrinsic-no-unnamed-attr.rs
+++ b/tests/codegen/intrinsic-no-unnamed-attr.rs
@@ -1,10 +1,9 @@
 //@ compile-flags: -C no-prepopulate-passes
 
-#![feature(intrinsics)]
+#![feature(core_intrinsics)]
+
+use std::intrinsics::sqrtf32;
 
-extern "rust-intrinsic" {
-    fn sqrtf32(x: f32) -> f32;
-}
 // CHECK: @llvm.sqrt.f32(float) #{{[0-9]*}}
 
 fn main() {
diff --git a/tests/codegen/intrinsics/nontemporal.rs b/tests/codegen/intrinsics/nontemporal.rs
index 1d4fae83c29..a151d4bd297 100644
--- a/tests/codegen/intrinsics/nontemporal.rs
+++ b/tests/codegen/intrinsics/nontemporal.rs
@@ -18,9 +18,8 @@
 extern crate minicore;
 use minicore::*;
 
-extern "rust-intrinsic" {
-    pub fn nontemporal_store<T>(ptr: *mut T, val: T);
-}
+#[rustc_intrinsic]
+pub unsafe fn nontemporal_store<T>(ptr: *mut T, val: T);
 
 #[no_mangle]
 pub fn a(a: &mut u32, b: u32) {
diff --git a/tests/codegen/intrinsics/select_unpredictable.rs b/tests/codegen/intrinsics/select_unpredictable.rs
index 68a02c8342d..2db4ae174b3 100644
--- a/tests/codegen/intrinsics/select_unpredictable.rs
+++ b/tests/codegen/intrinsics/select_unpredictable.rs
@@ -46,21 +46,21 @@ pub fn test_zst(p: bool, a: (), b: ()) -> () {
 pub fn test_int2(p: bool, a: u64, b: u64) -> u64 {
     // CHECK-LABEL: define{{.*}} @test_int2
     // CHECK: select i1 %p, i64 %a, i64 %b, !unpredictable
-    p.select_unpredictable(a, b)
+    core::hint::select_unpredictable(p, a, b)
 }
 
 #[no_mangle]
 pub fn test_pair2(p: bool, a: (u64, u64), b: (u64, u64)) -> (u64, u64) {
     // CHECK-LABEL: define{{.*}} @test_pair2
     // CHECK: select i1 %p, {{.*}}, !unpredictable
-    p.select_unpredictable(a, b)
+    core::hint::select_unpredictable(p, a, b)
 }
 
 #[no_mangle]
 pub fn test_struct2(p: bool, a: Large, b: Large) -> Large {
     // CHECK-LABEL: define{{.*}} @test_struct2
     // CHECK: select i1 %p, {{.*}}, !unpredictable
-    p.select_unpredictable(a, b)
+    core::hint::select_unpredictable(p, a, b)
 }
 
 #[no_mangle]
@@ -68,5 +68,5 @@ pub fn test_zst2(p: bool, a: (), b: ()) -> () {
     // CHECK-LABEL: define{{.*}} @test_zst2
     // CHECK-NEXT: start:
     // CHECK-NEXT: ret void
-    p.select_unpredictable(a, b)
+    core::hint::select_unpredictable(p, a, b)
 }
diff --git a/tests/codegen/issues/issue-101082.rs b/tests/codegen/issues/issue-101082.rs
index 7fb850ca253..8d15921ddb4 100644
--- a/tests/codegen/issues/issue-101082.rs
+++ b/tests/codegen/issues/issue-101082.rs
@@ -1,9 +1,18 @@
 //@ compile-flags: -Copt-level=3
-//@ revisions: host x86-64-v3
+//@ revisions: host x86-64 x86-64-v3
 //@ min-llvm-version: 20
 
-// This particular CPU regressed in #131563
+//@[host] ignore-x86_64
+
+// Set the base cpu explicitly, in case the default has been changed.
+//@[x86-64] only-x86_64
+//@[x86-64] compile-flags: -Ctarget-cpu=x86-64
+
+// FIXME(cuviper) x86-64-v3 in particular regressed in #131563, and the workaround
+// at the time still sometimes fails, so only verify it for the power-of-two size
+// - https://github.com/llvm/llvm-project/issues/134735
 //@[x86-64-v3] only-x86_64
+//@[x86-64-v3] min-llvm-version: 21
 //@[x86-64-v3] compile-flags: -Ctarget-cpu=x86-64-v3
 
 #![crate_type = "lib"]
@@ -19,3 +28,15 @@ pub fn test() -> usize {
     }
     acc
 }
+
+#[no_mangle]
+pub fn test_eight() -> usize {
+    // CHECK-LABEL: @test_eight(
+    // CHECK: ret {{i64|i32}} 220
+    let values = [23, 16, 54, 3, 60, 9, 13, 42];
+    let mut acc = 0;
+    for item in values {
+        acc += item;
+    }
+    acc
+}
diff --git a/tests/codegen/issues/issue-119422.rs b/tests/codegen/issues/issue-119422.rs
index e1a082c377f..17ae71605b5 100644
--- a/tests/codegen/issues/issue-119422.rs
+++ b/tests/codegen/issues/issue-119422.rs
@@ -1,7 +1,8 @@
 //! This test checks that compiler don't generate useless compares to zeros
 //! for `NonZero` integer types.
 //!
-//@ compile-flags: -Copt-level=3 --edition=2021 -Zmerge-functions=disabled
+//@ compile-flags: -Copt-level=3 -Zmerge-functions=disabled
+//@ edition: 2021
 //@ only-64bit (because the LLVM type of i64 for usize shows up)
 #![crate_type = "lib"]
 
diff --git a/tests/codegen/remap_path_prefix/aux_mod.rs b/tests/codegen/remap_path_prefix/aux_mod.rs
index c37e91c705c..3217e9e51e7 100644
--- a/tests/codegen/remap_path_prefix/aux_mod.rs
+++ b/tests/codegen/remap_path_prefix/aux_mod.rs
@@ -1,4 +1,4 @@
-//@ ignore-test: this is not a test
+//@ ignore-auxiliary (used by `./main.rs`)
 
 #[inline]
 pub fn some_aux_mod_function() -> i32 {
diff --git a/tests/codegen/simd/extract-insert-dyn.rs b/tests/codegen/simd/extract-insert-dyn.rs
new file mode 100644
index 00000000000..584e2c7887a
--- /dev/null
+++ b/tests/codegen/simd/extract-insert-dyn.rs
@@ -0,0 +1,75 @@
+//@compile-flags: -C opt-level=3 -C no-prepopulate-passes
+
+#![feature(core_intrinsics, repr_simd)]
+#![no_std]
+#![crate_type = "lib"]
+#![allow(non_camel_case_types)]
+
+// Test that `core::intrinsics::simd::{simd_extract_dyn, simd_insert_dyn}`
+// lower to an LLVM extractelement or insertelement operation.
+
+use core::intrinsics::simd::{simd_extract, simd_extract_dyn, simd_insert, simd_insert_dyn};
+
+#[repr(simd)]
+#[derive(Clone, Copy)]
+pub struct u32x16([u32; 16]);
+
+#[repr(simd)]
+#[derive(Clone, Copy)]
+pub struct i8x16([i8; 16]);
+
+// CHECK-LABEL: dyn_simd_extract
+// CHECK: extractelement <16 x i8> %x, i32 %idx
+#[no_mangle]
+unsafe extern "C" fn dyn_simd_extract(x: i8x16, idx: u32) -> i8 {
+    simd_extract_dyn(x, idx)
+}
+
+// CHECK-LABEL: literal_dyn_simd_extract
+// CHECK: extractelement <16 x i8> %x, i32 7
+#[no_mangle]
+unsafe extern "C" fn literal_dyn_simd_extract(x: i8x16) -> i8 {
+    simd_extract_dyn(x, 7)
+}
+
+// CHECK-LABEL: const_dyn_simd_extract
+// CHECK: extractelement <16 x i8> %x, i32 7
+#[no_mangle]
+unsafe extern "C" fn const_dyn_simd_extract(x: i8x16) -> i8 {
+    simd_extract_dyn(x, const { 3 + 4 })
+}
+
+// CHECK-LABEL: const_simd_extract
+// CHECK: extractelement <16 x i8> %x, i32 7
+#[no_mangle]
+unsafe extern "C" fn const_simd_extract(x: i8x16) -> i8 {
+    simd_extract(x, const { 3 + 4 })
+}
+
+// CHECK-LABEL: dyn_simd_insert
+// CHECK: insertelement <16 x i8> %x, i8 %e, i32 %idx
+#[no_mangle]
+unsafe extern "C" fn dyn_simd_insert(x: i8x16, e: i8, idx: u32) -> i8x16 {
+    simd_insert_dyn(x, idx, e)
+}
+
+// CHECK-LABEL: literal_dyn_simd_insert
+// CHECK: insertelement <16 x i8> %x, i8 %e, i32 7
+#[no_mangle]
+unsafe extern "C" fn literal_dyn_simd_insert(x: i8x16, e: i8) -> i8x16 {
+    simd_insert_dyn(x, 7, e)
+}
+
+// CHECK-LABEL: const_dyn_simd_insert
+// CHECK: insertelement <16 x i8> %x, i8 %e, i32 7
+#[no_mangle]
+unsafe extern "C" fn const_dyn_simd_insert(x: i8x16, e: i8) -> i8x16 {
+    simd_insert_dyn(x, const { 3 + 4 }, e)
+}
+
+// CHECK-LABEL: const_simd_insert
+// CHECK: insertelement <16 x i8> %x, i8 %e, i32 7
+#[no_mangle]
+unsafe extern "C" fn const_simd_insert(x: i8x16, e: i8) -> i8x16 {
+    simd_insert(x, const { 3 + 4 }, e)
+}
diff --git a/tests/codegen/simd/simd-wide-sum.rs b/tests/codegen/simd/simd-wide-sum.rs
index fb9b61884e7..95117b2c748 100644
--- a/tests/codegen/simd/simd-wide-sum.rs
+++ b/tests/codegen/simd/simd-wide-sum.rs
@@ -1,5 +1,6 @@
 //@ revisions: llvm mir-opt3
-//@ compile-flags: -C opt-level=3 -Z merge-functions=disabled --edition=2021
+//@ compile-flags: -C opt-level=3 -Z merge-functions=disabled
+//@ edition: 2021
 //@ only-x86_64
 //@ [mir-opt3]compile-flags: -Zmir-opt-level=3
 //@ [mir-opt3]build-pass
diff --git a/tests/codegen/simd/swap-simd-types.rs b/tests/codegen/simd/swap-simd-types.rs
index 69767d0a755..c063cc683a6 100644
--- a/tests/codegen/simd/swap-simd-types.rs
+++ b/tests/codegen/simd/swap-simd-types.rs
@@ -23,8 +23,8 @@ pub fn swap_single_m256(x: &mut __m256, y: &mut __m256) {
 #[no_mangle]
 pub fn swap_m256_slice(x: &mut [__m256], y: &mut [__m256]) {
     // CHECK-NOT: alloca
-    // CHECK: load <8 x float>{{.+}}align 32
-    // CHECK: store <8 x float>{{.+}}align 32
+    // CHECK-COUNT-2: load <4 x i64>{{.+}}align 32
+    // CHECK-COUNT-2: store <4 x i64>{{.+}}align 32
     if x.len() == y.len() {
         x.swap_with_slice(y);
     }
@@ -34,7 +34,7 @@ pub fn swap_m256_slice(x: &mut [__m256], y: &mut [__m256]) {
 #[no_mangle]
 pub fn swap_bytes32(x: &mut [u8; 32], y: &mut [u8; 32]) {
     // CHECK-NOT: alloca
-    // CHECK: load <32 x i8>{{.+}}align 1
-    // CHECK: store <32 x i8>{{.+}}align 1
+    // CHECK-COUNT-2: load <4 x i64>{{.+}}align 1
+    // CHECK-COUNT-2: store <4 x i64>{{.+}}align 1
     swap(x, y)
 }
diff --git a/tests/codegen/string-push.rs b/tests/codegen/string-push.rs
new file mode 100644
index 00000000000..cf5f6bb1aa3
--- /dev/null
+++ b/tests/codegen/string-push.rs
@@ -0,0 +1,11 @@
+//! Check that `String::push` is optimized enough not to call `memcpy`.
+
+//@ compile-flags: -O
+#![crate_type = "lib"]
+
+// CHECK-LABEL: @string_push_does_not_call_memcpy
+#[no_mangle]
+pub fn string_push_does_not_call_memcpy(s: &mut String, ch: char) {
+    // CHECK-NOT: call void @llvm.memcpy
+    s.push(ch);
+}
diff --git a/tests/codegen/swap-large-types.rs b/tests/codegen/swap-large-types.rs
index 49a41bb1469..08c486affd9 100644
--- a/tests/codegen/swap-large-types.rs
+++ b/tests/codegen/swap-large-types.rs
@@ -12,6 +12,16 @@ type KeccakBuffer = [[u64; 5]; 5];
 // to stack for large types, which is completely unnecessary as the lack of
 // overlap means we can just do whatever fits in registers at a time.
 
+// The tests here (after the first one showing that the problem still exists)
+// are less about testing *exactly* what the codegen is, and more about testing
+// 1) That things are swapped directly from one argument to the other,
+//    never going through stack along the way, and
+// 2) That we're doing the swapping for big things using large vector types,
+//    rather then `i64` or `<8 x i8>` (or, even worse, `i8`) at a time.
+//
+// (There are separate tests for intrinsics::typed_swap_nonoverlapping that
+//  check that it, as an intrinsic, are emitting exactly what it should.)
+
 // CHECK-LABEL: @swap_basic
 #[no_mangle]
 pub fn swap_basic(x: &mut KeccakBuffer, y: &mut KeccakBuffer) {
@@ -26,55 +36,55 @@ pub fn swap_basic(x: &mut KeccakBuffer, y: &mut KeccakBuffer) {
     }
 }
 
-// This test verifies that the library does something smarter, and thus
-// doesn't need any scratch space on the stack.
-
 // CHECK-LABEL: @swap_std
 #[no_mangle]
 pub fn swap_std(x: &mut KeccakBuffer, y: &mut KeccakBuffer) {
     // CHECK-NOT: alloca
-    // CHECK: load <{{[0-9]+}} x i64>
-    // CHECK: store <{{[0-9]+}} x i64>
+    // CHECK: load <{{2|4}} x i64>
+    // CHECK: store <{{2|4}} x i64>
     swap(x, y)
 }
 
-// Verify that types with usize alignment are swapped via vectored usizes,
-// not falling back to byte-level code.
-
 // CHECK-LABEL: @swap_slice
 #[no_mangle]
 pub fn swap_slice(x: &mut [KeccakBuffer], y: &mut [KeccakBuffer]) {
     // CHECK-NOT: alloca
-    // CHECK: load <{{[0-9]+}} x i64>
-    // CHECK: store <{{[0-9]+}} x i64>
+    // CHECK: load <{{2|4}} x i64>
+    // CHECK: store <{{2|4}} x i64>
     if x.len() == y.len() {
         x.swap_with_slice(y);
     }
 }
 
-// But for a large align-1 type, vectorized byte copying is what we want.
-
 type OneKilobyteBuffer = [u8; 1024];
 
 // CHECK-LABEL: @swap_1kb_slices
 #[no_mangle]
 pub fn swap_1kb_slices(x: &mut [OneKilobyteBuffer], y: &mut [OneKilobyteBuffer]) {
     // CHECK-NOT: alloca
-    // CHECK: load <{{[0-9]+}} x i8>
-    // CHECK: store <{{[0-9]+}} x i8>
+
+    // CHECK-NOT: load i32
+    // CHECK-NOT: store i32
+    // CHECK-NOT: load i16
+    // CHECK-NOT: store i16
+    // CHECK-NOT: load i8
+    // CHECK-NOT: store i8
+
+    // CHECK: load <{{2|4}} x i64>{{.+}}align 1,
+    // CHECK: store <{{2|4}} x i64>{{.+}}align 1,
+
+    // CHECK-NOT: load i32
+    // CHECK-NOT: store i32
+    // CHECK-NOT: load i16
+    // CHECK-NOT: store i16
+    // CHECK-NOT: load i8
+    // CHECK-NOT: store i8
+
     if x.len() == y.len() {
         x.swap_with_slice(y);
     }
 }
 
-// This verifies that the 2×read + 2×write optimizes to just 3 memcpys
-// for an unusual type like this.  It's not clear whether we should do anything
-// smarter in Rust for these, so for now it's fine to leave these up to the backend.
-// That's not as bad as it might seem, as for example, LLVM will lower the
-// memcpys below to VMOVAPS on YMMs if one enables the AVX target feature.
-// Eventually we'll be able to pass `align_of::<T>` to a const generic and
-// thus pick a smarter chunk size ourselves without huge code duplication.
-
 #[repr(align(64))]
 pub struct BigButHighlyAligned([u8; 64 * 3]);
 
@@ -82,9 +92,25 @@ pub struct BigButHighlyAligned([u8; 64 * 3]);
 #[no_mangle]
 pub fn swap_big_aligned(x: &mut BigButHighlyAligned, y: &mut BigButHighlyAligned) {
     // CHECK-NOT: call void @llvm.memcpy
-    // CHECK: call void @llvm.memcpy.{{.+}}(ptr noundef nonnull align 64 dereferenceable(192)
-    // CHECK: call void @llvm.memcpy.{{.+}}(ptr noundef nonnull align 64 dereferenceable(192)
-    // CHECK: call void @llvm.memcpy.{{.+}}(ptr noundef nonnull align 64 dereferenceable(192)
+    // CHECK-NOT: load i32
+    // CHECK-NOT: store i32
+    // CHECK-NOT: load i16
+    // CHECK-NOT: store i16
+    // CHECK-NOT: load i8
+    // CHECK-NOT: store i8
+
+    // CHECK-COUNT-2: load <{{2|4}} x i64>{{.+}}align 64,
+    // CHECK-COUNT-2: store <{{2|4}} x i64>{{.+}}align 64,
+
+    // CHECK-COUNT-2: load <{{2|4}} x i64>{{.+}}align 32,
+    // CHECK-COUNT-2: store <{{2|4}} x i64>{{.+}}align 32,
+
+    // CHECK-NOT: load i32
+    // CHECK-NOT: store i32
+    // CHECK-NOT: load i16
+    // CHECK-NOT: store i16
+    // CHECK-NOT: load i8
+    // CHECK-NOT: store i8
     // CHECK-NOT: call void @llvm.memcpy
     swap(x, y)
 }
diff --git a/tests/codegen/swap-small-types.rs b/tests/codegen/swap-small-types.rs
index 76bb853e642..7aa613ae9c2 100644
--- a/tests/codegen/swap-small-types.rs
+++ b/tests/codegen/swap-small-types.rs
@@ -1,5 +1,7 @@
 //@ compile-flags: -Copt-level=3 -Z merge-functions=disabled
 //@ only-x86_64
+//@ min-llvm-version: 20
+//@ ignore-std-debug-assertions (`ptr::swap_nonoverlapping` has one which blocks some optimizations)
 
 #![crate_type = "lib"]
 
@@ -27,13 +29,19 @@ pub fn swap_rgb48_manually(x: &mut RGB48, y: &mut RGB48) {
 pub fn swap_rgb48(x: &mut RGB48, y: &mut RGB48) {
     // CHECK-NOT: alloca
 
-    // Whether `i8` is the best for this is unclear, but
-    // might as well record what's actually happening right now.
-
-    // CHECK: load i8
-    // CHECK: load i8
-    // CHECK: store i8
-    // CHECK: store i8
+    // Swapping `i48` might be cleaner in LLVM-IR here, but `i32`+`i16` isn't bad,
+    // and is closer to the assembly it generates anyway.
+
+    // CHECK-NOT: load{{ }}
+    // CHECK: load i32{{.+}}align 2
+    // CHECK-NEXT: load i32{{.+}}align 2
+    // CHECK-NEXT: store i32{{.+}}align 2
+    // CHECK-NEXT: store i32{{.+}}align 2
+    // CHECK: load i16{{.+}}align 2
+    // CHECK-NEXT: load i16{{.+}}align 2
+    // CHECK-NEXT: store i16{{.+}}align 2
+    // CHECK-NEXT: store i16{{.+}}align 2
+    // CHECK-NOT: store{{ }}
     swap(x, y)
 }
 
@@ -76,30 +84,49 @@ pub fn swap_slices<'a>(x: &mut &'a [u32], y: &mut &'a [u32]) {
     swap(x, y)
 }
 
-// LLVM doesn't vectorize a loop over 3-byte elements,
-// so we chunk it down to bytes and loop over those instead.
 type RGB24 = [u8; 3];
 
 // CHECK-LABEL: @swap_rgb24_slices
 #[no_mangle]
 pub fn swap_rgb24_slices(x: &mut [RGB24], y: &mut [RGB24]) {
     // CHECK-NOT: alloca
-    // CHECK: load <{{[0-9]+}} x i8>
-    // CHECK: store <{{[0-9]+}} x i8>
+
+    // CHECK: mul nuw nsw i64 %{{x|y}}.1, 3
+
+    // CHECK: load <{{[0-9]+}} x i64>
+    // CHECK: store <{{[0-9]+}} x i64>
+
+    // CHECK-COUNT-2: load i32
+    // CHECK-COUNT-2: store i32
+    // CHECK-COUNT-2: load i16
+    // CHECK-COUNT-2: store i16
+    // CHECK-COUNT-2: load i8
+    // CHECK-COUNT-2: store i8
     if x.len() == y.len() {
         x.swap_with_slice(y);
     }
 }
 
-// This one has a power-of-two size, so we iterate over it directly
 type RGBA32 = [u8; 4];
 
 // CHECK-LABEL: @swap_rgba32_slices
 #[no_mangle]
 pub fn swap_rgba32_slices(x: &mut [RGBA32], y: &mut [RGBA32]) {
     // CHECK-NOT: alloca
-    // CHECK: load <{{[0-9]+}} x i32>
-    // CHECK: store <{{[0-9]+}} x i32>
+
+    // Because the size in bytes in a multiple of 4, we can skip the smallest sizes.
+
+    // CHECK: load <{{[0-9]+}} x i64>
+    // CHECK: store <{{[0-9]+}} x i64>
+
+    // CHECK-COUNT-2: load i32
+    // CHECK-COUNT-2: store i32
+
+    // CHECK-NOT: load i16
+    // CHECK-NOT: store i16
+    // CHECK-NOT: load i8
+    // CHECK-NOT: store i8
+
     if x.len() == y.len() {
         x.swap_with_slice(y);
     }
@@ -113,8 +140,8 @@ const _: () = assert!(!std::mem::size_of::<String>().is_power_of_two());
 #[no_mangle]
 pub fn swap_string_slices(x: &mut [String], y: &mut [String]) {
     // CHECK-NOT: alloca
-    // CHECK: load <{{[0-9]+}} x i64>
-    // CHECK: store <{{[0-9]+}} x i64>
+    // CHECK: load <{{[0-9]+}} x i64>{{.+}}, align 8,
+    // CHECK: store <{{[0-9]+}} x i64>{{.+}}, align 8,
     if x.len() == y.len() {
         x.swap_with_slice(y);
     }
@@ -130,6 +157,26 @@ pub struct Packed {
 #[no_mangle]
 pub fn swap_packed_structs(x: &mut Packed, y: &mut Packed) {
     // CHECK-NOT: alloca
+
+    // CHECK-NOT: load
+    // CHECK-NOT: store
+
+    // CHECK: %[[A:.+]] = load i64, ptr %x, align 1,
+    // CHECK-NEXT: %[[B:.+]] = load i64, ptr %y, align 1,
+    // CHECK-NEXT: store i64 %[[B]], ptr %x, align 1,
+    // CHECK-NEXT: store i64 %[[A]], ptr %y, align 1,
+
+    // CHECK-NOT: load
+    // CHECK-NOT: store
+
+    // CHECK: %[[C:.+]] = load i8, ptr %[[X8:.+]], align 1,
+    // CHECK-NEXT: %[[D:.+]] = load i8, ptr %[[Y8:.+]], align 1,
+    // CHECK-NEXT: store i8 %[[D]], ptr %[[X8]], align 1,
+    // CHECK-NEXT: store i8 %[[C]], ptr %[[Y8]], align 1,
+
+    // CHECK-NOT: load
+    // CHECK-NOT: store
+
     // CHECK: ret void
     swap(x, y)
 }
diff --git a/tests/codegen/try_question_mark_nop.rs b/tests/codegen/try_question_mark_nop.rs
index 9f68d742a75..398c9a580bc 100644
--- a/tests/codegen/try_question_mark_nop.rs
+++ b/tests/codegen/try_question_mark_nop.rs
@@ -1,4 +1,5 @@
-//@ compile-flags: -Copt-level=3 -Z merge-functions=disabled --edition=2021
+//@ compile-flags: -Copt-level=3 -Z merge-functions=disabled
+//@ edition: 2021
 //@ only-x86_64
 //@ revisions: NINETEEN TWENTY
 //@[NINETEEN] exact-llvm-major-version: 19