Merge from rustc

12 files changed, 586 insertions, 43 deletions

diff --git a/tests/codegen/autodiffv.rs b/tests/codegen/autodiff/batched.rs
index e0047116405..e0047116405 100644
--- a/tests/codegen/autodiffv.rs
+++ b/tests/codegen/autodiff/batched.rs
diff --git a/tests/codegen/autodiff.rs b/tests/codegen/autodiff/scalar.rs
index 85358f5fcb6..85358f5fcb6 100644
--- a/tests/codegen/autodiff.rs
+++ b/tests/codegen/autodiff/scalar.rs
diff --git a/tests/codegen/autodiff/sret.rs b/tests/codegen/autodiff/sret.rs
new file mode 100644
index 00000000000..5ead90041ed
--- /dev/null
+++ b/tests/codegen/autodiff/sret.rs
@@ -0,0 +1,45 @@
+//@ compile-flags: -Zautodiff=Enable -C opt-level=3  -Clto=fat
+//@ no-prefer-dynamic
+//@ needs-enzyme
+
+// This test is almost identical to the scalar.rs one,
+// but we intentionally add a few more floats.
+// `df` would ret `{ f64, f32, f32 }`, but is lowered as an sret.
+// We therefore use this test to verify some of our sret handling.
+
+#![feature(autodiff)]
+
+use std::autodiff::autodiff;
+
+#[no_mangle]
+#[autodiff(df, Reverse, Active, Active, Active)]
+fn primal(x: f32, y: f32) -> f64 {
+    (x * x * y) as f64
+}
+
+// CHECK:define internal fastcc void @_ZN4sret2df17h93be4316dd8ea006E(ptr dead_on_unwind noalias nocapture noundef nonnull writable writeonly align 8 dereferenceable(16) initializes((0, 16)) %_0, float noundef %x, float noundef %y)
+// CHECK-NEXT:start:
+// CHECK-NEXT:  %0 = tail call fastcc { double, float, float } @diffeprimal(float %x, float %y)
+// CHECK-NEXT:  %.elt = extractvalue { double, float, float } %0, 0
+// CHECK-NEXT:  store double %.elt, ptr %_0, align 8
+// CHECK-NEXT:  %_0.repack1 = getelementptr inbounds nuw i8, ptr %_0, i64 8
+// CHECK-NEXT:  %.elt2 = extractvalue { double, float, float } %0, 1
+// CHECK-NEXT:  store float %.elt2, ptr %_0.repack1, align 8
+// CHECK-NEXT:  %_0.repack3 = getelementptr inbounds nuw i8, ptr %_0, i64 12
+// CHECK-NEXT:  %.elt4 = extractvalue { double, float, float } %0, 2
+// CHECK-NEXT:  store float %.elt4, ptr %_0.repack3, align 4
+// CHECK-NEXT:  ret void
+// CHECK-NEXT:}
+
+fn main() {
+    let x = std::hint::black_box(3.0);
+    let y = std::hint::black_box(2.5);
+    let scalar = std::hint::black_box(1.0);
+    let (r1, r2, r3) = df(x, y, scalar);
+    // 3*3*1.5 = 22.5
+    assert_eq!(r1, 22.5);
+    // 2*x*y = 2*3*2.5 = 15.0
+    assert_eq!(r2, 15.0);
+    // x*x*1 = 3*3 = 9
+    assert_eq!(r3, 9.0);
+}
diff --git a/tests/codegen/avr/avr-func-addrspace.rs b/tests/codegen/avr/avr-func-addrspace.rs
index 2ae2f40d7b3..e0192f8b45a 100644
--- a/tests/codegen/avr/avr-func-addrspace.rs
+++ b/tests/codegen/avr/avr-func-addrspace.rs
@@ -17,9 +17,8 @@
 extern crate minicore;
 use minicore::*;
 
-extern "rust-intrinsic" {
-    pub fn transmute<Src, Dst>(src: Src) -> Dst;
-}
+#[rustc_intrinsic]
+pub unsafe fn transmute<Src, Dst>(src: Src) -> Dst;
 
 pub static mut STORAGE_FOO: fn(&usize, &mut u32) -> Result<(), ()> = arbitrary_black_box;
 pub static mut STORAGE_BAR: u32 = 12;
diff --git a/tests/codegen/emscripten-catch-unwind-js-eh.rs b/tests/codegen/emscripten-catch-unwind-js-eh.rs
index 018ad5454fc..3ab4b5c9c63 100644
--- a/tests/codegen/emscripten-catch-unwind-js-eh.rs
+++ b/tests/codegen/emscripten-catch-unwind-js-eh.rs
@@ -23,13 +23,12 @@ fn size_of<T>() -> usize {
     loop {}
 }
 
-extern "rust-intrinsic" {
-    fn catch_unwind(
-        try_fn: fn(_: *mut u8),
-        data: *mut u8,
-        catch_fn: fn(_: *mut u8, _: *mut u8),
-    ) -> i32;
-}
+#[rustc_intrinsic]
+unsafe fn catch_unwind(
+    try_fn: fn(_: *mut u8),
+    data: *mut u8,
+    catch_fn: fn(_: *mut u8, _: *mut u8),
+) -> i32;
 
 // CHECK-LABEL: @ptr_size
 #[no_mangle]
diff --git a/tests/codegen/emscripten-catch-unwind-wasm-eh.rs b/tests/codegen/emscripten-catch-unwind-wasm-eh.rs
index 0fc9ae96720..d0571e4df08 100644
--- a/tests/codegen/emscripten-catch-unwind-wasm-eh.rs
+++ b/tests/codegen/emscripten-catch-unwind-wasm-eh.rs
@@ -21,14 +21,12 @@ impl<T> Copy for *mut T {}
 fn size_of<T>() -> usize {
     loop {}
 }
-
-extern "rust-intrinsic" {
-    fn catch_unwind(
-        try_fn: fn(_: *mut u8),
-        data: *mut u8,
-        catch_fn: fn(_: *mut u8, _: *mut u8),
-    ) -> i32;
-}
+#[rustc_intrinsic]
+unsafe fn catch_unwind(
+    try_fn: fn(_: *mut u8),
+    data: *mut u8,
+    catch_fn: fn(_: *mut u8, _: *mut u8),
+) -> i32;
 
 // CHECK-LABEL: @ptr_size
 #[no_mangle]
diff --git a/tests/codegen/enum/enum-match.rs b/tests/codegen/enum/enum-match.rs
index a24b98050d2..6e185cf8932 100644
--- a/tests/codegen/enum/enum-match.rs
+++ b/tests/codegen/enum/enum-match.rs
@@ -1,21 +1,26 @@
 //@ compile-flags: -Copt-level=1
-//@ only-x86_64
+//@ only-64bit
 
 #![crate_type = "lib"]
+#![feature(core_intrinsics)]
 
 // Check each of the 3 cases for `codegen_get_discr`.
 
+// FIXME: once our min-bar LLVM has `range` attributes, update the various
+// tests here to no longer have the `range`s and `nsw`s as optional.
+
 // Case 0: One tagged variant.
 pub enum Enum0 {
     A(bool),
     B,
 }
 
-// CHECK: define noundef{{( range\(i8 [0-9]+, [0-9]+\))?}} i8 @match0{{.*}}
+// CHECK-LABEL: define noundef{{( range\(i8 [0-9]+, [0-9]+\))?}} i8 @match0(i8{{.+}}%0)
 // CHECK-NEXT: start:
-// CHECK-NEXT: %1 = icmp eq i8 %0, 2
-// CHECK-NEXT: %2 = and i8 %0, 1
-// CHECK-NEXT: %{{.+}} = select i1 %1, i8 13, i8 %2
+// CHECK-NEXT: %[[IS_B:.+]] = icmp eq i8 %0, 2
+// CHECK-NEXT: %[[TRUNC:.+]] = and i8 %0, 1
+// CHECK-NEXT: %[[R:.+]] = select i1 %[[IS_B]], i8 13, i8 %[[TRUNC]]
+// CHECK-NEXT: ret i8 %[[R]]
 #[no_mangle]
 pub fn match0(e: Enum0) -> u8 {
     use Enum0::*;
@@ -32,13 +37,14 @@ pub enum Enum1 {
     C,
 }
 
-// CHECK: define noundef{{( range\(i8 [0-9]+, [0-9]+\))?}} i8 @match1{{.*}}
+// CHECK-LABEL: define noundef{{( range\(i8 [0-9]+, [0-9]+\))?}} i8 @match1(i8{{.+}}%0)
 // CHECK-NEXT: start:
-// CHECK-NEXT: %1 = add{{( nsw)?}} i8 %0, -2
-// CHECK-NEXT: %2 = zext i8 %1 to i64
-// CHECK-NEXT: %3 = icmp ult i8 %1, 2
-// CHECK-NEXT: %4 = add nuw nsw i64 %2, 1
-// CHECK-NEXT: %_2 = select i1 %3, i64 %4, i64 0
+// CHECK-NEXT: %[[REL_VAR:.+]] = add{{( nsw)?}} i8 %0, -2
+// CHECK-NEXT: %[[REL_VAR_WIDE:.+]] = zext i8 %[[REL_VAR]] to i64
+// CHECK-NEXT: %[[IS_NICHE:.+]] = icmp ult i8 %[[REL_VAR]], 2
+// CHECK-NEXT: %[[NICHE_DISCR:.+]] = add nuw nsw i64 %[[REL_VAR_WIDE]], 1
+// CHECK-NEXT: %[[DISCR:.+]] = select i1 %[[IS_NICHE]], i64 %[[NICHE_DISCR]], i64 0
+// CHECK-NEXT: switch i64 %[[DISCR]]
 #[no_mangle]
 pub fn match1(e: Enum1) -> u8 {
     use Enum1::*;
@@ -92,14 +98,14 @@ pub enum Enum2 {
     E,
 }
 
-// CHECK: define noundef{{( range\(i8 [0-9]+, [0-9]+\))?}} i8 @match2{{.*}}
+// CHECK-LABEL: define noundef{{( range\(i8 [0-9]+, [0-9]+\))?}} i8 @match2(i8{{.+}}%0)
 // CHECK-NEXT: start:
-// CHECK-NEXT: %1 = add i8 %0, 2
-// CHECK-NEXT: %2 = zext i8 %1 to i64
-// CHECK-NEXT: %3 = icmp ult i8 %1, 4
-// CHECK-NEXT: %4 = add nuw nsw i64 %2, 1
-// CHECK-NEXT: %_2 = select i1 %3, i64 %4, i64 0
-// CHECK-NEXT: switch i64 %_2, label {{.*}} [
+// CHECK-NEXT: %[[REL_VAR:.+]] = add i8 %0, 2
+// CHECK-NEXT: %[[REL_VAR_WIDE:.+]] = zext i8 %[[REL_VAR]] to i64
+// CHECK-NEXT: %[[IS_NICHE:.+]] = icmp ult i8 %[[REL_VAR]], 4
+// CHECK-NEXT: %[[NICHE_DISCR:.+]] = add nuw nsw i64 %[[REL_VAR_WIDE]], 1
+// CHECK-NEXT: %[[DISCR:.+]] = select i1 %[[IS_NICHE]], i64 %[[NICHE_DISCR]], i64 0
+// CHECK-NEXT: switch i64 %[[DISCR]]
 #[no_mangle]
 pub fn match2(e: Enum2) -> u8 {
     use Enum2::*;
@@ -111,3 +117,357 @@ pub fn match2(e: Enum2) -> u8 {
         E => 250,
     }
 }
+
+// And make sure it works even if the niched scalar is a pointer.
+// (For example, that we don't try to `sub` on pointers.)
+
+// CHECK-LABEL: define noundef{{( range\(i16 -?[0-9]+, -?[0-9]+\))?}} i16 @match3(ptr{{.+}}%0)
+// CHECK-NEXT: start:
+// CHECK-NEXT: %[[IS_NULL:.+]] = icmp eq ptr %0, null
+// CHECK-NEXT: br i1 %[[IS_NULL]]
+#[no_mangle]
+pub fn match3(e: Option<&u8>) -> i16 {
+    match e {
+        Some(r) => *r as _,
+        None => -1,
+    }
+}
+
+// If the untagged variant is in the middle, there's an impossible value that's
+// not reflected in the `range` parameter attribute, so we assume it away.
+
+#[derive(PartialEq)]
+pub enum MiddleNiche {
+    A,
+    B,
+    C(bool),
+    D,
+    E,
+}
+
+// CHECK-LABEL: define noundef{{( range\(i8 -?[0-9]+, -?[0-9]+\))?}} i8 @match4(i8{{.+}}%0)
+// CHECK-NEXT: start:
+// CHECK-NEXT: %[[REL_VAR:.+]] = add{{( nsw)?}} i8 %0, -2
+// CHECK-NEXT: %[[IS_NICHE:.+]] = icmp ult i8 %[[REL_VAR]], 5
+// CHECK-NEXT: %[[NOT_IMPOSSIBLE:.+]] = icmp ne i8 %[[REL_VAR]], 2
+// CHECK-NEXT: call void @llvm.assume(i1 %[[NOT_IMPOSSIBLE]])
+// CHECK-NEXT: %[[DISCR:.+]] = select i1 %[[IS_NICHE]], i8 %[[REL_VAR]], i8 2
+// CHECK-NEXT: switch i8 %[[DISCR]]
+#[no_mangle]
+pub fn match4(e: MiddleNiche) -> u8 {
+    use MiddleNiche::*;
+    match e {
+        A => 13,
+        B => 100,
+        C(b) => b as u8,
+        D => 200,
+        E => 250,
+    }
+}
+
+// CHECK-LABEL: define{{.+}}i1 @match4_is_c(i8{{.+}}%e)
+// CHECK-NEXT: start
+// CHECK-NEXT: %[[REL_VAR:.+]] = add{{( nsw)?}} i8 %e, -2
+// CHECK-NEXT: %[[NOT_NICHE:.+]] = icmp ugt i8 %[[REL_VAR]], 4
+// CHECK-NEXT: %[[NOT_IMPOSSIBLE:.+]] = icmp ne i8 %[[REL_VAR]], 2
+// CHECK-NEXT: call void @llvm.assume(i1 %[[NOT_IMPOSSIBLE]])
+// CHECK-NEXT: ret i1 %[[NOT_NICHE]]
+#[no_mangle]
+pub fn match4_is_c(e: MiddleNiche) -> bool {
+    // Before #139098, this couldn't optimize out the `select` because it looked
+    // like it was possible for a `2` to be produced on both sides.
+
+    std::intrinsics::discriminant_value(&e) == 2
+}
+
+// You have to do something pretty obnoxious to get a variant index that doesn't
+// fit in the tag size, but it's possible
+
+pub enum Never {}
+
+pub enum HugeVariantIndex {
+    V000(Never),
+    V001(Never),
+    V002(Never),
+    V003(Never),
+    V004(Never),
+    V005(Never),
+    V006(Never),
+    V007(Never),
+    V008(Never),
+    V009(Never),
+    V010(Never),
+    V011(Never),
+    V012(Never),
+    V013(Never),
+    V014(Never),
+    V015(Never),
+    V016(Never),
+    V017(Never),
+    V018(Never),
+    V019(Never),
+    V020(Never),
+    V021(Never),
+    V022(Never),
+    V023(Never),
+    V024(Never),
+    V025(Never),
+    V026(Never),
+    V027(Never),
+    V028(Never),
+    V029(Never),
+    V030(Never),
+    V031(Never),
+    V032(Never),
+    V033(Never),
+    V034(Never),
+    V035(Never),
+    V036(Never),
+    V037(Never),
+    V038(Never),
+    V039(Never),
+    V040(Never),
+    V041(Never),
+    V042(Never),
+    V043(Never),
+    V044(Never),
+    V045(Never),
+    V046(Never),
+    V047(Never),
+    V048(Never),
+    V049(Never),
+    V050(Never),
+    V051(Never),
+    V052(Never),
+    V053(Never),
+    V054(Never),
+    V055(Never),
+    V056(Never),
+    V057(Never),
+    V058(Never),
+    V059(Never),
+    V060(Never),
+    V061(Never),
+    V062(Never),
+    V063(Never),
+    V064(Never),
+    V065(Never),
+    V066(Never),
+    V067(Never),
+    V068(Never),
+    V069(Never),
+    V070(Never),
+    V071(Never),
+    V072(Never),
+    V073(Never),
+    V074(Never),
+    V075(Never),
+    V076(Never),
+    V077(Never),
+    V078(Never),
+    V079(Never),
+    V080(Never),
+    V081(Never),
+    V082(Never),
+    V083(Never),
+    V084(Never),
+    V085(Never),
+    V086(Never),
+    V087(Never),
+    V088(Never),
+    V089(Never),
+    V090(Never),
+    V091(Never),
+    V092(Never),
+    V093(Never),
+    V094(Never),
+    V095(Never),
+    V096(Never),
+    V097(Never),
+    V098(Never),
+    V099(Never),
+    V100(Never),
+    V101(Never),
+    V102(Never),
+    V103(Never),
+    V104(Never),
+    V105(Never),
+    V106(Never),
+    V107(Never),
+    V108(Never),
+    V109(Never),
+    V110(Never),
+    V111(Never),
+    V112(Never),
+    V113(Never),
+    V114(Never),
+    V115(Never),
+    V116(Never),
+    V117(Never),
+    V118(Never),
+    V119(Never),
+    V120(Never),
+    V121(Never),
+    V122(Never),
+    V123(Never),
+    V124(Never),
+    V125(Never),
+    V126(Never),
+    V127(Never),
+    V128(Never),
+    V129(Never),
+    V130(Never),
+    V131(Never),
+    V132(Never),
+    V133(Never),
+    V134(Never),
+    V135(Never),
+    V136(Never),
+    V137(Never),
+    V138(Never),
+    V139(Never),
+    V140(Never),
+    V141(Never),
+    V142(Never),
+    V143(Never),
+    V144(Never),
+    V145(Never),
+    V146(Never),
+    V147(Never),
+    V148(Never),
+    V149(Never),
+    V150(Never),
+    V151(Never),
+    V152(Never),
+    V153(Never),
+    V154(Never),
+    V155(Never),
+    V156(Never),
+    V157(Never),
+    V158(Never),
+    V159(Never),
+    V160(Never),
+    V161(Never),
+    V162(Never),
+    V163(Never),
+    V164(Never),
+    V165(Never),
+    V166(Never),
+    V167(Never),
+    V168(Never),
+    V169(Never),
+    V170(Never),
+    V171(Never),
+    V172(Never),
+    V173(Never),
+    V174(Never),
+    V175(Never),
+    V176(Never),
+    V177(Never),
+    V178(Never),
+    V179(Never),
+    V180(Never),
+    V181(Never),
+    V182(Never),
+    V183(Never),
+    V184(Never),
+    V185(Never),
+    V186(Never),
+    V187(Never),
+    V188(Never),
+    V189(Never),
+    V190(Never),
+    V191(Never),
+    V192(Never),
+    V193(Never),
+    V194(Never),
+    V195(Never),
+    V196(Never),
+    V197(Never),
+    V198(Never),
+    V199(Never),
+    V200(Never),
+    V201(Never),
+    V202(Never),
+    V203(Never),
+    V204(Never),
+    V205(Never),
+    V206(Never),
+    V207(Never),
+    V208(Never),
+    V209(Never),
+    V210(Never),
+    V211(Never),
+    V212(Never),
+    V213(Never),
+    V214(Never),
+    V215(Never),
+    V216(Never),
+    V217(Never),
+    V218(Never),
+    V219(Never),
+    V220(Never),
+    V221(Never),
+    V222(Never),
+    V223(Never),
+    V224(Never),
+    V225(Never),
+    V226(Never),
+    V227(Never),
+    V228(Never),
+    V229(Never),
+    V230(Never),
+    V231(Never),
+    V232(Never),
+    V233(Never),
+    V234(Never),
+    V235(Never),
+    V236(Never),
+    V237(Never),
+    V238(Never),
+    V239(Never),
+    V240(Never),
+    V241(Never),
+    V242(Never),
+    V243(Never),
+    V244(Never),
+    V245(Never),
+    V246(Never),
+    V247(Never),
+    V248(Never),
+    V249(Never),
+    V250(Never),
+    V251(Never),
+    V252(Never),
+    V253(Never),
+    V254(Never),
+    V255(Never),
+    V256(Never),
+
+    Possible257,
+    Bool258(bool),
+    Possible259,
+}
+
+// CHECK-LABEL: define noundef{{( range\(i8 [0-9]+, [0-9]+\))?}} i8 @match5(i8{{.+}}%0)
+// CHECK-NEXT: start:
+// CHECK-NEXT: %[[REL_VAR:.+]] = add{{( nsw)?}} i8 %0, -2
+// CHECK-NEXT: %[[REL_VAR_WIDE:.+]] = zext i8 %[[REL_VAR]] to i64
+// CHECK-NEXT: %[[IS_NICHE:.+]] = icmp ult i8 %[[REL_VAR]], 3
+// CHECK-NEXT: %[[NOT_IMPOSSIBLE:.+]] = icmp ne i8 %[[REL_VAR]], 1
+// CHECK-NEXT: call void @llvm.assume(i1 %[[NOT_IMPOSSIBLE]])
+// CHECK-NEXT: %[[NICHE_DISCR:.+]] = add nuw nsw i64 %[[REL_VAR_WIDE]], 257
+// CHECK-NEXT: %[[DISCR:.+]] = select i1 %[[IS_NICHE]], i64 %[[NICHE_DISCR]], i64 258
+// CHECK-NEXT: switch i64 %[[DISCR]],
+// CHECK-NEXT:   i64 257,
+// CHECK-NEXT:   i64 258,
+// CHECK-NEXT:   i64 259,
+#[no_mangle]
+pub fn match5(e: HugeVariantIndex) -> u8 {
+    use HugeVariantIndex::*;
+    match e {
+        Possible257 => 13,
+        Bool258(b) => b as u8,
+        Possible259 => 100,
+    }
+}
diff --git a/tests/codegen/enum/enum-two-variants-match.rs b/tests/codegen/enum/enum-two-variants-match.rs
index 21ae1f96bca..12d9edc4d62 100644
--- a/tests/codegen/enum/enum-two-variants-match.rs
+++ b/tests/codegen/enum/enum-two-variants-match.rs
@@ -1,8 +1,12 @@
 //@ compile-flags: -Copt-level=3 -C no-prepopulate-passes
-//@ only-x86_64 (because these discriminants are isize)
+//@ only-64bit (because these discriminants are isize)
 
 #![crate_type = "lib"]
 
+// This directly tests what we emit for these matches, rather than what happens
+// after optimization, so it doesn't need to worry about extra flags on the
+// instructions and is less susceptible to being broken on LLVM updates.
+
 // CHECK-LABEL: @option_match
 #[no_mangle]
 pub fn option_match(x: Option<i32>) -> u16 {
@@ -51,3 +55,76 @@ pub fn result_match(x: Result<u64, i64>) -> u16 {
         Ok(_) => 42,
     }
 }
+
+// CHECK-LABEL: @option_bool_match(
+#[no_mangle]
+pub fn option_bool_match(x: Option<bool>) -> char {
+    // CHECK: %[[RAW:.+]] = load i8, ptr %x
+    // CHECK: %[[IS_NONE:.+]] = icmp eq i8 %[[RAW]], 2
+    // CHECK: %[[OPT_DISCR:.+]] = select i1 %[[IS_NONE]], i64 0, i64 1
+    // CHECK: %[[OPT_DISCR_T:.+]] = trunc nuw i64 %[[OPT_DISCR]] to i1
+    // CHECK: br i1 %[[OPT_DISCR_T]], label %[[BB_SOME:.+]], label %[[BB_NONE:.+]]
+
+    // CHECK: [[BB_SOME]]:
+    // CHECK: %[[FIELD:.+]] = load i8, ptr %x
+    // CHECK: %[[FIELD_T:.+]] = trunc nuw i8 %[[FIELD]] to i1
+    // CHECK: br i1 %[[FIELD_T]]
+    match x {
+        None => 'n',
+        Some(false) => 'f',
+        Some(true) => 't',
+    }
+}
+
+use std::cmp::Ordering::{self, *};
+// CHECK-LABEL: @option_ordering_match(
+#[no_mangle]
+pub fn option_ordering_match(x: Option<Ordering>) -> char {
+    // CHECK: %[[RAW:.+]] = load i8, ptr %x
+    // CHECK: %[[IS_NONE:.+]] = icmp eq i8 %[[RAW]], 2
+    // CHECK: %[[OPT_DISCR:.+]] = select i1 %[[IS_NONE]], i64 0, i64 1
+    // CHECK: %[[OPT_DISCR_T:.+]] = trunc nuw i64 %[[OPT_DISCR]] to i1
+    // CHECK: br i1 %[[OPT_DISCR_T]], label %[[BB_SOME:.+]], label %[[BB_NONE:.+]]
+
+    // CHECK: [[BB_SOME]]:
+    // CHECK: %[[FIELD:.+]] = load i8, ptr %x
+    // CHECK: switch i8 %[[FIELD]], label %[[UNREACHABLE:.+]] [
+    // CHECK-NEXT: i8 -1, label
+    // CHECK-NEXT: i8 0, label
+    // CHECK-NEXT: i8 1, label
+    // CHECK-NEXT: ]
+
+    // CHECK: [[UNREACHABLE]]:
+    // CHECK-NEXT: unreachable
+    match x {
+        None => '?',
+        Some(Less) => '<',
+        Some(Equal) => '=',
+        Some(Greater) => '>',
+    }
+}
+
+// CHECK-LABEL: @option_nonzero_match(
+#[no_mangle]
+pub fn option_nonzero_match(x: Option<std::num::NonZero<u16>>) -> u16 {
+    // CHECK: %[[OUT:.+]] = alloca [2 x i8]
+
+    // CHECK: %[[IS_NONE:.+]] = icmp eq i16 %x, 0
+    // CHECK: %[[OPT_DISCR:.+]] = select i1 %[[IS_NONE]], i64 0, i64 1
+    // CHECK: %[[OPT_DISCR_T:.+]] = trunc nuw i64 %[[OPT_DISCR]] to i1
+    // CHECK: br i1 %[[OPT_DISCR_T]], label %[[BB_SOME:.+]], label %[[BB_NONE:.+]]
+
+    // CHECK: [[BB_SOME]]:
+    // CHECK: store i16 987, ptr %[[OUT]]
+
+    // CHECK: [[BB_NONE]]:
+    // CHECK: store i16 123, ptr %[[OUT]]
+
+    // CHECK: %[[RET:.+]] = load i16, ptr %[[OUT]]
+    // CHECK: ret i16 %[[RET]]
+
+    match x {
+        None => 123,
+        Some(_) => 987,
+    }
+}
diff --git a/tests/codegen/ergonomic-clones/closure.rs b/tests/codegen/ergonomic-clones/closure.rs
new file mode 100644
index 00000000000..b6fc8172641
--- /dev/null
+++ b/tests/codegen/ergonomic-clones/closure.rs
@@ -0,0 +1,55 @@
+//@ compile-flags: -C no-prepopulate-passes -Copt-level=0 -Zmir-opt-level=0
+
+#![crate_type = "lib"]
+
+#![feature(ergonomic_clones)]
+#![allow(incomplete_features)]
+
+use std::clone::UseCloned;
+
+pub fn ergonomic_clone_closure_move() -> String {
+    let s = String::from("hi");
+
+    // CHECK-NOT: ; call core::clone::impls::<impl core::clone::Clone for String>::clone
+    let cl = use || s;
+    cl()
+}
+
+#[derive(Clone)]
+struct Foo;
+
+impl UseCloned for Foo {}
+
+pub fn ergonomic_clone_closure_use_cloned() -> Foo {
+    let f = Foo;
+
+    // CHECK: ; call <closure::Foo as core::clone::Clone>::clone
+    let f1 = use || f;
+
+    // CHECK: ; call <closure::Foo as core::clone::Clone>::clone
+    let f2 = use || f;
+
+    f
+}
+
+pub fn ergonomic_clone_closure_copy() -> i32 {
+    let i = 1;
+
+    // CHECK-NOT: ; call core::clone::impls::<impl core::clone::Clone for i32>::clone
+    let i1 = use || i;
+
+    // CHECK-NOT: ; call core::clone::impls::<impl core::clone::Clone for i32>::clone
+    let i2 = use || i;
+
+    i
+}
+
+pub fn ergonomic_clone_closure_use_cloned_generics<T: UseCloned>(f: T) -> T {
+    // CHECK-NOT: ; call core::clone::impls::<impl core::clone::Clone for i32>::clone
+    let f1 = use || f;
+
+    // CHECK-NOT: ; call core::clone::impls::<impl core::clone::Clone for i32>::clone
+    let f2 = use || f;
+
+    f
+}
diff --git a/tests/codegen/intrinsic-no-unnamed-attr.rs b/tests/codegen/intrinsic-no-unnamed-attr.rs
index fce0de80d7b..35eb025ab6b 100644
--- a/tests/codegen/intrinsic-no-unnamed-attr.rs
+++ b/tests/codegen/intrinsic-no-unnamed-attr.rs
@@ -2,9 +2,9 @@
 
 #![feature(intrinsics)]
 
-extern "rust-intrinsic" {
-    fn sqrtf32(x: f32) -> f32;
-}
+#[rustc_intrinsic]
+unsafe fn sqrtf32(x: f32) -> f32;
+
 // CHECK: @llvm.sqrt.f32(float) #{{[0-9]*}}
 
 fn main() {
diff --git a/tests/codegen/intrinsics/nontemporal.rs b/tests/codegen/intrinsics/nontemporal.rs
index 1d4fae83c29..a151d4bd297 100644
--- a/tests/codegen/intrinsics/nontemporal.rs
+++ b/tests/codegen/intrinsics/nontemporal.rs
@@ -18,9 +18,8 @@
 extern crate minicore;
 use minicore::*;
 
-extern "rust-intrinsic" {
-    pub fn nontemporal_store<T>(ptr: *mut T, val: T);
-}
+#[rustc_intrinsic]
+pub unsafe fn nontemporal_store<T>(ptr: *mut T, val: T);
 
 #[no_mangle]
 pub fn a(a: &mut u32, b: u32) {
diff --git a/tests/codegen/string-push.rs b/tests/codegen/string-push.rs
new file mode 100644
index 00000000000..cf5f6bb1aa3
--- /dev/null
+++ b/tests/codegen/string-push.rs
@@ -0,0 +1,11 @@
+//! Check that `String::push` is optimized enough not to call `memcpy`.
+
+//@ compile-flags: -O
+#![crate_type = "lib"]
+
+// CHECK-LABEL: @string_push_does_not_call_memcpy
+#[no_mangle]
+pub fn string_push_does_not_call_memcpy(s: &mut String, ch: char) {
+    // CHECK-NOT: call void @llvm.memcpy
+    s.push(ch);
+}