Rename `tests/codegen` into `tests/codegen-llvm`

31 files changed, 1877 insertions, 0 deletions

diff --git a/tests/codegen-llvm/intrinsics/aggregate-thin-pointer.rs b/tests/codegen-llvm/intrinsics/aggregate-thin-pointer.rs
new file mode 100644
index 00000000000..bd590ce9180
--- /dev/null
+++ b/tests/codegen-llvm/intrinsics/aggregate-thin-pointer.rs
@@ -0,0 +1,23 @@
+//@ compile-flags: -Copt-level=3 -C no-prepopulate-passes -Z mir-enable-passes=-InstSimplify
+//@ only-64bit (so I don't need to worry about usize)
+
+#![crate_type = "lib"]
+#![feature(core_intrinsics)]
+
+use std::intrinsics::aggregate_raw_ptr;
+
+// InstSimplify replaces these with casts if it can, which means they're almost
+// never seen in codegen, but PR#121571 found a way, so add a test for it.
+
+#[inline(never)]
+pub fn opaque(_p: &*const i32) {}
+
+// CHECK-LABEL: @thin_ptr_via_aggregate(
+#[no_mangle]
+pub unsafe fn thin_ptr_via_aggregate(p: *const ()) {
+    // CHECK: %mem = alloca
+    // CHECK: store ptr %p, ptr %mem
+    // CHECK: call {{.+}}aggregate_thin_pointer{{.+}} %mem)
+    let mem = aggregate_raw_ptr(p, ());
+    opaque(&mem);
+}
diff --git a/tests/codegen-llvm/intrinsics/carrying_mul_add.rs b/tests/codegen-llvm/intrinsics/carrying_mul_add.rs
new file mode 100644
index 00000000000..21fb49a3786
--- /dev/null
+++ b/tests/codegen-llvm/intrinsics/carrying_mul_add.rs
@@ -0,0 +1,136 @@
+//@ revisions: RAW OPT
+//@ compile-flags: -C opt-level=1
+//@[RAW] compile-flags: -C no-prepopulate-passes
+
+#![crate_type = "lib"]
+#![feature(core_intrinsics)]
+#![feature(core_intrinsics_fallbacks)]
+
+// Note that LLVM seems to sometimes permute the order of arguments to mul and add,
+// so these tests don't check the arguments in the optimized revision.
+
+use std::intrinsics::{carrying_mul_add, fallback};
+
+// The fallbacks are emitted even when they're never used, but optimize out.
+
+// RAW: wide_mul_u128
+// OPT-NOT: wide_mul_u128
+
+// CHECK-LABEL: @cma_u8
+#[no_mangle]
+pub unsafe fn cma_u8(a: u8, b: u8, c: u8, d: u8) -> (u8, u8) {
+    // CHECK: [[A:%.+]] = zext i8 %a to i16
+    // CHECK: [[B:%.+]] = zext i8 %b to i16
+    // CHECK: [[C:%.+]] = zext i8 %c to i16
+    // CHECK: [[D:%.+]] = zext i8 %d to i16
+    // CHECK: [[AB:%.+]] = mul nuw i16
+    // RAW-SAME: [[A]], [[B]]
+    // CHECK: [[ABC:%.+]] = add nuw i16
+    // RAW-SAME: [[AB]], [[C]]
+    // CHECK: [[ABCD:%.+]] = add nuw i16
+    // RAW-SAME: [[ABC]], [[D]]
+    // CHECK: [[LOW:%.+]] = trunc i16 [[ABCD]] to i8
+    // CHECK: [[HIGHW:%.+]] = lshr i16 [[ABCD]], 8
+    // RAW: [[HIGH:%.+]] = trunc i16 [[HIGHW]] to i8
+    // OPT: [[HIGH:%.+]] = trunc nuw i16 [[HIGHW]] to i8
+    // CHECK: [[PAIR0:%.+]] = insertvalue { i8, i8 } poison, i8 [[LOW]], 0
+    // CHECK: [[PAIR1:%.+]] = insertvalue { i8, i8 } [[PAIR0]], i8 [[HIGH]], 1
+    // OPT: ret { i8, i8 } [[PAIR1]]
+    carrying_mul_add(a, b, c, d)
+}
+
+// CHECK-LABEL: @cma_u32
+#[no_mangle]
+pub unsafe fn cma_u32(a: u32, b: u32, c: u32, d: u32) -> (u32, u32) {
+    // CHECK: [[A:%.+]] = zext i32 %a to i64
+    // CHECK: [[B:%.+]] = zext i32 %b to i64
+    // CHECK: [[C:%.+]] = zext i32 %c to i64
+    // CHECK: [[D:%.+]] = zext i32 %d to i64
+    // CHECK: [[AB:%.+]] = mul nuw i64
+    // RAW-SAME: [[A]], [[B]]
+    // CHECK: [[ABC:%.+]] = add nuw i64
+    // RAW-SAME: [[AB]], [[C]]
+    // CHECK: [[ABCD:%.+]] = add nuw i64
+    // RAW-SAME: [[ABC]], [[D]]
+    // CHECK: [[LOW:%.+]] = trunc i64 [[ABCD]] to i32
+    // CHECK: [[HIGHW:%.+]] = lshr i64 [[ABCD]], 32
+    // RAW: [[HIGH:%.+]] = trunc i64 [[HIGHW]] to i32
+    // OPT: [[HIGH:%.+]] = trunc nuw i64 [[HIGHW]] to i32
+    // CHECK: [[PAIR0:%.+]] = insertvalue { i32, i32 } poison, i32 [[LOW]], 0
+    // CHECK: [[PAIR1:%.+]] = insertvalue { i32, i32 } [[PAIR0]], i32 [[HIGH]], 1
+    // OPT: ret { i32, i32 } [[PAIR1]]
+    carrying_mul_add(a, b, c, d)
+}
+
+// CHECK-LABEL: @cma_u128
+// CHECK-SAME: sret{{.+}}dereferenceable(32){{.+}}%_0,{{.+}}%a,{{.+}}%b,{{.+}}%c,{{.+}}%d
+#[no_mangle]
+pub unsafe fn cma_u128(a: u128, b: u128, c: u128, d: u128) -> (u128, u128) {
+    // CHECK: [[A:%.+]] = zext i128 %a to i256
+    // CHECK: [[B:%.+]] = zext i128 %b to i256
+    // CHECK: [[C:%.+]] = zext i128 %c to i256
+    // CHECK: [[D:%.+]] = zext i128 %d to i256
+    // CHECK: [[AB:%.+]] = mul nuw i256
+    // RAW-SAME: [[A]], [[B]]
+    // CHECK: [[ABC:%.+]] = add nuw i256
+    // RAW-SAME: [[AB]], [[C]]
+    // CHECK: [[ABCD:%.+]] = add nuw i256
+    // RAW-SAME: [[ABC]], [[D]]
+    // CHECK: [[LOW:%.+]] = trunc i256 [[ABCD]] to i128
+    // CHECK: [[HIGHW:%.+]] = lshr i256 [[ABCD]], 128
+    // RAW: [[HIGH:%.+]] = trunc i256 [[HIGHW]] to i128
+    // OPT: [[HIGH:%.+]] = trunc nuw i256 [[HIGHW]] to i128
+    // RAW: [[PAIR0:%.+]] = insertvalue { i128, i128 } poison, i128 [[LOW]], 0
+    // RAW: [[PAIR1:%.+]] = insertvalue { i128, i128 } [[PAIR0]], i128 [[HIGH]], 1
+    // OPT: store i128 [[LOW]], ptr %_0
+    // OPT: [[P1:%.+]] = getelementptr inbounds{{( nuw)?}} i8, ptr %_0, {{i32|i64}} 16
+    // OPT: store i128 [[HIGH]], ptr [[P1]]
+    // CHECK: ret void
+    carrying_mul_add(a, b, c, d)
+}
+
+// CHECK-LABEL: @cma_i128
+// CHECK-SAME: sret{{.+}}dereferenceable(32){{.+}}%_0,{{.+}}%a,{{.+}}%b,{{.+}}%c,{{.+}}%d
+#[no_mangle]
+pub unsafe fn cma_i128(a: i128, b: i128, c: i128, d: i128) -> (u128, i128) {
+    // CHECK: [[A:%.+]] = sext i128 %a to i256
+    // CHECK: [[B:%.+]] = sext i128 %b to i256
+    // CHECK: [[C:%.+]] = sext i128 %c to i256
+    // CHECK: [[D:%.+]] = sext i128 %d to i256
+    // CHECK: [[AB:%.+]] = mul nsw i256
+    // RAW-SAME: [[A]], [[B]]
+    // CHECK: [[ABC:%.+]] = add nsw i256
+    // RAW-SAME: [[AB]], [[C]]
+    // CHECK: [[ABCD:%.+]] = add nsw i256
+    // RAW-SAME: [[ABC]], [[D]]
+    // CHECK: [[LOW:%.+]] = trunc i256 [[ABCD]] to i128
+    // CHECK: [[HIGHW:%.+]] = lshr i256 [[ABCD]], 128
+    // RAW: [[HIGH:%.+]] = trunc i256 [[HIGHW]] to i128
+    // OPT: [[HIGH:%.+]] = trunc nuw i256 [[HIGHW]] to i128
+    // RAW: [[PAIR0:%.+]] = insertvalue { i128, i128 } poison, i128 [[LOW]], 0
+    // RAW: [[PAIR1:%.+]] = insertvalue { i128, i128 } [[PAIR0]], i128 [[HIGH]], 1
+    // OPT: store i128 [[LOW]], ptr %_0
+    // OPT: [[P1:%.+]] = getelementptr inbounds{{( nuw)?}} i8, ptr %_0, {{i32|i64}} 16
+    // OPT: store i128 [[HIGH]], ptr [[P1]]
+    // CHECK: ret void
+    carrying_mul_add(a, b, c, d)
+}
+
+// CHECK-LABEL: @fallback_cma_u32
+#[no_mangle]
+pub unsafe fn fallback_cma_u32(a: u32, b: u32, c: u32, d: u32) -> (u32, u32) {
+    // OPT-DAG: [[A:%.+]] = zext i32 %a to i64
+    // OPT-DAG: [[B:%.+]] = zext i32 %b to i64
+    // OPT-DAG: [[AB:%.+]] = mul nuw i64
+    // OPT-DAG: [[C:%.+]] = zext i32 %c to i64
+    // OPT-DAG: [[ABC:%.+]] = add nuw i64{{.+}}[[C]]
+    // OPT-DAG: [[D:%.+]] = zext i32 %d to i64
+    // OPT-DAG: [[ABCD:%.+]] = add nuw i64{{.+}}[[D]]
+    // OPT-DAG: [[LOW:%.+]] = trunc i64 [[ABCD]] to i32
+    // OPT-DAG: [[HIGHW:%.+]] = lshr i64 [[ABCD]], 32
+    // OPT-DAG: [[HIGH:%.+]] = trunc nuw i64 [[HIGHW]] to i32
+    // OPT-DAG: [[PAIR0:%.+]] = insertvalue { i32, i32 } poison, i32 [[LOW]], 0
+    // OPT-DAG: [[PAIR1:%.+]] = insertvalue { i32, i32 } [[PAIR0]], i32 [[HIGH]], 1
+    // OPT-DAG: ret { i32, i32 } [[PAIR1]]
+    fallback::CarryingMulAdd::carrying_mul_add(a, b, c, d)
+}
diff --git a/tests/codegen-llvm/intrinsics/cold_path.rs b/tests/codegen-llvm/intrinsics/cold_path.rs
new file mode 100644
index 00000000000..fd75324b671
--- /dev/null
+++ b/tests/codegen-llvm/intrinsics/cold_path.rs
@@ -0,0 +1,13 @@
+//@ compile-flags: -Copt-level=3
+#![crate_type = "lib"]
+#![feature(core_intrinsics)]
+
+use std::intrinsics::cold_path;
+
+#[no_mangle]
+pub fn test_cold_path(x: bool) {
+    cold_path();
+}
+
+// CHECK-LABEL: @test_cold_path(
+// CHECK-NOT: cold_path
diff --git a/tests/codegen-llvm/intrinsics/cold_path2.rs b/tests/codegen-llvm/intrinsics/cold_path2.rs
new file mode 100644
index 00000000000..0891c878fd9
--- /dev/null
+++ b/tests/codegen-llvm/intrinsics/cold_path2.rs
@@ -0,0 +1,37 @@
+//@ compile-flags: -O
+#![crate_type = "lib"]
+#![feature(core_intrinsics)]
+
+use std::intrinsics::cold_path;
+
+#[inline(never)]
+#[no_mangle]
+pub fn path_a() {
+    println!("path a");
+}
+
+#[inline(never)]
+#[no_mangle]
+pub fn path_b() {
+    println!("path b");
+}
+
+#[no_mangle]
+pub fn test(x: Option<bool>) {
+    if let Some(_) = x {
+        path_a();
+    } else {
+        cold_path();
+        path_b();
+    }
+
+    // CHECK-LABEL: void @test(i8{{.+}}%x)
+    // CHECK: %[[IS_NONE:.+]] = icmp eq i8 %x, 2
+    // CHECK: br i1 %[[IS_NONE]], label %bb2, label %bb1, !prof ![[NUM:[0-9]+]]
+    // CHECK: bb1:
+    // CHECK: path_a
+    // CHECK: bb2:
+    // CHECK: path_b
+}
+
+// CHECK: ![[NUM]] = !{!"branch_weights", {{(!"expected", )?}}i32 1, i32 2000}
diff --git a/tests/codegen-llvm/intrinsics/cold_path3.rs b/tests/codegen-llvm/intrinsics/cold_path3.rs
new file mode 100644
index 00000000000..bf3347de665
--- /dev/null
+++ b/tests/codegen-llvm/intrinsics/cold_path3.rs
@@ -0,0 +1,87 @@
+//@ compile-flags: -O
+#![crate_type = "lib"]
+#![feature(core_intrinsics)]
+
+use std::intrinsics::cold_path;
+
+#[inline(never)]
+#[no_mangle]
+pub fn path_a() {
+    println!("path a");
+}
+
+#[inline(never)]
+#[no_mangle]
+pub fn path_b() {
+    println!("path b");
+}
+
+#[inline(never)]
+#[no_mangle]
+pub fn path_c() {
+    println!("path c");
+}
+
+#[inline(never)]
+#[no_mangle]
+pub fn path_d() {
+    println!("path d");
+}
+
+#[no_mangle]
+pub fn test(x: Option<u32>) {
+    match x {
+        Some(0) => path_a(),
+        Some(1) => {
+            cold_path();
+            path_b()
+        }
+        Some(2) => path_c(),
+        Some(3) => {
+            cold_path();
+            path_d()
+        }
+        _ => path_a(),
+    }
+
+    // CHECK-LABEL: @test(
+    // CHECK: switch i32 %1, label %bb1 [
+    // CHECK: i32 0, label %bb6
+    // CHECK: i32 1, label %bb5
+    // CHECK: i32 2, label %bb4
+    // CHECK: i32 3, label %bb3
+    // CHECK: ], !prof ![[NUM1:[0-9]+]]
+}
+
+#[no_mangle]
+pub fn test2(x: Option<u32>) {
+    match x {
+        Some(10) => path_a(),
+        Some(11) => {
+            cold_path();
+            path_b()
+        }
+        Some(12) => {
+            unsafe { core::intrinsics::unreachable() };
+            path_c()
+        }
+        Some(13) => {
+            cold_path();
+            path_d()
+        }
+        _ => {
+            cold_path();
+            path_a()
+        }
+    }
+
+    // CHECK-LABEL: @test2(
+    // CHECK: switch i32 %1, label %bb1 [
+    // CHECK: i32 10, label %bb5
+    // CHECK: i32 11, label %bb4
+    // CHECK: i32 13, label %bb3
+    // CHECK: ], !prof ![[NUM2:[0-9]+]]
+}
+
+// CHECK: ![[NUM1]] = !{!"branch_weights", i32 2000, i32 2000, i32 1, i32 2000, i32 1}
+// CHECK: ![[NUM2]] = !{!"branch_weights", i32 1, i32 2000, i32 1, i32 1}
diff --git a/tests/codegen-llvm/intrinsics/compare_bytes.rs b/tests/codegen-llvm/intrinsics/compare_bytes.rs
new file mode 100644
index 00000000000..3ab0e4e97e0
--- /dev/null
+++ b/tests/codegen-llvm/intrinsics/compare_bytes.rs
@@ -0,0 +1,34 @@
+//@ revisions: INT32 INT16
+//@ compile-flags: -Copt-level=3
+//@ [INT32] ignore-16bit
+//@ [INT16] only-16bit
+
+#![crate_type = "lib"]
+#![feature(core_intrinsics)]
+
+use std::intrinsics::compare_bytes;
+
+#[no_mangle]
+// CHECK-LABEL: @bytes_cmp(
+pub unsafe fn bytes_cmp(a: *const u8, b: *const u8, n: usize) -> i32 {
+    // INT32: %[[TEMP:.+]] = tail call i32 @memcmp(ptr %a, ptr %b, {{i32|i64}} %n)
+    // INT32-NOT: sext
+    // INT32: ret i32 %[[TEMP]]
+
+    // INT16: %[[TEMP1:.+]] = tail call i16 @memcmp(ptr %a, ptr %b, i16 %n)
+    // INT16: %[[TEMP2:.+]] = sext i16 %[[TEMP1]] to i32
+    // INT16: ret i32 %[[TEMP2]]
+    compare_bytes(a, b, n)
+}
+
+// Ensure that, even though there's an `sext` emitted by the intrinsic,
+// that doesn't end up pessiming checks against zero.
+#[no_mangle]
+// CHECK-LABEL: @bytes_eq(
+pub unsafe fn bytes_eq(a: *const u8, b: *const u8, n: usize) -> bool {
+    // CHECK: call {{.+}} @{{bcmp|memcmp}}(ptr %a, ptr %b, {{i16|i32|i64}} %n)
+    // CHECK-NOT: sext
+    // INT32: icmp eq i32
+    // INT16: icmp eq i16
+    compare_bytes(a, b, n) == 0_i32
+}
diff --git a/tests/codegen-llvm/intrinsics/const_eval_select.rs b/tests/codegen-llvm/intrinsics/const_eval_select.rs
new file mode 100644
index 00000000000..baa985b00cd
--- /dev/null
+++ b/tests/codegen-llvm/intrinsics/const_eval_select.rs
@@ -0,0 +1,22 @@
+//@ compile-flags: -C no-prepopulate-passes -Copt-level=0
+
+#![crate_type = "lib"]
+#![feature(const_eval_select)]
+#![feature(core_intrinsics)]
+
+use std::intrinsics::const_eval_select;
+
+const fn foo(_: i32) -> i32 {
+    1
+}
+
+#[no_mangle]
+pub fn hi(n: i32) -> i32 {
+    n
+}
+
+#[no_mangle]
+pub unsafe fn hey() {
+    // CHECK: call i32 @hi(i32
+    const_eval_select((42,), foo, hi);
+}
diff --git a/tests/codegen-llvm/intrinsics/ctlz.rs b/tests/codegen-llvm/intrinsics/ctlz.rs
new file mode 100644
index 00000000000..0d54d21ce12
--- /dev/null
+++ b/tests/codegen-llvm/intrinsics/ctlz.rs
@@ -0,0 +1,56 @@
+//@ compile-flags: -C no-prepopulate-passes
+
+#![crate_type = "lib"]
+#![feature(core_intrinsics)]
+
+use std::intrinsics::{ctlz, ctlz_nonzero};
+
+// CHECK-LABEL: @ctlz_u16
+#[no_mangle]
+pub unsafe fn ctlz_u16(x: u16) -> u32 {
+    // CHECK: %[[tmp:.*]] = call i16 @llvm.ctlz.i16(i16 %x, i1 false)
+    // CHECK: zext i16 %[[tmp]] to i32
+    ctlz(x)
+}
+
+// CHECK-LABEL: @ctlz_nzu16
+#[no_mangle]
+pub unsafe fn ctlz_nzu16(x: u16) -> u32 {
+    // CHECK: %[[tmp:.*]] = call i16 @llvm.ctlz.i16(i16 %x, i1 true)
+    // CHECK: zext i16 %[[tmp]] to i32
+    ctlz_nonzero(x)
+}
+
+// CHECK-LABEL: @ctlz_u32
+#[no_mangle]
+pub unsafe fn ctlz_u32(x: u32) -> u32 {
+    // CHECK: call i32 @llvm.ctlz.i32(i32 %x, i1 false)
+    // CHECK-NOT: zext
+    // CHECK-NOT: trunc
+    ctlz(x)
+}
+
+// CHECK-LABEL: @ctlz_nzu32
+#[no_mangle]
+pub unsafe fn ctlz_nzu32(x: u32) -> u32 {
+    // CHECK: call i32 @llvm.ctlz.i32(i32 %x, i1 true)
+    // CHECK-NOT: zext
+    // CHECK-NOT: trunc
+    ctlz_nonzero(x)
+}
+
+// CHECK-LABEL: @ctlz_u64
+#[no_mangle]
+pub unsafe fn ctlz_u64(x: u64) -> u32 {
+    // CHECK: %[[tmp:.*]] = call i64 @llvm.ctlz.i64(i64 %x, i1 false)
+    // CHECK: trunc i64 %[[tmp]] to i32
+    ctlz(x)
+}
+
+// CHECK-LABEL: @ctlz_nzu64
+#[no_mangle]
+pub unsafe fn ctlz_nzu64(x: u64) -> u32 {
+    // CHECK: %[[tmp:.*]] = call i64 @llvm.ctlz.i64(i64 %x, i1 true)
+    // CHECK: trunc i64 %[[tmp]] to i32
+    ctlz_nonzero(x)
+}
diff --git a/tests/codegen-llvm/intrinsics/ctpop.rs b/tests/codegen-llvm/intrinsics/ctpop.rs
new file mode 100644
index 00000000000..f4043325de9
--- /dev/null
+++ b/tests/codegen-llvm/intrinsics/ctpop.rs
@@ -0,0 +1,31 @@
+//@ compile-flags: -C no-prepopulate-passes
+
+#![crate_type = "lib"]
+#![feature(core_intrinsics)]
+
+use std::intrinsics::ctpop;
+
+// CHECK-LABEL: @ctpop_u16
+#[no_mangle]
+pub unsafe fn ctpop_u16(x: u16) -> u32 {
+    // CHECK: %[[tmp:.*]] = call i16 @llvm.ctpop.i16(i16 %x)
+    // CHECK: zext i16 %[[tmp]] to i32
+    ctpop(x)
+}
+
+// CHECK-LABEL: @ctpop_u32
+#[no_mangle]
+pub unsafe fn ctpop_u32(x: u32) -> u32 {
+    // CHECK: call i32 @llvm.ctpop.i32(i32 %x)
+    // CHECK-NOT: zext
+    // CHECK-NOT: trunc
+    ctpop(x)
+}
+
+// CHECK-LABEL: @ctpop_u64
+#[no_mangle]
+pub unsafe fn ctpop_u64(x: u64) -> u32 {
+    // CHECK: %[[tmp:.*]] = call i64 @llvm.ctpop.i64(i64 %x)
+    // CHECK: trunc i64 %[[tmp]] to i32
+    ctpop(x)
+}
diff --git a/tests/codegen-llvm/intrinsics/disjoint_bitor.rs b/tests/codegen-llvm/intrinsics/disjoint_bitor.rs
new file mode 100644
index 00000000000..fc45439ee0b
--- /dev/null
+++ b/tests/codegen-llvm/intrinsics/disjoint_bitor.rs
@@ -0,0 +1,30 @@
+//@ compile-flags: -C no-prepopulate-passes -Z mir-opt-level=0
+
+#![crate_type = "lib"]
+#![feature(core_intrinsics)]
+
+use std::intrinsics::disjoint_bitor;
+
+// CHECK-LABEL: @disjoint_bitor_signed
+#[no_mangle]
+pub unsafe fn disjoint_bitor_signed(x: i32, y: i32) -> i32 {
+    // CHECK: or disjoint i32 %x, %y
+    disjoint_bitor(x, y)
+}
+
+// CHECK-LABEL: @disjoint_bitor_unsigned
+#[no_mangle]
+pub unsafe fn disjoint_bitor_unsigned(x: u64, y: u64) -> u64 {
+    // CHECK: or disjoint i64 %x, %y
+    disjoint_bitor(x, y)
+}
+
+// CHECK-LABEL: @disjoint_bitor_literal
+#[no_mangle]
+pub unsafe fn disjoint_bitor_literal() -> u8 {
+    // This is a separate check because even without any passes,
+    // LLVM will fold so it's not an instruction, which can assert in LLVM.
+
+    // CHECK: store i8 3
+    disjoint_bitor(1, 2)
+}
diff --git a/tests/codegen-llvm/intrinsics/exact_div.rs b/tests/codegen-llvm/intrinsics/exact_div.rs
new file mode 100644
index 00000000000..dc625ba7fe4
--- /dev/null
+++ b/tests/codegen-llvm/intrinsics/exact_div.rs
@@ -0,0 +1,20 @@
+//@ compile-flags: -C no-prepopulate-passes
+
+#![crate_type = "lib"]
+#![feature(core_intrinsics)]
+
+use std::intrinsics::exact_div;
+
+// CHECK-LABEL: @exact_sdiv
+#[no_mangle]
+pub unsafe fn exact_sdiv(x: i32, y: i32) -> i32 {
+    // CHECK: sdiv exact
+    exact_div(x, y)
+}
+
+// CHECK-LABEL: @exact_udiv
+#[no_mangle]
+pub unsafe fn exact_udiv(x: u32, y: u32) -> u32 {
+    // CHECK: udiv exact
+    exact_div(x, y)
+}
diff --git a/tests/codegen-llvm/intrinsics/likely.rs b/tests/codegen-llvm/intrinsics/likely.rs
new file mode 100644
index 00000000000..c5e3c466f45
--- /dev/null
+++ b/tests/codegen-llvm/intrinsics/likely.rs
@@ -0,0 +1,35 @@
+//@ compile-flags: -Copt-level=3
+#![crate_type = "lib"]
+#![feature(core_intrinsics)]
+
+use std::intrinsics::likely;
+
+#[inline(never)]
+#[no_mangle]
+pub fn path_a() {
+    println!("path a");
+}
+
+#[inline(never)]
+#[no_mangle]
+pub fn path_b() {
+    println!("path b");
+}
+
+#[no_mangle]
+pub fn test_likely(x: bool) {
+    if likely(x) {
+        path_a();
+    } else {
+        path_b();
+    }
+}
+
+// CHECK-LABEL: @test_likely(
+// CHECK: br i1 %x, label %bb2, label %bb3, !prof ![[NUM:[0-9]+]]
+// CHECK: bb3:
+// CHECK-NOT: cold_path
+// CHECK: path_b
+// CHECK: bb2:
+// CHECK: path_a
+// CHECK: ![[NUM]] = !{!"branch_weights", {{(!"expected", )?}}i32 2000, i32 1}
diff --git a/tests/codegen-llvm/intrinsics/likely_assert.rs b/tests/codegen-llvm/intrinsics/likely_assert.rs
new file mode 100644
index 00000000000..87ffb4ee3fb
--- /dev/null
+++ b/tests/codegen-llvm/intrinsics/likely_assert.rs
@@ -0,0 +1,17 @@
+//@ compile-flags: -Copt-level=3
+#![crate_type = "lib"]
+
+#[no_mangle]
+pub fn test_assert(x: bool) {
+    assert!(x);
+}
+
+// check that assert! emits branch weights
+
+// CHECK-LABEL: @test_assert(
+// CHECK: br i1 %x, label %bb2, label %bb1, !prof ![[NUM:[0-9]+]]
+// CHECK: bb1:
+// CHECK: panic
+// CHECK: bb2:
+// CHECK: ret void
+// CHECK: ![[NUM]] = !{!"branch_weights", {{(!"expected", )?}}i32 2000, i32 1}
diff --git a/tests/codegen-llvm/intrinsics/mask.rs b/tests/codegen-llvm/intrinsics/mask.rs
new file mode 100644
index 00000000000..5344274678c
--- /dev/null
+++ b/tests/codegen-llvm/intrinsics/mask.rs
@@ -0,0 +1,12 @@
+//@ compile-flags: -Copt-level=0
+#![crate_type = "lib"]
+#![feature(core_intrinsics)]
+
+// CHECK-LABEL: @mask_ptr
+// CHECK-SAME: [[WORD:i[0-9]+]] %mask
+#[no_mangle]
+pub fn mask_ptr(ptr: *const u16, mask: usize) -> *const u16 {
+    // CHECK: call
+    // CHECK-SAME: @llvm.ptrmask.{{p0|p0i8}}.[[WORD]](ptr {{%ptr|%1}}, [[WORD]] %mask)
+    core::intrinsics::ptr_mask(ptr, mask)
+}
diff --git a/tests/codegen-llvm/intrinsics/nontemporal.rs b/tests/codegen-llvm/intrinsics/nontemporal.rs
new file mode 100644
index 00000000000..a151d4bd297
--- /dev/null
+++ b/tests/codegen-llvm/intrinsics/nontemporal.rs
@@ -0,0 +1,32 @@
+//@ add-core-stubs
+//@ compile-flags: -Copt-level=3
+//@revisions: with_nontemporal without_nontemporal
+//@[with_nontemporal] compile-flags: --target aarch64-unknown-linux-gnu
+//@[with_nontemporal] needs-llvm-components: aarch64
+//@[without_nontemporal] compile-flags: --target x86_64-unknown-linux-gnu
+//@[without_nontemporal] needs-llvm-components: x86
+
+// Ensure that we *do* emit the `!nontemporal` flag on architectures where it
+// is well-behaved, but do *not* emit it on architectures where it is ill-behaved.
+// For more context, see <https://github.com/rust-lang/rust/issues/114582> and
+// <https://github.com/llvm/llvm-project/issues/64521>.
+
+#![feature(no_core, lang_items, intrinsics)]
+#![no_core]
+#![crate_type = "lib"]
+
+extern crate minicore;
+use minicore::*;
+
+#[rustc_intrinsic]
+pub unsafe fn nontemporal_store<T>(ptr: *mut T, val: T);
+
+#[no_mangle]
+pub fn a(a: &mut u32, b: u32) {
+    // CHECK-LABEL: define{{.*}}void @a
+    // with_nontemporal: store i32 %b, ptr %a, align 4, !nontemporal
+    // without_nontemporal-NOT: nontemporal
+    unsafe {
+        nontemporal_store(a, b);
+    }
+}
diff --git a/tests/codegen-llvm/intrinsics/offset.rs b/tests/codegen-llvm/intrinsics/offset.rs
new file mode 100644
index 00000000000..cf0c7c7ac7d
--- /dev/null
+++ b/tests/codegen-llvm/intrinsics/offset.rs
@@ -0,0 +1,33 @@
+//@ compile-flags: -Copt-level=3 -C no-prepopulate-passes
+
+#![crate_type = "lib"]
+#![feature(core_intrinsics)]
+
+use std::intrinsics::offset;
+
+// CHECK-LABEL: ptr @offset_zst
+// CHECK-SAME: (ptr noundef %p, [[SIZE:i[0-9]+]] noundef %d)
+#[no_mangle]
+pub unsafe fn offset_zst(p: *const (), d: usize) -> *const () {
+    // CHECK-NOT: getelementptr
+    // CHECK: ret ptr %p
+    offset(p, d)
+}
+
+// CHECK-LABEL: ptr @offset_isize
+// CHECK-SAME: (ptr noundef %p, [[SIZE]] noundef %d)
+#[no_mangle]
+pub unsafe fn offset_isize(p: *const u32, d: isize) -> *const u32 {
+    // CHECK: %[[R:.*]] = getelementptr inbounds i32, ptr %p, [[SIZE]] %d
+    // CHECK-NEXT: ret ptr %[[R]]
+    offset(p, d)
+}
+
+// CHECK-LABEL: ptr @offset_usize
+// CHECK-SAME: (ptr noundef %p, [[SIZE]] noundef %d)
+#[no_mangle]
+pub unsafe fn offset_usize(p: *const u64, d: usize) -> *const u64 {
+    // CHECK: %[[R:.*]] = getelementptr inbounds{{( nuw)?}} i64, ptr %p, [[SIZE]] %d
+    // CHECK-NEXT: ret ptr %[[R]]
+    offset(p, d)
+}
diff --git a/tests/codegen-llvm/intrinsics/offset_from.rs b/tests/codegen-llvm/intrinsics/offset_from.rs
new file mode 100644
index 00000000000..ef1a77ef184
--- /dev/null
+++ b/tests/codegen-llvm/intrinsics/offset_from.rs
@@ -0,0 +1,36 @@
+//@ compile-flags: -C opt-level=1
+//@ only-64bit (because we're using [ui]size)
+
+#![crate_type = "lib"]
+#![feature(core_intrinsics)]
+
+//! Basic optimizations are enabled because otherwise `x86_64-gnu-nopt` had an alloca.
+//! Uses a type with non-power-of-two size to avoid normalizations to shifts.
+
+use std::intrinsics::*;
+
+type RGB = [u8; 3];
+
+// CHECK-LABEL: @offset_from_odd_size
+#[no_mangle]
+pub unsafe fn offset_from_odd_size(a: *const RGB, b: *const RGB) -> isize {
+    // CHECK: start
+    // CHECK-NEXT: ptrtoint
+    // CHECK-NEXT: ptrtoint
+    // CHECK-NEXT: sub i64
+    // CHECK-NEXT: sdiv exact i64 %{{[0-9]+}}, 3
+    // CHECK-NEXT: ret i64
+    ptr_offset_from(a, b)
+}
+
+// CHECK-LABEL: @offset_from_unsigned_odd_size
+#[no_mangle]
+pub unsafe fn offset_from_unsigned_odd_size(a: *const RGB, b: *const RGB) -> usize {
+    // CHECK: start
+    // CHECK-NEXT: ptrtoint
+    // CHECK-NEXT: ptrtoint
+    // CHECK-NEXT: sub nuw i64
+    // CHECK-NEXT: udiv exact i64 %{{[0-9]+}}, 3
+    // CHECK-NEXT: ret i64
+    ptr_offset_from_unsigned(a, b)
+}
diff --git a/tests/codegen-llvm/intrinsics/prefetch.rs b/tests/codegen-llvm/intrinsics/prefetch.rs
new file mode 100644
index 00000000000..3f9f21c85cb
--- /dev/null
+++ b/tests/codegen-llvm/intrinsics/prefetch.rs
@@ -0,0 +1,64 @@
+//@ compile-flags: -C no-prepopulate-passes
+
+#![crate_type = "lib"]
+#![feature(core_intrinsics)]
+
+use std::intrinsics::{
+    prefetch_read_data, prefetch_read_instruction, prefetch_write_data, prefetch_write_instruction,
+};
+
+#[no_mangle]
+pub fn check_prefetch_read_data(data: &[i8]) {
+    unsafe {
+        // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 0, i32 0, i32 1)
+        prefetch_read_data(data.as_ptr(), 0);
+        // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 0, i32 1, i32 1)
+        prefetch_read_data(data.as_ptr(), 1);
+        // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 0, i32 2, i32 1)
+        prefetch_read_data(data.as_ptr(), 2);
+        // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 0, i32 3, i32 1)
+        prefetch_read_data(data.as_ptr(), 3);
+    }
+}
+
+#[no_mangle]
+pub fn check_prefetch_write_data(data: &[i8]) {
+    unsafe {
+        // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 1, i32 0, i32 1)
+        prefetch_write_data(data.as_ptr(), 0);
+        // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 1, i32 1, i32 1)
+        prefetch_write_data(data.as_ptr(), 1);
+        // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 1, i32 2, i32 1)
+        prefetch_write_data(data.as_ptr(), 2);
+        // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 1, i32 3, i32 1)
+        prefetch_write_data(data.as_ptr(), 3);
+    }
+}
+
+#[no_mangle]
+pub fn check_prefetch_read_instruction(data: &[i8]) {
+    unsafe {
+        // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 0, i32 0, i32 0)
+        prefetch_read_instruction(data.as_ptr(), 0);
+        // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 0, i32 1, i32 0)
+        prefetch_read_instruction(data.as_ptr(), 1);
+        // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 0, i32 2, i32 0)
+        prefetch_read_instruction(data.as_ptr(), 2);
+        // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 0, i32 3, i32 0)
+        prefetch_read_instruction(data.as_ptr(), 3);
+    }
+}
+
+#[no_mangle]
+pub fn check_prefetch_write_instruction(data: &[i8]) {
+    unsafe {
+        // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 1, i32 0, i32 0)
+        prefetch_write_instruction(data.as_ptr(), 0);
+        // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 1, i32 1, i32 0)
+        prefetch_write_instruction(data.as_ptr(), 1);
+        // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 1, i32 2, i32 0)
+        prefetch_write_instruction(data.as_ptr(), 2);
+        // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 1, i32 3, i32 0)
+        prefetch_write_instruction(data.as_ptr(), 3);
+    }
+}
diff --git a/tests/codegen-llvm/intrinsics/ptr_metadata.rs b/tests/codegen-llvm/intrinsics/ptr_metadata.rs
new file mode 100644
index 00000000000..044dbc20486
--- /dev/null
+++ b/tests/codegen-llvm/intrinsics/ptr_metadata.rs
@@ -0,0 +1,36 @@
+//@ compile-flags: -Copt-level=3 -C no-prepopulate-passes -Z inline-mir
+//@ only-64bit (so I don't need to worry about usize)
+
+#![crate_type = "lib"]
+#![feature(core_intrinsics)]
+
+use std::intrinsics::ptr_metadata;
+
+// CHECK-LABEL: @thin_metadata(
+#[no_mangle]
+pub fn thin_metadata(p: *const ()) {
+    // CHECK: start
+    // CHECK-NEXT: ret void
+    ptr_metadata(p)
+}
+
+// CHECK-LABEL: @slice_metadata(
+#[no_mangle]
+pub fn slice_metadata(p: *const [u8]) -> usize {
+    // CHECK: start
+    // CHECK-NEXT: ret i64 %p.1
+    ptr_metadata(p)
+}
+
+// CHECK-LABEL: @dyn_byte_offset(
+#[no_mangle]
+pub unsafe fn dyn_byte_offset(
+    p: *const dyn std::fmt::Debug,
+    n: usize,
+) -> *const dyn std::fmt::Debug {
+    // CHECK: %[[Q:.+]] = getelementptr inbounds{{( nuw)?}} i8, ptr %p.0, i64 %n
+    // CHECK: %[[TEMP1:.+]] = insertvalue { ptr, ptr } poison, ptr %[[Q]], 0
+    // CHECK: %[[TEMP2:.+]] = insertvalue { ptr, ptr } %[[TEMP1]], ptr %p.1, 1
+    // CHECK: ret { ptr, ptr } %[[TEMP2]]
+    p.byte_add(n)
+}
diff --git a/tests/codegen-llvm/intrinsics/rotate_left.rs b/tests/codegen-llvm/intrinsics/rotate_left.rs
new file mode 100644
index 00000000000..4f6c5cbaed6
--- /dev/null
+++ b/tests/codegen-llvm/intrinsics/rotate_left.rs
@@ -0,0 +1,31 @@
+//@ compile-flags: -C no-prepopulate-passes
+
+#![crate_type = "lib"]
+#![feature(core_intrinsics)]
+
+use std::intrinsics::rotate_left;
+
+// CHECK-LABEL: @rotate_left_u16
+#[no_mangle]
+pub unsafe fn rotate_left_u16(x: u16, shift: u32) -> u16 {
+    // CHECK: %[[tmp:.*]] = trunc i32 %shift to i16
+    // CHECK: call i16 @llvm.fshl.i16(i16 %x, i16 %x, i16 %[[tmp]])
+    rotate_left(x, shift)
+}
+
+// CHECK-LABEL: @rotate_left_u32
+#[no_mangle]
+pub unsafe fn rotate_left_u32(x: u32, shift: u32) -> u32 {
+    // CHECK-NOT: trunc
+    // CHECK-NOT: zext
+    // CHECK: call i32 @llvm.fshl.i32(i32 %x, i32 %x, i32 %shift)
+    rotate_left(x, shift)
+}
+
+// CHECK-LABEL: @rotate_left_u64
+#[no_mangle]
+pub unsafe fn rotate_left_u64(x: u64, shift: u32) -> u64 {
+    // CHECK: %[[tmp:.*]] = zext i32 %shift to i64
+    // CHECK: call i64 @llvm.fshl.i64(i64 %x, i64 %x, i64 %[[tmp]])
+    rotate_left(x, shift)
+}
diff --git a/tests/codegen-llvm/intrinsics/rustc_intrinsic_must_be_overridden.rs b/tests/codegen-llvm/intrinsics/rustc_intrinsic_must_be_overridden.rs
new file mode 100644
index 00000000000..b41e441d309
--- /dev/null
+++ b/tests/codegen-llvm/intrinsics/rustc_intrinsic_must_be_overridden.rs
@@ -0,0 +1,14 @@
+//@ revisions: OPT0 OPT1
+//@ [OPT0] compile-flags: -Copt-level=0
+//@ [OPT1] compile-flags: -Copt-level=1
+//@ compile-flags: -Cno-prepopulate-passes
+
+#![crate_type = "lib"]
+#![feature(core_intrinsics)]
+
+// CHECK-NOT: core::intrinsics::size_of_val
+
+#[no_mangle]
+pub unsafe fn size_of_val(ptr: *const i32) -> usize {
+    core::intrinsics::size_of_val(ptr)
+}
diff --git a/tests/codegen-llvm/intrinsics/select_unpredictable.rs b/tests/codegen-llvm/intrinsics/select_unpredictable.rs
new file mode 100644
index 00000000000..ad7120c6fb8
--- /dev/null
+++ b/tests/codegen-llvm/intrinsics/select_unpredictable.rs
@@ -0,0 +1,71 @@
+//@ compile-flags: -Copt-level=3 -Zmerge-functions=disabled
+
+#![feature(core_intrinsics)]
+#![crate_type = "lib"]
+
+/* Test the intrinsic */
+
+#[no_mangle]
+pub fn test_int(p: bool, a: u64, b: u64) -> u64 {
+    // CHECK-LABEL: define{{.*}} @test_int
+    // CHECK: select i1 %p, i64 %a, i64 %b, !unpredictable
+    core::intrinsics::select_unpredictable(p, a, b)
+}
+
+#[no_mangle]
+pub fn test_pair(p: bool, a: (u64, u64), b: (u64, u64)) -> (u64, u64) {
+    // CHECK-LABEL: define{{.*}} @test_pair
+    // CHECK: select i1 %p, {{.*}}, !unpredictable
+    core::intrinsics::select_unpredictable(p, a, b)
+}
+
+struct Large {
+    e: [u64; 100],
+}
+
+#[no_mangle]
+pub fn test_struct(p: bool, a: Large, b: Large) -> Large {
+    // CHECK-LABEL: define{{.*}} @test_struct
+    // CHECK: select i1 %p, {{.*}}, !unpredictable
+    core::intrinsics::select_unpredictable(p, a, b)
+}
+
+// ZSTs should not need a `select` expression.
+#[no_mangle]
+pub fn test_zst(p: bool, a: (), b: ()) -> () {
+    // CHECK-LABEL: define{{.*}} @test_zst
+    // CHECK-NEXT: start:
+    // CHECK-NEXT: ret void
+    core::intrinsics::select_unpredictable(p, a, b)
+}
+
+/* Test the user-facing version */
+
+#[no_mangle]
+pub fn test_int2(p: bool, a: u64, b: u64) -> u64 {
+    // CHECK-LABEL: define{{.*}} @test_int2
+    // CHECK: select i1 %p, i64 %a, i64 %b, !unpredictable
+    core::hint::select_unpredictable(p, a, b)
+}
+
+#[no_mangle]
+pub fn test_pair2(p: bool, a: (u64, u64), b: (u64, u64)) -> (u64, u64) {
+    // CHECK-LABEL: define{{.*}} @test_pair2
+    // CHECK: select i1 %p, {{.*}}, !unpredictable
+    core::hint::select_unpredictable(p, a, b)
+}
+
+#[no_mangle]
+pub fn test_struct2(p: bool, a: Large, b: Large) -> Large {
+    // CHECK-LABEL: define{{.*}} @test_struct2
+    // CHECK: select i1 %p, {{.*}}, !unpredictable
+    core::hint::select_unpredictable(p, a, b)
+}
+
+#[no_mangle]
+pub fn test_zst2(p: bool, a: (), b: ()) -> () {
+    // CHECK-LABEL: define{{.*}} @test_zst2
+    // CHECK-NEXT: start:
+    // CHECK-NEXT: ret void
+    core::hint::select_unpredictable(p, a, b)
+}
diff --git a/tests/codegen-llvm/intrinsics/three_way_compare.rs b/tests/codegen-llvm/intrinsics/three_way_compare.rs
new file mode 100644
index 00000000000..95fcb636f7c
--- /dev/null
+++ b/tests/codegen-llvm/intrinsics/three_way_compare.rs
@@ -0,0 +1,28 @@
+//@ revisions: DEBUG OPTIM
+//@ [DEBUG] compile-flags: -C opt-level=0
+//@ [OPTIM] compile-flags: -C opt-level=3
+//@ compile-flags: -C no-prepopulate-passes
+//@ min-llvm-version: 20
+
+#![crate_type = "lib"]
+#![feature(core_intrinsics)]
+
+use std::intrinsics::three_way_compare;
+
+#[no_mangle]
+// CHECK-LABEL: @signed_cmp
+// CHECK-SAME: (i16{{.*}} %a, i16{{.*}} %b)
+pub fn signed_cmp(a: i16, b: i16) -> std::cmp::Ordering {
+    // CHECK: %[[CMP:.+]] = call i8 @llvm.scmp.i8.i16(i16 %a, i16 %b)
+    // CHECK-NEXT: ret i8 %[[CMP]]
+    three_way_compare(a, b)
+}
+
+#[no_mangle]
+// CHECK-LABEL: @unsigned_cmp
+// CHECK-SAME: (i16{{.*}} %a, i16{{.*}} %b)
+pub fn unsigned_cmp(a: u16, b: u16) -> std::cmp::Ordering {
+    // CHECK: %[[CMP:.+]] = call i8 @llvm.ucmp.i8.i16(i16 %a, i16 %b)
+    // CHECK-NEXT: ret i8 %[[CMP]]
+    three_way_compare(a, b)
+}
diff --git a/tests/codegen-llvm/intrinsics/transmute-niched.rs b/tests/codegen-llvm/intrinsics/transmute-niched.rs
new file mode 100644
index 00000000000..8ff5cc8ee4f
--- /dev/null
+++ b/tests/codegen-llvm/intrinsics/transmute-niched.rs
@@ -0,0 +1,223 @@
+//@ revisions: OPT DBG
+//@ [OPT] compile-flags: -C opt-level=3 -C no-prepopulate-passes
+//@ [DBG] compile-flags: -C opt-level=0 -C no-prepopulate-passes
+//@ only-64bit (so I don't need to worry about usize)
+#![crate_type = "lib"]
+
+use std::mem::transmute;
+use std::num::NonZero;
+use std::ptr::NonNull;
+
+#[repr(u8)]
+pub enum SmallEnum {
+    A = 10,
+    B = 11,
+    C = 12,
+}
+
+// CHECK-LABEL: @check_to_enum(
+#[no_mangle]
+pub unsafe fn check_to_enum(x: i8) -> SmallEnum {
+    // CHECK-NOT: icmp
+    // CHECK-NOT: assume
+    // OPT: %0 = sub i8 %x, 10
+    // OPT: %1 = icmp ule i8 %0, 2
+    // OPT: call void @llvm.assume(i1 %1)
+    // CHECK-NOT: icmp
+    // CHECK-NOT: assume
+    // CHECK: ret i8 %x
+
+    transmute(x)
+}
+
+// CHECK-LABEL: @check_from_enum(
+#[no_mangle]
+pub unsafe fn check_from_enum(x: SmallEnum) -> i8 {
+    // CHECK-NOT: icmp
+    // CHECK-NOT: assume
+    // OPT: %0 = sub i8 %x, 10
+    // OPT: %1 = icmp ule i8 %0, 2
+    // OPT: call void @llvm.assume(i1 %1)
+    // CHECK-NOT: icmp
+    // CHECK-NOT: assume
+    // CHECK: ret i8 %x
+
+    transmute(x)
+}
+
+// CHECK-LABEL: @check_to_ordering(
+#[no_mangle]
+pub unsafe fn check_to_ordering(x: u8) -> std::cmp::Ordering {
+    // CHECK-NOT: icmp
+    // CHECK-NOT: assume
+    // OPT: %0 = sub i8 %x, -1
+    // OPT: %1 = icmp ule i8 %0, 2
+    // OPT: call void @llvm.assume(i1 %1)
+    // CHECK-NOT: icmp
+    // CHECK-NOT: assume
+    // CHECK: ret i8 %x
+
+    transmute(x)
+}
+
+// CHECK-LABEL: @check_from_ordering(
+#[no_mangle]
+pub unsafe fn check_from_ordering(x: std::cmp::Ordering) -> u8 {
+    // CHECK-NOT: icmp
+    // CHECK-NOT: assume
+    // OPT: %0 = sub i8 %x, -1
+    // OPT: %1 = icmp ule i8 %0, 2
+    // OPT: call void @llvm.assume(i1 %1)
+    // CHECK-NOT: icmp
+    // CHECK-NOT: assume
+    // CHECK: ret i8 %x
+
+    transmute(x)
+}
+
+#[repr(i32)]
+pub enum Minus100ToPlus100 {
+    A = -100,
+    B = -90,
+    C = -80,
+    D = -70,
+    E = -60,
+    F = -50,
+    G = -40,
+    H = -30,
+    I = -20,
+    J = -10,
+    K = 0,
+    L = 10,
+    M = 20,
+    N = 30,
+    O = 40,
+    P = 50,
+    Q = 60,
+    R = 70,
+    S = 80,
+    T = 90,
+    U = 100,
+}
+
+// CHECK-LABEL: @check_enum_from_char(
+#[no_mangle]
+pub unsafe fn check_enum_from_char(x: char) -> Minus100ToPlus100 {
+    // CHECK-NOT: icmp
+    // CHECK-NOT: assume
+    // OPT: %0 = icmp ule i32 %x, 1114111
+    // OPT: call void @llvm.assume(i1 %0)
+    // OPT: %1 = sub i32 %x, -100
+    // OPT: %2 = icmp ule i32 %1, 200
+    // OPT: call void @llvm.assume(i1 %2)
+    // CHECK-NOT: icmp
+    // CHECK-NOT: assume
+    // CHECK: ret i32 %x
+
+    transmute(x)
+}
+
+// CHECK-LABEL: @check_enum_to_char(
+#[no_mangle]
+pub unsafe fn check_enum_to_char(x: Minus100ToPlus100) -> char {
+    // CHECK-NOT: icmp
+    // CHECK-NOT: assume
+    // OPT: %0 = sub i32 %x, -100
+    // OPT: %1 = icmp ule i32 %0, 200
+    // OPT: call void @llvm.assume(i1 %1)
+    // OPT: %2 = icmp ule i32 %x, 1114111
+    // OPT: call void @llvm.assume(i1 %2)
+    // CHECK-NOT: icmp
+    // CHECK-NOT: assume
+    // CHECK: ret i32 %x
+
+    transmute(x)
+}
+
+// CHECK-LABEL: @check_swap_pair(
+#[no_mangle]
+pub unsafe fn check_swap_pair(x: (char, NonZero<u32>)) -> (NonZero<u32>, char) {
+    // CHECK-NOT: icmp
+    // CHECK-NOT: assume
+    // OPT: %0 = icmp ule i32 %x.0, 1114111
+    // OPT: call void @llvm.assume(i1 %0)
+    // OPT: %1 = sub i32 %x.0, 1
+    // OPT: %2 = icmp ule i32 %1, -2
+    // OPT: call void @llvm.assume(i1 %2)
+    // OPT: %3 = sub i32 %x.1, 1
+    // OPT: %4 = icmp ule i32 %3, -2
+    // OPT: call void @llvm.assume(i1 %4)
+    // OPT: %5 = icmp ule i32 %x.1, 1114111
+    // OPT: call void @llvm.assume(i1 %5)
+    // CHECK-NOT: icmp
+    // CHECK-NOT: assume
+    // CHECK: %[[P1:.+]] = insertvalue { i32, i32 } poison, i32 %x.0, 0
+    // CHECK: %[[P2:.+]] = insertvalue { i32, i32 } %[[P1]], i32 %x.1, 1
+    // CHECK: ret { i32, i32 } %[[P2]]
+
+    transmute(x)
+}
+
+// CHECK-LABEL: @check_bool_from_ordering(
+#[no_mangle]
+pub unsafe fn check_bool_from_ordering(x: std::cmp::Ordering) -> bool {
+    // CHECK-NOT: icmp
+    // CHECK-NOT: assume
+    // OPT: %0 = sub i8 %x, -1
+    // OPT: %1 = icmp ule i8 %0, 2
+    // OPT: call void @llvm.assume(i1 %1)
+    // OPT: %2 = icmp ule i8 %x, 1
+    // OPT: call void @llvm.assume(i1 %2)
+    // CHECK-NOT: icmp
+    // CHECK-NOT: assume
+    // CHECK: %[[R:.+]] = trunc{{( nuw)?}} i8 %x to i1
+    // CHECK: ret i1 %[[R]]
+
+    transmute(x)
+}
+
+// CHECK-LABEL: @check_bool_to_ordering(
+#[no_mangle]
+pub unsafe fn check_bool_to_ordering(x: bool) -> std::cmp::Ordering {
+    // CHECK: %_0 = zext i1 %x to i8
+    // CHECK-NOT: icmp
+    // CHECK-NOT: assume
+    // OPT: %0 = icmp ule i8 %_0, 1
+    // OPT: call void @llvm.assume(i1 %0)
+    // OPT: %1 = sub i8 %_0, -1
+    // OPT: %2 = icmp ule i8 %1, 2
+    // OPT: call void @llvm.assume(i1 %2)
+    // CHECK-NOT: icmp
+    // CHECK-NOT: assume
+    // CHECK: ret i8 %_0
+
+    transmute(x)
+}
+
+// CHECK-LABEL: @check_nonnull_to_ptr(
+#[no_mangle]
+pub unsafe fn check_nonnull_to_ptr(x: NonNull<u8>) -> *const u8 {
+    // CHECK-NOT: icmp
+    // CHECK-NOT: assume
+    // OPT: %0 = icmp ne ptr %x, null
+    // OPT: call void @llvm.assume(i1 %0)
+    // CHECK-NOT: icmp
+    // CHECK-NOT: assume
+    // CHECK: ret ptr %x
+
+    transmute(x)
+}
+
+// CHECK-LABEL: @check_ptr_to_nonnull(
+#[no_mangle]
+pub unsafe fn check_ptr_to_nonnull(x: *const u8) -> NonNull<u8> {
+    // CHECK-NOT: icmp
+    // CHECK-NOT: assume
+    // OPT: %0 = icmp ne ptr %x, null
+    // OPT: call void @llvm.assume(i1 %0)
+    // CHECK-NOT: icmp
+    // CHECK-NOT: assume
+    // CHECK: ret ptr %x
+
+    transmute(x)
+}
diff --git a/tests/codegen-llvm/intrinsics/transmute-x64.rs b/tests/codegen-llvm/intrinsics/transmute-x64.rs
new file mode 100644
index 00000000000..8c9480ab091
--- /dev/null
+++ b/tests/codegen-llvm/intrinsics/transmute-x64.rs
@@ -0,0 +1,28 @@
+//@ compile-flags: -Copt-level=3 -C no-prepopulate-passes
+//@ only-x86_64 (it's using arch-specific types)
+
+#![crate_type = "lib"]
+
+use std::arch::x86_64::{__m128, __m128i, __m256i};
+use std::mem::transmute;
+
+// CHECK-LABEL: @check_sse_pair_to_avx(
+#[no_mangle]
+pub unsafe fn check_sse_pair_to_avx(x: (__m128i, __m128i)) -> __m256i {
+    // CHECK: start:
+    // CHECK-NOT: alloca
+    // CHECK-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr align 32 %_0, ptr align 16 %x, i64 32, i1 false)
+    // CHECK-NEXT: ret void
+    transmute(x)
+}
+
+// CHECK-LABEL: @check_sse_pair_from_avx(
+#[no_mangle]
+pub unsafe fn check_sse_pair_from_avx(x: __m256i) -> (__m128i, __m128i) {
+    // CHECK: start:
+    // CHECK-NOT: alloca
+    // CHECK-NEXT: %[[TEMP:.+]] = load <4 x i64>, ptr %x, align 32
+    // CHECK-NEXT: store <4 x i64> %[[TEMP]], ptr %_0, align 16
+    // CHECK-NEXT: ret void
+    transmute(x)
+}
diff --git a/tests/codegen-llvm/intrinsics/transmute.rs b/tests/codegen-llvm/intrinsics/transmute.rs
new file mode 100644
index 00000000000..c9a1cd58af3
--- /dev/null
+++ b/tests/codegen-llvm/intrinsics/transmute.rs
@@ -0,0 +1,497 @@
+//@ compile-flags: -Copt-level=3 -C no-prepopulate-passes
+//@ only-64bit (so I don't need to worry about usize)
+
+#![crate_type = "lib"]
+#![feature(core_intrinsics)]
+#![feature(custom_mir)]
+#![allow(unreachable_code)]
+
+// Some of these need custom MIR to not get removed by MIR optimizations.
+use std::intrinsics::mir::*;
+use std::intrinsics::{transmute, transmute_unchecked};
+use std::mem::MaybeUninit;
+use std::num::NonZero;
+
+pub enum ZstNever {}
+
+#[repr(align(2))]
+pub struct BigNever(ZstNever, u16, ZstNever);
+
+#[repr(align(8))]
+pub struct Scalar64(i64);
+
+#[repr(C, align(4))]
+pub struct Aggregate64(u16, u8, i8, f32);
+
+#[repr(C)]
+pub struct Aggregate8(u8);
+
+// CHECK-LABEL: @check_bigger_size(
+#[no_mangle]
+pub unsafe fn check_bigger_size(x: u16) -> u32 {
+    // CHECK: store i1 true, ptr poison, align 1
+    transmute_unchecked(x)
+}
+
+// CHECK-LABEL: @check_smaller_size(
+#[no_mangle]
+pub unsafe fn check_smaller_size(x: u32) -> u16 {
+    // CHECK: store i1 true, ptr poison, align 1
+    transmute_unchecked(x)
+}
+
+// CHECK-LABEL: @check_smaller_array(
+#[no_mangle]
+pub unsafe fn check_smaller_array(x: [u32; 7]) -> [u32; 3] {
+    // CHECK: store i1 true, ptr poison, align 1
+    transmute_unchecked(x)
+}
+
+// CHECK-LABEL: @check_bigger_array(
+#[no_mangle]
+pub unsafe fn check_bigger_array(x: [u32; 3]) -> [u32; 7] {
+    // CHECK: store i1 true, ptr poison, align 1
+    transmute_unchecked(x)
+}
+
+// CHECK-LABEL: @check_to_empty_array(
+#[no_mangle]
+#[custom_mir(dialect = "runtime", phase = "optimized")]
+pub unsafe fn check_to_empty_array(x: [u32; 5]) -> [u32; 0] {
+    // CHECK: start
+    // CHECK-NEXT: store i1 true, ptr poison, align 1
+    // CHECK-NEXT: ret void
+    mir! {
+        {
+            RET = CastTransmute(x);
+            Return()
+        }
+    }
+}
+
+// CHECK-LABEL: @check_from_empty_array(
+#[no_mangle]
+#[custom_mir(dialect = "runtime", phase = "optimized")]
+pub unsafe fn check_from_empty_array(x: [u32; 0]) -> [u32; 5] {
+    // CHECK: start
+    // CHECK-NEXT: store i1 true, ptr poison, align 1
+    // CHECK-NEXT: ret void
+    mir! {
+        {
+            RET = CastTransmute(x);
+            Return()
+        }
+    }
+}
+
+// CHECK-LABEL: @check_to_uninhabited(
+#[no_mangle]
+#[custom_mir(dialect = "runtime", phase = "optimized")]
+pub unsafe fn check_to_uninhabited(x: u16) {
+    // CHECK: start
+    // CHECK-NEXT: store i1 true, ptr poison, align 1
+    // CHECK-NEXT: ret void
+    mir! {
+        let temp: BigNever;
+        {
+            temp = CastTransmute(x);
+            Return()
+        }
+    }
+}
+
+// CHECK-LABEL: @check_from_uninhabited(
+#[no_mangle]
+#[custom_mir(dialect = "runtime", phase = "optimized")]
+pub unsafe fn check_from_uninhabited(x: BigNever) -> u16 {
+    // CHECK: start
+    // CHECK-NEXT: store i1 true, ptr poison, align 1
+    // CHECK-NEXT: ret i16 poison
+    mir! {
+        {
+            RET = CastTransmute(x);
+            Return()
+        }
+    }
+}
+
+// CHECK-LABEL: @check_intermediate_passthrough(
+#[no_mangle]
+pub unsafe fn check_intermediate_passthrough(x: u32) -> i32 {
+    // CHECK: start
+    // CHECK: %[[TMP:.+]] = add i32 1, %x
+    // CHECK: %[[RET:.+]] = add i32 %[[TMP]], 1
+    // CHECK: ret i32 %[[RET]]
+    unsafe { transmute::<u32, i32>(1 + x) + 1 }
+}
+
+// CHECK-LABEL: @check_nop_pair(
+#[no_mangle]
+pub unsafe fn check_nop_pair(x: (u8, i8)) -> (i8, u8) {
+    // CHECK-NOT: alloca
+    // CHECK: %0 = insertvalue { i8, i8 } poison, i8 %x.0, 0
+    // CHECK: %1 = insertvalue { i8, i8 } %0, i8 %x.1, 1
+    // CHECK: ret { i8, i8 } %1
+    unsafe { transmute(x) }
+}
+
+// CHECK-LABEL: @check_to_newtype(
+#[no_mangle]
+pub unsafe fn check_to_newtype(x: u64) -> Scalar64 {
+    // CHECK-NOT: alloca
+    // CHECK: ret i64 %x
+    transmute(x)
+}
+
+// CHECK-LABEL: @check_from_newtype(
+#[no_mangle]
+pub unsafe fn check_from_newtype(x: Scalar64) -> u64 {
+    // CHECK-NOT: alloca
+    // CHECK: ret i64 %x
+    transmute(x)
+}
+
+// CHECK-LABEL: @check_aggregate_to_bool(
+#[no_mangle]
+pub unsafe fn check_aggregate_to_bool(x: Aggregate8) -> bool {
+    // CHECK: %x = alloca [1 x i8], align 1
+    // CHECK: %[[BYTE:.+]] = load i8, ptr %x, align 1
+    // CHECK: %[[BOOL:.+]] = trunc nuw i8 %[[BYTE]] to i1
+    // CHECK: ret i1 %[[BOOL]]
+    transmute(x)
+}
+
+// CHECK-LABEL: @check_aggregate_from_bool(
+#[no_mangle]
+pub unsafe fn check_aggregate_from_bool(x: bool) -> Aggregate8 {
+    // CHECK: %_0 = alloca [1 x i8], align 1
+    // CHECK: %[[BYTE:.+]] = zext i1 %x to i8
+    // CHECK: store i8 %[[BYTE]], ptr %_0, align 1
+    transmute(x)
+}
+
+// CHECK-LABEL: @check_byte_to_bool(
+#[no_mangle]
+pub unsafe fn check_byte_to_bool(x: u8) -> bool {
+    // CHECK-NOT: alloca
+    // CHECK: %[[R:.+]] = trunc nuw i8 %x to i1
+    // CHECK: ret i1 %[[R]]
+    transmute(x)
+}
+
+// CHECK-LABEL: @check_byte_from_bool(
+#[no_mangle]
+pub unsafe fn check_byte_from_bool(x: bool) -> u8 {
+    // CHECK-NOT: alloca
+    // CHECK: %[[R:.+]] = zext i1 %x to i8
+    // CHECK: ret i8 %[[R:.+]]
+    transmute(x)
+}
+
+// CHECK-LABEL: @check_to_pair(
+#[no_mangle]
+pub unsafe fn check_to_pair(x: u64) -> Option<i32> {
+    // CHECK: %_0 = alloca [8 x i8], align 4
+    // CHECK: store i64 %x, ptr %_0, align 4
+    transmute(x)
+}
+
+// CHECK-LABEL: @check_from_pair(
+#[no_mangle]
+pub unsafe fn check_from_pair(x: Option<i32>) -> u64 {
+    // The two arguments are of types that are only 4-aligned, but they're
+    // immediates so we can write using the destination alloca's alignment.
+    const { assert!(std::mem::align_of::<Option<i32>>() == 4) };
+
+    // CHECK: %_0 = alloca [8 x i8], align 8
+    // CHECK: store i32 %x.0, ptr %_0, align 8
+    // CHECK: store i32 %x.1, ptr %0, align 4
+    // CHECK: %[[R:.+]] = load i64, ptr %_0, align 8
+    // CHECK: ret i64 %[[R]]
+    transmute(x)
+}
+
+// CHECK-LABEL: @check_to_float(
+#[no_mangle]
+pub unsafe fn check_to_float(x: u32) -> f32 {
+    // CHECK-NOT: alloca
+    // CHECK: %_0 = bitcast i32 %x to float
+    // CHECK: ret float %_0
+    transmute(x)
+}
+
+// CHECK-LABEL: @check_from_float(
+#[no_mangle]
+pub unsafe fn check_from_float(x: f32) -> u32 {
+    // CHECK-NOT: alloca
+    // CHECK: %_0 = bitcast float %x to i32
+    // CHECK: ret i32 %_0
+    transmute(x)
+}
+
+// CHECK-LABEL: @check_to_bytes(
+#[no_mangle]
+pub unsafe fn check_to_bytes(x: u32) -> [u8; 4] {
+    // CHECK: %_0 = alloca [4 x i8], align 1
+    // CHECK: store i32 %x, ptr %_0, align 1
+    transmute(x)
+}
+
+// CHECK-LABEL: @check_from_bytes(
+#[no_mangle]
+pub unsafe fn check_from_bytes(x: [u8; 4]) -> u32 {
+    // CHECK: %x = alloca [4 x i8], align 1
+    // CHECK: %[[VAL:.+]] = load i32, ptr %x, align 1
+    // CHECK: ret i32 %[[VAL]]
+    transmute(x)
+}
+
+// CHECK-LABEL: @check_to_aggregate(
+#[no_mangle]
+pub unsafe fn check_to_aggregate(x: u64) -> Aggregate64 {
+    // CHECK: %_0 = alloca [8 x i8], align 4
+    // CHECK: store i64 %x, ptr %_0, align 4
+    // CHECK: %0 = load i64, ptr %_0, align 4
+    // CHECK: ret i64 %0
+    transmute(x)
+}
+
+// CHECK-LABEL: @check_from_aggregate(
+#[no_mangle]
+pub unsafe fn check_from_aggregate(x: Aggregate64) -> u64 {
+    // CHECK: %x = alloca [8 x i8], align 4
+    // CHECK: %[[VAL:.+]] = load i64, ptr %x, align 4
+    // CHECK: ret i64 %[[VAL]]
+    transmute(x)
+}
+
+// CHECK-LABEL: @check_long_array_less_aligned(
+#[no_mangle]
+pub unsafe fn check_long_array_less_aligned(x: [u64; 100]) -> [u16; 400] {
+    // CHECK-NEXT: start
+    // CHECK-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr align 2 %_0, ptr align 8 %x, i64 800, i1 false)
+    // CHECK-NEXT: ret void
+    transmute(x)
+}
+
+// CHECK-LABEL: @check_long_array_more_aligned(
+#[no_mangle]
+pub unsafe fn check_long_array_more_aligned(x: [u8; 100]) -> [u32; 25] {
+    // CHECK-NEXT: start
+    // CHECK-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr align 4 %_0, ptr align 1 %x, i64 100, i1 false)
+    // CHECK-NEXT: ret void
+    transmute(x)
+}
+
+// CHECK-LABEL: @check_pair_with_bool(
+#[no_mangle]
+pub unsafe fn check_pair_with_bool(x: (u8, bool)) -> (bool, i8) {
+    // CHECK-NOT: alloca
+    // CHECK: trunc nuw i8 %x.0 to i1
+    // CHECK: zext i1 %x.1 to i8
+    transmute(x)
+}
+
+// CHECK-LABEL: @check_float_to_pointer(
+#[no_mangle]
+pub unsafe fn check_float_to_pointer(x: f64) -> *const () {
+    // CHECK-NOT: alloca
+    // CHECK: %0 = bitcast double %x to i64
+    // CHECK: %_0 = getelementptr i8, ptr null, i64 %0
+    // CHECK: ret ptr %_0
+    transmute(x)
+}
+
+// CHECK-LABEL: @check_float_from_pointer(
+#[no_mangle]
+pub unsafe fn check_float_from_pointer(x: *const ()) -> f64 {
+    // CHECK-NOT: alloca
+    // CHECK: %0 = ptrtoint ptr %x to i64
+    // CHECK: %_0 = bitcast i64 %0 to double
+    // CHECK: ret double %_0
+    transmute(x)
+}
+
+// CHECK-LABEL: @check_array_to_pair(
+#[no_mangle]
+pub unsafe fn check_array_to_pair(x: [u8; 16]) -> (i64, u64) {
+    // CHECK-NOT: alloca
+    // CHECK: %[[FST:.+]] = load i64, ptr %{{.+}}, align 1, !noundef !
+    // CHECK: %[[SND:.+]] = load i64, ptr %{{.+}}, align 1, !noundef !
+    // CHECK: %[[PAIR0:.+]] = insertvalue { i64, i64 } poison, i64 %[[FST]], 0
+    // CHECK: %[[PAIR01:.+]] = insertvalue { i64, i64 } %[[PAIR0]], i64 %[[SND]], 1
+    // CHECK: ret { i64, i64 } %[[PAIR01]]
+    transmute(x)
+}
+
+// CHECK-LABEL: @check_pair_to_array(
+#[no_mangle]
+pub unsafe fn check_pair_to_array(x: (i64, u64)) -> [u8; 16] {
+    // CHECK-NOT: alloca
+    // CHECK: store i64 %x.0, ptr %{{.+}}, align 1
+    // CHECK: store i64 %x.1, ptr %{{.+}}, align 1
+    transmute(x)
+}
+
+// CHECK-LABEL: @check_heterogeneous_integer_pair(
+#[no_mangle]
+pub unsafe fn check_heterogeneous_integer_pair(x: (i32, bool)) -> (bool, u32) {
+    // CHECK: store i32 %x.0
+    // CHECK: %[[WIDER:.+]] = zext i1 %x.1 to i8
+    // CHECK: store i8 %[[WIDER]]
+
+    // CHECK: %[[BYTE:.+]] = load i8
+    // CHECK: trunc nuw i8 %[[BYTE:.+]] to i1
+    // CHECK: load i32
+    transmute(x)
+}
+
+// CHECK-LABEL: @check_heterogeneous_float_pair(
+#[no_mangle]
+pub unsafe fn check_heterogeneous_float_pair(x: (f64, f32)) -> (f32, f64) {
+    // CHECK: store double %x.0
+    // CHECK: store float %x.1
+    // CHECK: %[[A:.+]] = load float
+    // CHECK: %[[B:.+]] = load double
+    // CHECK: %[[P:.+]] = insertvalue { float, double } poison, float %[[A]], 0
+    // CHECK: insertvalue { float, double } %[[P]], double %[[B]], 1
+    transmute(x)
+}
+
+// CHECK-LABEL: @check_issue_110005(
+#[no_mangle]
+pub unsafe fn check_issue_110005(x: (usize, bool)) -> Option<Box<[u8]>> {
+    // CHECK: store i64 %x.0
+    // CHECK: %[[WIDER:.+]] = zext i1 %x.1 to i8
+    // CHECK: store i8 %[[WIDER]]
+    // CHECK: load ptr
+    // CHECK: load i64
+    transmute(x)
+}
+
+// CHECK-LABEL: @check_pair_to_dst_ref(
+#[no_mangle]
+pub unsafe fn check_pair_to_dst_ref<'a>(x: (usize, usize)) -> &'a [u8] {
+    // CHECK: %_0.0 = getelementptr i8, ptr null, i64 %x.0
+    // CHECK: %0 = icmp ne ptr %_0.0, null
+    // CHECK: call void @llvm.assume(i1 %0)
+    // CHECK: %1 = insertvalue { ptr, i64 } poison, ptr %_0.0, 0
+    // CHECK: %2 = insertvalue { ptr, i64 } %1, i64 %x.1, 1
+    // CHECK: ret { ptr, i64 } %2
+    transmute(x)
+}
+
+// CHECK-LABEL: @check_issue_109992(
+#[no_mangle]
+#[custom_mir(dialect = "runtime", phase = "optimized")]
+pub unsafe fn check_issue_109992(x: ()) -> [(); 1] {
+    // This uses custom MIR to avoid MIR optimizations having removed ZST ops.
+
+    // CHECK: start
+    // CHECK-NEXT: ret void
+    mir! {
+        {
+            RET = CastTransmute(x);
+            Return()
+        }
+    }
+}
+
+// CHECK-LABEL: @check_unit_to_never(
+#[no_mangle]
+#[custom_mir(dialect = "runtime", phase = "optimized")]
+pub unsafe fn check_unit_to_never(x: ()) {
+    // This uses custom MIR to avoid MIR optimizations having removed ZST ops.
+
+    // CHECK: start
+    // CHECK-NEXT: store i1 true, ptr poison, align 1
+    // CHECK-NEXT: ret void
+    mir! {
+        let temp: ZstNever;
+        {
+            temp = CastTransmute(x);
+            Return()
+        }
+    }
+}
+
+// CHECK-LABEL: @check_unit_from_never(
+#[no_mangle]
+#[custom_mir(dialect = "runtime", phase = "optimized")]
+pub unsafe fn check_unit_from_never(x: ZstNever) -> () {
+    // This uses custom MIR to avoid MIR optimizations having removed ZST ops.
+
+    // CHECK: start
+    // CHECK-NEXT: store i1 true, ptr poison, align 1
+    // CHECK-NEXT: ret void
+    mir! {
+        {
+            RET = CastTransmute(x);
+            Return()
+        }
+    }
+}
+
+// CHECK-LABEL: @check_maybe_uninit_pair(i16 %x.0, i64 %x.1)
+#[no_mangle]
+pub unsafe fn check_maybe_uninit_pair(
+    x: (MaybeUninit<u16>, MaybeUninit<u64>),
+) -> (MaybeUninit<i64>, MaybeUninit<i16>) {
+    // Thanks to `MaybeUninit` this is actually defined behaviour,
+    // unlike the examples above with pairs of primitives.
+
+    // CHECK: store i16 %x.0
+    // CHECK: store i64 %x.1
+    // CHECK: load i64
+    // CHECK-NOT: noundef
+    // CHECK: load i16
+    // CHECK-NOT: noundef
+    // CHECK: ret { i64, i16 }
+    transmute(x)
+}
+
+#[repr(align(8))]
+pub struct HighAlignScalar(u8);
+
+// CHECK-LABEL: @check_to_overalign(
+#[no_mangle]
+pub unsafe fn check_to_overalign(x: u64) -> HighAlignScalar {
+    // CHECK: %_0 = alloca [8 x i8], align 8
+    // CHECK: store i64 %x, ptr %_0, align 8
+    // CHECK: %0 = load i64, ptr %_0, align 8
+    // CHECK: ret i64 %0
+    transmute(x)
+}
+
+// CHECK-LABEL: @check_from_overalign(
+#[no_mangle]
+pub unsafe fn check_from_overalign(x: HighAlignScalar) -> u64 {
+    // CHECK: %x = alloca [8 x i8], align 8
+    // CHECK: %[[VAL:.+]] = load i64, ptr %x, align 8
+    // CHECK: ret i64 %[[VAL]]
+    transmute(x)
+}
+
+#[repr(transparent)]
+struct Level1(std::num::NonZero<u32>);
+#[repr(transparent)]
+struct Level2(Level1);
+#[repr(transparent)]
+struct Level3(Level2);
+
+// CHECK-LABEL: @repeatedly_transparent_transmute
+// CHECK-SAME: (i32{{.+}}%[[ARG:[^)]+]])
+#[no_mangle]
+#[custom_mir(dialect = "runtime", phase = "optimized")]
+pub unsafe fn repeatedly_transparent_transmute(x: NonZero<u32>) -> Level3 {
+    // CHECK: start
+    // CHECK-NEXT: ret i32 %[[ARG]]
+    mir! {
+        {
+            let A = CastTransmute::<NonZero<u32>, Level1>(x);
+            let B = CastTransmute::<Level1, Level2>(A);
+            RET = CastTransmute::<Level2, Level3>(B);
+            Return()
+        }
+    }
+}
diff --git a/tests/codegen-llvm/intrinsics/typed_swap.rs b/tests/codegen-llvm/intrinsics/typed_swap.rs
new file mode 100644
index 00000000000..6b55078407a
--- /dev/null
+++ b/tests/codegen-llvm/intrinsics/typed_swap.rs
@@ -0,0 +1,77 @@
+//@ revisions: OPT0 OPT3
+//@ [OPT0] compile-flags: -Copt-level=0
+//@ [OPT3] compile-flags: -Copt-level=3
+//@ compile-flags: -C no-prepopulate-passes
+//@ only-64bit (so I don't need to worry about usize)
+// ignore-tidy-linelength (the memcpy calls get long)
+
+#![crate_type = "lib"]
+#![feature(core_intrinsics)]
+
+use std::intrinsics::typed_swap_nonoverlapping;
+
+// CHECK-LABEL: @swap_unit(
+#[no_mangle]
+pub unsafe fn swap_unit(x: &mut (), y: &mut ()) {
+    // CHECK: start
+    // CHECK-NEXT: ret void
+    typed_swap_nonoverlapping(x, y)
+}
+
+// CHECK-LABEL: @swap_i32(
+#[no_mangle]
+pub unsafe fn swap_i32(x: &mut i32, y: &mut i32) {
+    // CHECK-NOT: alloca
+
+    // CHECK: %[[TEMP:.+]] = load i32, ptr %x, align 4
+    // OPT3-SAME: !noundef
+    // OPT0: %[[TEMP2:.+]] = load i32, ptr %y, align 4
+    // OPT0: store i32 %[[TEMP2]], ptr %x, align 4
+    // OPT0-NOT: memcpy
+    // OPT3-NOT: load
+    // OPT3: call void @llvm.memcpy.p0.p0.i64(ptr align 4 %x, ptr align 4 %y, i64 4, i1 false)
+    // CHECK: store i32 %[[TEMP]], ptr %y, align 4
+    // CHECK: ret void
+    typed_swap_nonoverlapping(x, y)
+}
+
+// CHECK-LABEL: @swap_pair(
+#[no_mangle]
+pub unsafe fn swap_pair(x: &mut (i32, u32), y: &mut (i32, u32)) {
+    // CHECK-NOT: alloca
+
+    // CHECK: load i32
+    // OPT3-SAME: !noundef
+    // CHECK: load i32
+    // OPT3-SAME: !noundef
+    // CHECK: call void @llvm.memcpy.p0.p0.i64(ptr align 4 %x, ptr align 4 %y, i64 8, i1 false)
+    // CHECK: store i32
+    // CHECK: store i32
+    typed_swap_nonoverlapping(x, y)
+}
+
+// CHECK-LABEL: @swap_str(
+#[no_mangle]
+pub unsafe fn swap_str<'a>(x: &mut &'a str, y: &mut &'a str) {
+    // CHECK-NOT: alloca
+
+    // CHECK: load ptr
+    // OPT3-SAME: !nonnull
+    // OPT3-SAME: !noundef
+    // CHECK: load i64
+    // OPT3-SAME: !noundef
+    // CHECK: call void @llvm.memcpy.p0.p0.i64(ptr align 8 %x, ptr align 8 %y, i64 16, i1 false)
+    // CHECK: store ptr
+    // CHECK: store i64
+    typed_swap_nonoverlapping(x, y)
+}
+
+// OPT0-LABEL: @swap_string(
+#[no_mangle]
+pub unsafe fn swap_string(x: &mut String, y: &mut String) {
+    // OPT0: %[[TEMP:.+]] = alloca {{.+}}, align 8
+    // OPT0: call void @llvm.memcpy.p0.p0.i64(ptr align 8 %[[TEMP]], ptr align 8 %x, i64 24, i1 false)
+    // OPT0: call void @llvm.memcpy.p0.p0.i64(ptr align 8 %x, ptr align 8 %y, i64 24, i1 false)
+    // OPT0: call void @llvm.memcpy.p0.p0.i64(ptr align 8 %y, ptr align 8 %[[TEMP]], i64 24, i1 false)
+    typed_swap_nonoverlapping(x, y)
+}
diff --git a/tests/codegen-llvm/intrinsics/unchecked_math.rs b/tests/codegen-llvm/intrinsics/unchecked_math.rs
new file mode 100644
index 00000000000..419c120ede9
--- /dev/null
+++ b/tests/codegen-llvm/intrinsics/unchecked_math.rs
@@ -0,0 +1,46 @@
+#![crate_type = "lib"]
+#![feature(core_intrinsics)]
+
+use std::intrinsics::*;
+
+// CHECK-LABEL: @unchecked_add_signed
+#[no_mangle]
+pub unsafe fn unchecked_add_signed(a: i32, b: i32) -> i32 {
+    // CHECK: add nsw
+    unchecked_add(a, b)
+}
+
+// CHECK-LABEL: @unchecked_add_unsigned
+#[no_mangle]
+pub unsafe fn unchecked_add_unsigned(a: u32, b: u32) -> u32 {
+    // CHECK: add nuw
+    unchecked_add(a, b)
+}
+
+// CHECK-LABEL: @unchecked_sub_signed
+#[no_mangle]
+pub unsafe fn unchecked_sub_signed(a: i32, b: i32) -> i32 {
+    // CHECK: sub nsw
+    unchecked_sub(a, b)
+}
+
+// CHECK-LABEL: @unchecked_sub_unsigned
+#[no_mangle]
+pub unsafe fn unchecked_sub_unsigned(a: u32, b: u32) -> u32 {
+    // CHECK: sub nuw
+    unchecked_sub(a, b)
+}
+
+// CHECK-LABEL: @unchecked_mul_signed
+#[no_mangle]
+pub unsafe fn unchecked_mul_signed(a: i32, b: i32) -> i32 {
+    // CHECK: mul nsw
+    unchecked_mul(a, b)
+}
+
+// CHECK-LABEL: @unchecked_mul_unsigned
+#[no_mangle]
+pub unsafe fn unchecked_mul_unsigned(a: u32, b: u32) -> u32 {
+    // CHECK: mul nuw
+    unchecked_mul(a, b)
+}
diff --git a/tests/codegen-llvm/intrinsics/unlikely.rs b/tests/codegen-llvm/intrinsics/unlikely.rs
new file mode 100644
index 00000000000..90ebf070d27
--- /dev/null
+++ b/tests/codegen-llvm/intrinsics/unlikely.rs
@@ -0,0 +1,35 @@
+//@ compile-flags: -Copt-level=3
+#![crate_type = "lib"]
+#![feature(core_intrinsics)]
+
+use std::intrinsics::unlikely;
+
+#[inline(never)]
+#[no_mangle]
+pub fn path_a() {
+    println!("path a");
+}
+
+#[inline(never)]
+#[no_mangle]
+pub fn path_b() {
+    println!("path b");
+}
+
+#[no_mangle]
+pub fn test_unlikely(x: bool) {
+    if unlikely(x) {
+        path_a();
+    } else {
+        path_b();
+    }
+}
+
+// CHECK-LABEL: @test_unlikely(
+// CHECK: br i1 %x, label %bb2, label %bb4, !prof ![[NUM:[0-9]+]]
+// CHECK: bb4:
+// CHECK: path_b
+// CHECK: bb2:
+// CHECK-NOT: cold_path
+// CHECK: path_a
+// CHECK: ![[NUM]] = !{!"branch_weights", {{(!"expected", )?}}i32 1, i32 2000}
diff --git a/tests/codegen-llvm/intrinsics/volatile.rs b/tests/codegen-llvm/intrinsics/volatile.rs
new file mode 100644
index 00000000000..2dea5ecb2ca
--- /dev/null
+++ b/tests/codegen-llvm/intrinsics/volatile.rs
@@ -0,0 +1,55 @@
+//@ compile-flags: -C no-prepopulate-passes
+
+#![crate_type = "lib"]
+#![feature(core_intrinsics)]
+
+use std::intrinsics;
+
+// CHECK-LABEL: @volatile_copy_memory
+#[no_mangle]
+pub unsafe fn volatile_copy_memory(a: *mut u8, b: *const u8) {
+    // CHECK: llvm.memmove.{{\w*(.*true)}}
+    intrinsics::volatile_copy_memory(a, b, 1)
+}
+
+// CHECK-LABEL: @volatile_copy_nonoverlapping_memory
+#[no_mangle]
+pub unsafe fn volatile_copy_nonoverlapping_memory(a: *mut u8, b: *const u8) {
+    // CHECK: llvm.memcpy.{{\w*(.*true)}}
+    intrinsics::volatile_copy_nonoverlapping_memory(a, b, 1)
+}
+
+// CHECK-LABEL: @volatile_set_memory
+#[no_mangle]
+pub unsafe fn volatile_set_memory(a: *mut u8, b: u8) {
+    // CHECK: llvm.memset.{{\w*(.*true)}}
+    intrinsics::volatile_set_memory(a, b, 1)
+}
+
+// CHECK-LABEL: @volatile_load
+#[no_mangle]
+pub unsafe fn volatile_load(a: *const u8) -> u8 {
+    // CHECK: load volatile
+    intrinsics::volatile_load(a)
+}
+
+// CHECK-LABEL: @volatile_store
+#[no_mangle]
+pub unsafe fn volatile_store(a: *mut u8, b: u8) {
+    // CHECK: store volatile
+    intrinsics::volatile_store(a, b)
+}
+
+// CHECK-LABEL: @unaligned_volatile_load
+#[no_mangle]
+pub unsafe fn unaligned_volatile_load(a: *const u8) -> u8 {
+    // CHECK: load volatile
+    intrinsics::unaligned_volatile_load(a)
+}
+
+// CHECK-LABEL: @unaligned_volatile_store
+#[no_mangle]
+pub unsafe fn unaligned_volatile_store(a: *mut u8, b: u8) {
+    // CHECK: store volatile
+    intrinsics::unaligned_volatile_store(a, b)
+}
diff --git a/tests/codegen-llvm/intrinsics/volatile_order.rs b/tests/codegen-llvm/intrinsics/volatile_order.rs
new file mode 100644
index 00000000000..99469831a6c
--- /dev/null
+++ b/tests/codegen-llvm/intrinsics/volatile_order.rs
@@ -0,0 +1,18 @@
+#![crate_type = "lib"]
+#![feature(core_intrinsics)]
+
+use std::intrinsics::*;
+
+pub unsafe fn test_volatile_order() {
+    let mut a: Box<u8> = Box::new(0);
+    // CHECK: load volatile
+    let x = volatile_load(&*a);
+    // CHECK: load volatile
+    let x = volatile_load(&*a);
+    // CHECK: store volatile
+    volatile_store(&mut *a, 12);
+    // CHECK: store volatile
+    unaligned_volatile_store(&mut *a, 12);
+    // CHECK: llvm.memset.p0
+    volatile_set_memory(&mut *a, 12, 1)
+}