Insert checks for enum discriminants when debug assertions are enabled

Similar to the existing nullpointer and alignment checks, this checks
for valid enum discriminants on creation of enums through unsafe
transmutes. Essentially this sanitizes patterns like the following:
```rust
let val: MyEnum = unsafe { std::mem::transmute<u32, MyEnum>(42) };
```
An extension of this check will be done in a follow-up that explicitly
sanitizes for extern enum values that come into Rust from e.g. C/C++.

This check is similar to Miri's capabilities of checking for valid
construction of enum values.

This PR is inspired by saethlin@'s PR
https://github.com/rust-lang/rust/pull/104862. Thank you so much for
keeping this code up and the detailed comments!

I also pair-programmed large parts of this together with vabr-g@.

20 files changed, 339 insertions, 0 deletions

diff --git a/tests/ui/mir/enum/convert_non_enum_break.rs b/tests/ui/mir/enum/convert_non_enum_break.rs
new file mode 100644
index 00000000000..de062c39907
--- /dev/null
+++ b/tests/ui/mir/enum/convert_non_enum_break.rs
@@ -0,0 +1,20 @@
+//@ run-fail
+//@ compile-flags: -C debug-assertions
+//@ error-pattern: trying to construct an enum from an invalid value 0x10000
+
+#[allow(dead_code)]
+#[repr(u32)]
+enum Foo {
+    A,
+    B,
+}
+
+#[allow(dead_code)]
+struct Bar {
+    a: u16,
+    b: u16,
+}
+
+fn main() {
+    let _val: Foo = unsafe { std::mem::transmute::<_, Foo>(Bar { a: 0, b: 1 }) };
+}
diff --git a/tests/ui/mir/enum/convert_non_enum_niche_break.rs b/tests/ui/mir/enum/convert_non_enum_niche_break.rs
new file mode 100644
index 00000000000..9ff4849c5b1
--- /dev/null
+++ b/tests/ui/mir/enum/convert_non_enum_niche_break.rs
@@ -0,0 +1,27 @@
+//@ run-fail
+//@ compile-flags: -C debug-assertions
+//@ error-pattern: trying to construct an enum from an invalid value 0x5
+
+#[allow(dead_code)]
+#[repr(u16)]
+enum Mix {
+    A,
+    B(u16),
+}
+
+#[allow(dead_code)]
+enum Nested {
+    C(Mix),
+    D,
+    E,
+}
+
+#[allow(dead_code)]
+struct Bar {
+    a: u16,
+    b: u16,
+}
+
+fn main() {
+    let _val: Nested = unsafe { std::mem::transmute::<_, Nested>(Bar { a: 5, b: 0 }) };
+}
diff --git a/tests/ui/mir/enum/convert_non_enum_niche_ok.rs b/tests/ui/mir/enum/convert_non_enum_niche_ok.rs
new file mode 100644
index 00000000000..24027da5458
--- /dev/null
+++ b/tests/ui/mir/enum/convert_non_enum_niche_ok.rs
@@ -0,0 +1,29 @@
+//@ run-pass
+//@ compile-flags: -C debug-assertions
+
+#[allow(dead_code)]
+#[repr(u16)]
+enum Mix {
+    A,
+    B(u16),
+}
+
+#[allow(dead_code)]
+enum Nested {
+    C(Mix),
+    D,
+    E,
+}
+
+#[allow(dead_code)]
+struct Bar {
+    a: u16,
+    b: u16,
+}
+
+fn main() {
+    let _val: Nested = unsafe { std::mem::transmute::<_, Nested>(Bar { a: 0, b: 0 }) };
+    let _val: Nested = unsafe { std::mem::transmute::<_, Nested>(Bar { a: 1, b: 0 }) };
+    let _val: Nested = unsafe { std::mem::transmute::<_, Nested>(Bar { a: 2, b: 0 }) };
+    let _val: Nested = unsafe { std::mem::transmute::<_, Nested>(Bar { a: 3, b: 0 }) };
+}
diff --git a/tests/ui/mir/enum/convert_non_enum_ok.rs b/tests/ui/mir/enum/convert_non_enum_ok.rs
new file mode 100644
index 00000000000..37fc64342ca
--- /dev/null
+++ b/tests/ui/mir/enum/convert_non_enum_ok.rs
@@ -0,0 +1,20 @@
+//@ run-pass
+//@ compile-flags: -C debug-assertions
+
+#[allow(dead_code)]
+#[repr(u32)]
+enum Foo {
+    A,
+    B,
+}
+
+#[allow(dead_code)]
+struct Bar {
+    a: u16,
+    b: u16,
+}
+
+fn main() {
+    let _val: Foo = unsafe { std::mem::transmute::<_, Foo>(Bar { a: 0, b: 0 }) };
+    let _val: Foo = unsafe { std::mem::transmute::<_, Foo>(Bar { a: 1, b: 0 }) };
+}
diff --git a/tests/ui/mir/enum/niche_option_tuple_break.rs b/tests/ui/mir/enum/niche_option_tuple_break.rs
new file mode 100644
index 00000000000..43eef3a4cc5
--- /dev/null
+++ b/tests/ui/mir/enum/niche_option_tuple_break.rs
@@ -0,0 +1,20 @@
+//@ run-fail
+//@ compile-flags: -C debug-assertions
+//@ error-pattern: trying to construct an enum from an invalid value 0x3
+
+#[allow(dead_code)]
+enum Foo {
+    A,
+    B,
+}
+
+#[allow(dead_code)]
+struct Bar {
+    a: usize,
+    b: usize,
+}
+
+fn main() {
+    let _val: Option<(usize, Foo)> =
+        unsafe { std::mem::transmute::<_, Option<(usize, Foo)>>(Bar { a: 3, b: 3 }) };
+}
diff --git a/tests/ui/mir/enum/niche_option_tuple_ok.rs b/tests/ui/mir/enum/niche_option_tuple_ok.rs
new file mode 100644
index 00000000000..71c885c7edb
--- /dev/null
+++ b/tests/ui/mir/enum/niche_option_tuple_ok.rs
@@ -0,0 +1,21 @@
+//@ run-pass
+//@ compile-flags: -C debug-assertions
+
+#[allow(dead_code)]
+enum Foo {
+    A,
+    B,
+}
+
+#[allow(dead_code)]
+struct Bar {
+    a: usize,
+    b: usize,
+}
+
+fn main() {
+    let _val: Option<(usize, Foo)> =
+        unsafe { std::mem::transmute::<_, Option<(usize, Foo)>>(Bar { a: 0, b: 0 }) };
+    let _val: Option<(usize, Foo)> =
+        unsafe { std::mem::transmute::<_, Option<(usize, Foo)>>(Bar { a: 1, b: 0 }) };
+}
diff --git a/tests/ui/mir/enum/numbered_variants_break.rs b/tests/ui/mir/enum/numbered_variants_break.rs
new file mode 100644
index 00000000000..e3e71dc8aec
--- /dev/null
+++ b/tests/ui/mir/enum/numbered_variants_break.rs
@@ -0,0 +1,13 @@
+//@ run-fail
+//@ compile-flags: -C debug-assertions
+//@ error-pattern: trying to construct an enum from an invalid value 0x3
+
+#[allow(dead_code)]
+enum Foo {
+    A,
+    B,
+}
+
+fn main() {
+    let _val: Foo = unsafe { std::mem::transmute::<u8, Foo>(3) };
+}
diff --git a/tests/ui/mir/enum/numbered_variants_ok.rs b/tests/ui/mir/enum/numbered_variants_ok.rs
new file mode 100644
index 00000000000..995a2f6511b
--- /dev/null
+++ b/tests/ui/mir/enum/numbered_variants_ok.rs
@@ -0,0 +1,13 @@
+//@ run-pass
+//@ compile-flags: -C debug-assertions
+
+#[allow(dead_code)]
+enum Foo {
+    A,
+    B,
+}
+
+fn main() {
+    let _val: Foo = unsafe { std::mem::transmute::<u8, Foo>(0) };
+    let _val: Foo = unsafe { std::mem::transmute::<u8, Foo>(1) };
+}
diff --git a/tests/ui/mir/enum/option_with_bigger_niche_break.rs b/tests/ui/mir/enum/option_with_bigger_niche_break.rs
new file mode 100644
index 00000000000..c66614b845b
--- /dev/null
+++ b/tests/ui/mir/enum/option_with_bigger_niche_break.rs
@@ -0,0 +1,14 @@
+//@ run-fail
+//@ compile-flags: -C debug-assertions
+//@ error-pattern: trying to construct an enum from an invalid value 0x0
+
+#[repr(u32)]
+#[allow(dead_code)]
+enum Foo {
+    A = 2,
+    B,
+}
+
+fn main() {
+    let _val: Option<Foo> = unsafe { std::mem::transmute::<u32, Option<Foo>>(0) };
+}
diff --git a/tests/ui/mir/enum/option_with_bigger_niche_ok.rs b/tests/ui/mir/enum/option_with_bigger_niche_ok.rs
new file mode 100644
index 00000000000..1d44ffd28fc
--- /dev/null
+++ b/tests/ui/mir/enum/option_with_bigger_niche_ok.rs
@@ -0,0 +1,14 @@
+//@ run-pass
+//@ compile-flags: -C debug-assertions
+
+#[repr(u32)]
+#[allow(dead_code)]
+enum Foo {
+    A = 2,
+    B,
+}
+
+fn main() {
+    let _val: Option<Foo> = unsafe { std::mem::transmute::<u32, Option<Foo>>(2) };
+    let _val: Option<Foo> = unsafe { std::mem::transmute::<u32, Option<Foo>>(3) };
+}
diff --git a/tests/ui/mir/enum/plain_no_data_break.rs b/tests/ui/mir/enum/plain_no_data_break.rs
new file mode 100644
index 00000000000..db68e752479
--- /dev/null
+++ b/tests/ui/mir/enum/plain_no_data_break.rs
@@ -0,0 +1,14 @@
+//@ run-fail
+//@ compile-flags: -C debug-assertions
+//@ error-pattern: trying to construct an enum from an invalid value 0x1
+
+#[repr(u32)]
+#[allow(dead_code)]
+enum Foo {
+    A = 2,
+    B,
+}
+
+fn main() {
+    let _val: Foo = unsafe { std::mem::transmute::<u32, Foo>(1) };
+}
diff --git a/tests/ui/mir/enum/plain_no_data_ok.rs b/tests/ui/mir/enum/plain_no_data_ok.rs
new file mode 100644
index 00000000000..bbdc18f96dc
--- /dev/null
+++ b/tests/ui/mir/enum/plain_no_data_ok.rs
@@ -0,0 +1,14 @@
+//@ run-pass
+//@ compile-flags: -C debug-assertions
+
+#[repr(u32)]
+#[allow(dead_code)]
+enum Foo {
+    A = 2,
+    B,
+}
+
+fn main() {
+    let _val: Foo = unsafe { std::mem::transmute::<u32, Foo>(2) };
+    let _val: Foo = unsafe { std::mem::transmute::<u32, Foo>(3) };
+}
diff --git a/tests/ui/mir/enum/single_ok.rs b/tests/ui/mir/enum/single_ok.rs
new file mode 100644
index 00000000000..06b5a237c68
--- /dev/null
+++ b/tests/ui/mir/enum/single_ok.rs
@@ -0,0 +1,11 @@
+//@ run-pass
+//@ compile-flags: -C debug-assertions
+
+#[allow(dead_code)]
+enum Single {
+    A
+}
+
+fn main() {
+    let _val: Single = unsafe { std::mem::transmute::<(), Single>(()) };
+}
diff --git a/tests/ui/mir/enum/single_with_repr_break.rs b/tests/ui/mir/enum/single_with_repr_break.rs
new file mode 100644
index 00000000000..5a4ec85a9b5
--- /dev/null
+++ b/tests/ui/mir/enum/single_with_repr_break.rs
@@ -0,0 +1,13 @@
+//@ run-fail
+//@ compile-flags: -C debug-assertions
+//@ error-pattern: trying to construct an enum from an invalid value 0x1
+
+#[allow(dead_code)]
+#[repr(u16)]
+enum Single {
+    A
+}
+
+fn main() {
+    let _val: Single = unsafe { std::mem::transmute::<u16, Single>(1) };
+}
diff --git a/tests/ui/mir/enum/single_with_repr_ok.rs b/tests/ui/mir/enum/single_with_repr_ok.rs
new file mode 100644
index 00000000000..b0ed2bad660
--- /dev/null
+++ b/tests/ui/mir/enum/single_with_repr_ok.rs
@@ -0,0 +1,12 @@
+//@ run-pass
+//@ compile-flags: -C debug-assertions
+
+#[allow(dead_code)]
+#[repr(u16)]
+enum Single {
+    A
+}
+
+fn main() {
+    let _val: Single = unsafe { std::mem::transmute::<u16, Single>(0) };
+}
diff --git a/tests/ui/mir/enum/with_niche_int_break.rs b/tests/ui/mir/enum/with_niche_int_break.rs
new file mode 100644
index 00000000000..0ec60a33564
--- /dev/null
+++ b/tests/ui/mir/enum/with_niche_int_break.rs
@@ -0,0 +1,21 @@
+//@ run-fail
+//@ compile-flags: -C debug-assertions
+//@ error-pattern: trying to construct an enum from an invalid value 0x4
+
+#[allow(dead_code)]
+#[repr(u16)]
+enum Mix {
+    A,
+    B(u16),
+}
+
+#[allow(dead_code)]
+enum Nested {
+    C(Mix),
+    D,
+    E,
+}
+
+fn main() {
+    let _val: Nested = unsafe { std::mem::transmute::<u32, Nested>(4) };
+}
diff --git a/tests/ui/mir/enum/with_niche_int_ok.rs b/tests/ui/mir/enum/with_niche_int_ok.rs
new file mode 100644
index 00000000000..9a3ff3a73be
--- /dev/null
+++ b/tests/ui/mir/enum/with_niche_int_ok.rs
@@ -0,0 +1,23 @@
+//@ run-pass
+//@ compile-flags: -C debug-assertions
+
+#[allow(dead_code)]
+#[repr(u16)]
+enum Mix {
+    A,
+    B(u16),
+}
+
+#[allow(dead_code)]
+enum Nested {
+    C(Mix),
+    D,
+    E,
+}
+
+fn main() {
+    let _val: Nested = unsafe { std::mem::transmute::<u32, Nested>(0) };
+    let _val: Nested = unsafe { std::mem::transmute::<u32, Nested>(1) };
+    let _val: Nested = unsafe { std::mem::transmute::<u32, Nested>(2) };
+    let _val: Nested = unsafe { std::mem::transmute::<u32, Nested>(3) };
+}
diff --git a/tests/ui/mir/enum/with_niche_ptr_ok.rs b/tests/ui/mir/enum/with_niche_ptr_ok.rs
new file mode 100644
index 00000000000..969d955f7a4
--- /dev/null
+++ b/tests/ui/mir/enum/with_niche_ptr_ok.rs
@@ -0,0 +1,14 @@
+//@ run-pass
+//@ compile-flags: -C debug-assertions
+
+fn main() {
+    let _val = unsafe {
+        std::mem::transmute::<*const usize, Option<unsafe extern "C" fn()>>(std::ptr::null())
+    };
+    let _val = unsafe {
+        std::mem::transmute::<*const usize, Option<unsafe extern "C" fn()>>(usize::MAX as *const _)
+    };
+    let _val = unsafe { std::mem::transmute::<usize, Option<unsafe extern "C" fn()>>(0) };
+    let _val = unsafe { std::mem::transmute::<usize, Option<unsafe extern "C" fn()>>(1) };
+    let _val = unsafe { std::mem::transmute::<usize, Option<unsafe extern "C" fn()>>(usize::MAX) };
+}
diff --git a/tests/ui/mir/enum/wrap_break.rs b/tests/ui/mir/enum/wrap_break.rs
new file mode 100644
index 00000000000..4491394ca5a
--- /dev/null
+++ b/tests/ui/mir/enum/wrap_break.rs
@@ -0,0 +1,14 @@
+//@ run-fail
+//@ compile-flags: -C debug-assertions
+//@ error-pattern: trying to construct an enum from an invalid value 0x0
+#![feature(never_type)]
+#![allow(invalid_value)]
+
+#[allow(dead_code)]
+enum Wrap {
+    A(!),
+}
+
+fn main() {
+    let _val: Wrap = unsafe { std::mem::transmute::<(), Wrap>(()) };
+}
diff --git a/tests/ui/mir/enum/wrap_ok.rs b/tests/ui/mir/enum/wrap_ok.rs
new file mode 100644
index 00000000000..2881675c9ce
--- /dev/null
+++ b/tests/ui/mir/enum/wrap_ok.rs
@@ -0,0 +1,12 @@
+//@ run-pass
+//@ compile-flags: -C debug-assertions
+
+#[allow(dead_code)]
+enum Wrap {
+    A(u32),
+}
+
+fn main() {
+    let _val: Wrap = unsafe { std::mem::transmute::<u32, Wrap>(2) };
+    let _val: Wrap = unsafe { std::mem::transmute::<u32, Wrap>(u32::MAX) };
+}